1 /* 2 * Driver for the Micron P320 SSD 3 * Copyright (C) 2011 Micron Technology, Inc. 4 * 5 * Portions of this code were derived from works subjected to the 6 * following copyright: 7 * Copyright (C) 2009 Integrated Device Technology, Inc. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published by 11 * the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 */ 20 21 #include <linux/pci.h> 22 #include <linux/interrupt.h> 23 #include <linux/ata.h> 24 #include <linux/delay.h> 25 #include <linux/hdreg.h> 26 #include <linux/uaccess.h> 27 #include <linux/random.h> 28 #include <linux/smp.h> 29 #include <linux/compat.h> 30 #include <linux/fs.h> 31 #include <linux/module.h> 32 #include <linux/genhd.h> 33 #include <linux/blkdev.h> 34 #include <linux/blk-mq.h> 35 #include <linux/bio.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/idr.h> 38 #include <linux/kthread.h> 39 #include <../drivers/ata/ahci.h> 40 #include <linux/export.h> 41 #include <linux/debugfs.h> 42 #include "mtip32xx.h" 43 44 #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) 45 46 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */ 47 #define AHCI_RX_FIS_SZ 0x100 48 #define AHCI_RX_FIS_OFFSET 0x0 49 #define AHCI_IDFY_SZ ATA_SECT_SIZE 50 #define AHCI_IDFY_OFFSET 0x400 51 #define AHCI_SECTBUF_SZ ATA_SECT_SIZE 52 #define AHCI_SECTBUF_OFFSET 0x800 53 #define AHCI_SMARTBUF_SZ ATA_SECT_SIZE 54 #define AHCI_SMARTBUF_OFFSET 0xC00 55 /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */ 56 #define BLOCK_DMA_ALLOC_SZ 4096 57 58 /* DMA region containing command table (should be 8192 bytes) */ 59 #define AHCI_CMD_SLOT_SZ sizeof(struct mtip_cmd_hdr) 60 #define AHCI_CMD_TBL_SZ (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ) 61 #define AHCI_CMD_TBL_OFFSET 0x0 62 63 /* DMA region per command (contains header and SGL) */ 64 #define AHCI_CMD_TBL_HDR_SZ 0x80 65 #define AHCI_CMD_TBL_HDR_OFFSET 0x0 66 #define AHCI_CMD_TBL_SGL_SZ (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg)) 67 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ 68 #define CMD_DMA_ALLOC_SZ (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ) 69 70 71 #define HOST_CAP_NZDMA (1 << 19) 72 #define HOST_HSORG 0xFC 73 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24) 74 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) 75 #define HSORG_HWREV 0xFF00 76 #define HSORG_STYLE 0x8 77 #define HSORG_SLOTGROUPS 0x7 78 79 #define PORT_COMMAND_ISSUE 0x38 80 #define PORT_SDBV 0x7C 81 82 #define PORT_OFFSET 0x100 83 #define PORT_MEM_SIZE 0x80 84 85 #define PORT_IRQ_ERR \ 86 (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \ 87 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \ 88 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \ 89 PORT_IRQ_OVERFLOW) 90 #define PORT_IRQ_LEGACY \ 91 (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS) 92 #define PORT_IRQ_HANDLED \ 93 (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \ 94 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \ 95 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY) 96 #define DEF_PORT_IRQ \ 97 (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS) 98 99 /* product numbers */ 100 #define MTIP_PRODUCT_UNKNOWN 0x00 101 #define MTIP_PRODUCT_ASICFPGA 0x11 102 103 /* Device instance number, incremented each time a device is probed. */ 104 static int instance; 105 106 struct list_head online_list; 107 struct list_head removing_list; 108 spinlock_t dev_lock; 109 110 /* 111 * Global variable used to hold the major block device number 112 * allocated in mtip_init(). 113 */ 114 static int mtip_major; 115 static struct dentry *dfs_parent; 116 static struct dentry *dfs_device_status; 117 118 static u32 cpu_use[NR_CPUS]; 119 120 static DEFINE_SPINLOCK(rssd_index_lock); 121 static DEFINE_IDA(rssd_index_ida); 122 123 static int mtip_block_initialize(struct driver_data *dd); 124 125 #ifdef CONFIG_COMPAT 126 struct mtip_compat_ide_task_request_s { 127 __u8 io_ports[8]; 128 __u8 hob_ports[8]; 129 ide_reg_valid_t out_flags; 130 ide_reg_valid_t in_flags; 131 int data_phase; 132 int req_cmd; 133 compat_ulong_t out_size; 134 compat_ulong_t in_size; 135 }; 136 #endif 137 138 /* 139 * This function check_for_surprise_removal is called 140 * while card is removed from the system and it will 141 * read the vendor id from the configration space 142 * 143 * @pdev Pointer to the pci_dev structure. 144 * 145 * return value 146 * true if device removed, else false 147 */ 148 static bool mtip_check_surprise_removal(struct pci_dev *pdev) 149 { 150 u16 vendor_id = 0; 151 struct driver_data *dd = pci_get_drvdata(pdev); 152 153 if (dd->sr) 154 return true; 155 156 /* Read the vendorID from the configuration space */ 157 pci_read_config_word(pdev, 0x00, &vendor_id); 158 if (vendor_id == 0xFFFF) { 159 dd->sr = true; 160 if (dd->queue) 161 set_bit(QUEUE_FLAG_DEAD, &dd->queue->queue_flags); 162 else 163 dev_warn(&dd->pdev->dev, 164 "%s: dd->queue is NULL\n", __func__); 165 if (dd->port) { 166 set_bit(MTIP_PF_SR_CLEANUP_BIT, &dd->port->flags); 167 wake_up_interruptible(&dd->port->svc_wait); 168 } else 169 dev_warn(&dd->pdev->dev, 170 "%s: dd->port is NULL\n", __func__); 171 return true; /* device removed */ 172 } 173 174 return false; /* device present */ 175 } 176 177 static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) 178 { 179 struct request *rq; 180 181 rq = blk_mq_alloc_request(dd->queue, 0, __GFP_WAIT, true); 182 return blk_mq_rq_to_pdu(rq); 183 } 184 185 static void mtip_put_int_command(struct driver_data *dd, struct mtip_cmd *cmd) 186 { 187 blk_put_request(blk_mq_rq_from_pdu(cmd)); 188 } 189 190 /* 191 * Once we add support for one hctx per mtip group, this will change a bit 192 */ 193 static struct request *mtip_rq_from_tag(struct driver_data *dd, 194 unsigned int tag) 195 { 196 return blk_mq_tag_to_rq(dd->queue->queue_hw_ctx[0], tag); 197 } 198 199 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, 200 unsigned int tag) 201 { 202 struct request *rq = mtip_rq_from_tag(dd, tag); 203 204 return blk_mq_rq_to_pdu(rq); 205 } 206 207 /* 208 * IO completion function. 209 * 210 * This completion function is called by the driver ISR when a 211 * command that was issued by the kernel completes. It first calls the 212 * asynchronous completion function which normally calls back into the block 213 * layer passing the asynchronous callback data, then unmaps the 214 * scatter list associated with the completed command, and finally 215 * clears the allocated bit associated with the completed command. 216 * 217 * @port Pointer to the port data structure. 218 * @tag Tag of the command. 219 * @data Pointer to driver_data. 220 * @status Completion status. 221 * 222 * return value 223 * None 224 */ 225 static void mtip_async_complete(struct mtip_port *port, 226 int tag, struct mtip_cmd *cmd, int status) 227 { 228 struct driver_data *dd = port->dd; 229 struct request *rq; 230 231 if (unlikely(!dd) || unlikely(!port)) 232 return; 233 234 if (unlikely(status == PORT_IRQ_TF_ERR)) { 235 dev_warn(&port->dd->pdev->dev, 236 "Command tag %d failed due to TFE\n", tag); 237 } 238 239 /* Unmap the DMA scatter list entries */ 240 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, cmd->direction); 241 242 rq = mtip_rq_from_tag(dd, tag); 243 244 if (unlikely(cmd->unaligned)) 245 up(&port->cmd_slot_unal); 246 247 blk_mq_end_io(rq, status ? -EIO : 0); 248 } 249 250 /* 251 * Reset the HBA (without sleeping) 252 * 253 * @dd Pointer to the driver data structure. 254 * 255 * return value 256 * 0 The reset was successful. 257 * -1 The HBA Reset bit did not clear. 258 */ 259 static int mtip_hba_reset(struct driver_data *dd) 260 { 261 unsigned long timeout; 262 263 /* Set the reset bit */ 264 writel(HOST_RESET, dd->mmio + HOST_CTL); 265 266 /* Flush */ 267 readl(dd->mmio + HOST_CTL); 268 269 /* Spin for up to 2 seconds, waiting for reset acknowledgement */ 270 timeout = jiffies + msecs_to_jiffies(2000); 271 do { 272 mdelay(10); 273 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) 274 return -1; 275 276 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) 277 && time_before(jiffies, timeout)); 278 279 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) 280 return -1; 281 282 return 0; 283 } 284 285 /* 286 * Issue a command to the hardware. 287 * 288 * Set the appropriate bit in the s_active and Command Issue hardware 289 * registers, causing hardware command processing to begin. 290 * 291 * @port Pointer to the port structure. 292 * @tag The tag of the command to be issued. 293 * 294 * return value 295 * None 296 */ 297 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) 298 { 299 int group = tag >> 5; 300 301 /* guard SACT and CI registers */ 302 spin_lock(&port->cmd_issue_lock[group]); 303 writel((1 << MTIP_TAG_BIT(tag)), 304 port->s_active[MTIP_TAG_INDEX(tag)]); 305 writel((1 << MTIP_TAG_BIT(tag)), 306 port->cmd_issue[MTIP_TAG_INDEX(tag)]); 307 spin_unlock(&port->cmd_issue_lock[group]); 308 } 309 310 /* 311 * Enable/disable the reception of FIS 312 * 313 * @port Pointer to the port data structure 314 * @enable 1 to enable, 0 to disable 315 * 316 * return value 317 * Previous state: 1 enabled, 0 disabled 318 */ 319 static int mtip_enable_fis(struct mtip_port *port, int enable) 320 { 321 u32 tmp; 322 323 /* enable FIS reception */ 324 tmp = readl(port->mmio + PORT_CMD); 325 if (enable) 326 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 327 else 328 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 329 330 /* Flush */ 331 readl(port->mmio + PORT_CMD); 332 333 return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX)); 334 } 335 336 /* 337 * Enable/disable the DMA engine 338 * 339 * @port Pointer to the port data structure 340 * @enable 1 to enable, 0 to disable 341 * 342 * return value 343 * Previous state: 1 enabled, 0 disabled. 344 */ 345 static int mtip_enable_engine(struct mtip_port *port, int enable) 346 { 347 u32 tmp; 348 349 /* enable FIS reception */ 350 tmp = readl(port->mmio + PORT_CMD); 351 if (enable) 352 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); 353 else 354 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); 355 356 readl(port->mmio + PORT_CMD); 357 return (((tmp & PORT_CMD_START) == PORT_CMD_START)); 358 } 359 360 /* 361 * Enables the port DMA engine and FIS reception. 362 * 363 * return value 364 * None 365 */ 366 static inline void mtip_start_port(struct mtip_port *port) 367 { 368 /* Enable FIS reception */ 369 mtip_enable_fis(port, 1); 370 371 /* Enable the DMA engine */ 372 mtip_enable_engine(port, 1); 373 } 374 375 /* 376 * Deinitialize a port by disabling port interrupts, the DMA engine, 377 * and FIS reception. 378 * 379 * @port Pointer to the port structure 380 * 381 * return value 382 * None 383 */ 384 static inline void mtip_deinit_port(struct mtip_port *port) 385 { 386 /* Disable interrupts on this port */ 387 writel(0, port->mmio + PORT_IRQ_MASK); 388 389 /* Disable the DMA engine */ 390 mtip_enable_engine(port, 0); 391 392 /* Disable FIS reception */ 393 mtip_enable_fis(port, 0); 394 } 395 396 /* 397 * Initialize a port. 398 * 399 * This function deinitializes the port by calling mtip_deinit_port() and 400 * then initializes it by setting the command header and RX FIS addresses, 401 * clearing the SError register and any pending port interrupts before 402 * re-enabling the default set of port interrupts. 403 * 404 * @port Pointer to the port structure. 405 * 406 * return value 407 * None 408 */ 409 static void mtip_init_port(struct mtip_port *port) 410 { 411 int i; 412 mtip_deinit_port(port); 413 414 /* Program the command list base and FIS base addresses */ 415 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { 416 writel((port->command_list_dma >> 16) >> 16, 417 port->mmio + PORT_LST_ADDR_HI); 418 writel((port->rxfis_dma >> 16) >> 16, 419 port->mmio + PORT_FIS_ADDR_HI); 420 } 421 422 writel(port->command_list_dma & 0xFFFFFFFF, 423 port->mmio + PORT_LST_ADDR); 424 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); 425 426 /* Clear SError */ 427 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); 428 429 /* reset the completed registers.*/ 430 for (i = 0; i < port->dd->slot_groups; i++) 431 writel(0xFFFFFFFF, port->completed[i]); 432 433 /* Clear any pending interrupts for this port */ 434 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); 435 436 /* Clear any pending interrupts on the HBA. */ 437 writel(readl(port->dd->mmio + HOST_IRQ_STAT), 438 port->dd->mmio + HOST_IRQ_STAT); 439 440 /* Enable port interrupts */ 441 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); 442 } 443 444 /* 445 * Restart a port 446 * 447 * @port Pointer to the port data structure. 448 * 449 * return value 450 * None 451 */ 452 static void mtip_restart_port(struct mtip_port *port) 453 { 454 unsigned long timeout; 455 456 /* Disable the DMA engine */ 457 mtip_enable_engine(port, 0); 458 459 /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */ 460 timeout = jiffies + msecs_to_jiffies(500); 461 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) 462 && time_before(jiffies, timeout)) 463 ; 464 465 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 466 return; 467 468 /* 469 * Chip quirk: escalate to hba reset if 470 * PxCMD.CR not clear after 500 ms 471 */ 472 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { 473 dev_warn(&port->dd->pdev->dev, 474 "PxCMD.CR not clear, escalating reset\n"); 475 476 if (mtip_hba_reset(port->dd)) 477 dev_err(&port->dd->pdev->dev, 478 "HBA reset escalation failed.\n"); 479 480 /* 30 ms delay before com reset to quiesce chip */ 481 mdelay(30); 482 } 483 484 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); 485 486 /* Set PxSCTL.DET */ 487 writel(readl(port->mmio + PORT_SCR_CTL) | 488 1, port->mmio + PORT_SCR_CTL); 489 readl(port->mmio + PORT_SCR_CTL); 490 491 /* Wait 1 ms to quiesce chip function */ 492 timeout = jiffies + msecs_to_jiffies(1); 493 while (time_before(jiffies, timeout)) 494 ; 495 496 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 497 return; 498 499 /* Clear PxSCTL.DET */ 500 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, 501 port->mmio + PORT_SCR_CTL); 502 readl(port->mmio + PORT_SCR_CTL); 503 504 /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */ 505 timeout = jiffies + msecs_to_jiffies(500); 506 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 507 && time_before(jiffies, timeout)) 508 ; 509 510 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 511 return; 512 513 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 514 dev_warn(&port->dd->pdev->dev, 515 "COM reset failed\n"); 516 517 mtip_init_port(port); 518 mtip_start_port(port); 519 520 } 521 522 static int mtip_device_reset(struct driver_data *dd) 523 { 524 int rv = 0; 525 526 if (mtip_check_surprise_removal(dd->pdev)) 527 return 0; 528 529 if (mtip_hba_reset(dd) < 0) 530 rv = -EFAULT; 531 532 mdelay(1); 533 mtip_init_port(dd->port); 534 mtip_start_port(dd->port); 535 536 /* Enable interrupts on the HBA. */ 537 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 538 dd->mmio + HOST_CTL); 539 return rv; 540 } 541 542 /* 543 * Helper function for tag logging 544 */ 545 static void print_tags(struct driver_data *dd, 546 char *msg, 547 unsigned long *tagbits, 548 int cnt) 549 { 550 unsigned char tagmap[128]; 551 int group, tagmap_len = 0; 552 553 memset(tagmap, 0, sizeof(tagmap)); 554 for (group = SLOTBITS_IN_LONGS; group > 0; group--) 555 tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ", 556 tagbits[group-1]); 557 dev_warn(&dd->pdev->dev, 558 "%d command(s) %s: tagmap [%s]", cnt, msg, tagmap); 559 } 560 561 /* 562 * Internal command completion callback function. 563 * 564 * This function is normally called by the driver ISR when an internal 565 * command completed. This function signals the command completion by 566 * calling complete(). 567 * 568 * @port Pointer to the port data structure. 569 * @tag Tag of the command that has completed. 570 * @data Pointer to a completion structure. 571 * @status Completion status. 572 * 573 * return value 574 * None 575 */ 576 static void mtip_completion(struct mtip_port *port, 577 int tag, struct mtip_cmd *command, int status) 578 { 579 struct completion *waiting = command->comp_data; 580 if (unlikely(status == PORT_IRQ_TF_ERR)) 581 dev_warn(&port->dd->pdev->dev, 582 "Internal command %d completed with TFE\n", tag); 583 584 complete(waiting); 585 } 586 587 static void mtip_null_completion(struct mtip_port *port, 588 int tag, struct mtip_cmd *command, int status) 589 { 590 } 591 592 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 593 dma_addr_t buffer_dma, unsigned int sectors); 594 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 595 struct smart_attr *attrib); 596 /* 597 * Handle an error. 598 * 599 * @dd Pointer to the DRIVER_DATA structure. 600 * 601 * return value 602 * None 603 */ 604 static void mtip_handle_tfe(struct driver_data *dd) 605 { 606 int group, tag, bit, reissue, rv; 607 struct mtip_port *port; 608 struct mtip_cmd *cmd; 609 u32 completed; 610 struct host_to_dev_fis *fis; 611 unsigned long tagaccum[SLOTBITS_IN_LONGS]; 612 unsigned int cmd_cnt = 0; 613 unsigned char *buf; 614 char *fail_reason = NULL; 615 int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0; 616 617 dev_warn(&dd->pdev->dev, "Taskfile error\n"); 618 619 port = dd->port; 620 621 set_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 622 623 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && 624 test_bit(MTIP_TAG_INTERNAL, port->allocated)) { 625 cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); 626 dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n"); 627 628 if (cmd->comp_data && cmd->comp_func) { 629 cmd->comp_func(port, MTIP_TAG_INTERNAL, 630 cmd, PORT_IRQ_TF_ERR); 631 } 632 goto handle_tfe_exit; 633 } 634 635 /* clear the tag accumulator */ 636 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 637 638 /* Loop through all the groups */ 639 for (group = 0; group < dd->slot_groups; group++) { 640 completed = readl(port->completed[group]); 641 642 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); 643 644 /* clear completed status register in the hardware.*/ 645 writel(completed, port->completed[group]); 646 647 /* Process successfully completed commands */ 648 for (bit = 0; bit < 32 && completed; bit++) { 649 if (!(completed & (1<<bit))) 650 continue; 651 tag = (group << 5) + bit; 652 653 /* Skip the internal command slot */ 654 if (tag == MTIP_TAG_INTERNAL) 655 continue; 656 657 cmd = mtip_cmd_from_tag(dd, tag); 658 if (likely(cmd->comp_func)) { 659 set_bit(tag, tagaccum); 660 cmd_cnt++; 661 cmd->comp_func(port, tag, cmd, 0); 662 } else { 663 dev_err(&port->dd->pdev->dev, 664 "Missing completion func for tag %d", 665 tag); 666 if (mtip_check_surprise_removal(dd->pdev)) { 667 /* don't proceed further */ 668 return; 669 } 670 } 671 } 672 } 673 674 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); 675 676 /* Restart the port */ 677 mdelay(20); 678 mtip_restart_port(port); 679 680 /* Trying to determine the cause of the error */ 681 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 682 dd->port->log_buf, 683 dd->port->log_buf_dma, 1); 684 if (rv) { 685 dev_warn(&dd->pdev->dev, 686 "Error in READ LOG EXT (10h) command\n"); 687 /* non-critical error, don't fail the load */ 688 } else { 689 buf = (unsigned char *)dd->port->log_buf; 690 if (buf[259] & 0x1) { 691 dev_info(&dd->pdev->dev, 692 "Write protect bit is set.\n"); 693 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 694 fail_all_ncq_write = 1; 695 fail_reason = "write protect"; 696 } 697 if (buf[288] == 0xF7) { 698 dev_info(&dd->pdev->dev, 699 "Exceeded Tmax, drive in thermal shutdown.\n"); 700 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 701 fail_all_ncq_cmds = 1; 702 fail_reason = "thermal shutdown"; 703 } 704 if (buf[288] == 0xBF) { 705 set_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); 706 dev_info(&dd->pdev->dev, 707 "Drive indicates rebuild has failed. Secure erase required.\n"); 708 fail_all_ncq_cmds = 1; 709 fail_reason = "rebuild failed"; 710 } 711 } 712 713 /* clear the tag accumulator */ 714 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 715 716 /* Loop through all the groups */ 717 for (group = 0; group < dd->slot_groups; group++) { 718 for (bit = 0; bit < 32; bit++) { 719 reissue = 1; 720 tag = (group << 5) + bit; 721 cmd = mtip_cmd_from_tag(dd, tag); 722 723 fis = (struct host_to_dev_fis *)cmd->command; 724 725 /* Should re-issue? */ 726 if (tag == MTIP_TAG_INTERNAL || 727 fis->command == ATA_CMD_SET_FEATURES) 728 reissue = 0; 729 else { 730 if (fail_all_ncq_cmds || 731 (fail_all_ncq_write && 732 fis->command == ATA_CMD_FPDMA_WRITE)) { 733 dev_warn(&dd->pdev->dev, 734 " Fail: %s w/tag %d [%s].\n", 735 fis->command == ATA_CMD_FPDMA_WRITE ? 736 "write" : "read", 737 tag, 738 fail_reason != NULL ? 739 fail_reason : "unknown"); 740 if (cmd->comp_func) { 741 cmd->comp_func(port, tag, 742 cmd, -ENODATA); 743 } 744 continue; 745 } 746 } 747 748 /* 749 * First check if this command has 750 * exceeded its retries. 751 */ 752 if (reissue && (cmd->retries-- > 0)) { 753 754 set_bit(tag, tagaccum); 755 756 /* Re-issue the command. */ 757 mtip_issue_ncq_command(port, tag); 758 759 continue; 760 } 761 762 /* Retire a command that will not be reissued */ 763 dev_warn(&port->dd->pdev->dev, 764 "retiring tag %d\n", tag); 765 766 if (cmd->comp_func) 767 cmd->comp_func(port, tag, cmd, PORT_IRQ_TF_ERR); 768 else 769 dev_warn(&port->dd->pdev->dev, 770 "Bad completion for tag %d\n", 771 tag); 772 } 773 } 774 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); 775 776 handle_tfe_exit: 777 /* clear eh_active */ 778 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 779 wake_up_interruptible(&port->svc_wait); 780 } 781 782 /* 783 * Handle a set device bits interrupt 784 */ 785 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group, 786 u32 completed) 787 { 788 struct driver_data *dd = port->dd; 789 int tag, bit; 790 struct mtip_cmd *command; 791 792 if (!completed) { 793 WARN_ON_ONCE(!completed); 794 return; 795 } 796 /* clear completed status register in the hardware.*/ 797 writel(completed, port->completed[group]); 798 799 /* Process completed commands. */ 800 for (bit = 0; (bit < 32) && completed; bit++) { 801 if (completed & 0x01) { 802 tag = (group << 5) | bit; 803 804 /* skip internal command slot. */ 805 if (unlikely(tag == MTIP_TAG_INTERNAL)) 806 continue; 807 808 command = mtip_cmd_from_tag(dd, tag); 809 if (likely(command->comp_func)) 810 command->comp_func(port, tag, command, 0); 811 else { 812 dev_dbg(&dd->pdev->dev, 813 "Null completion for tag %d", 814 tag); 815 816 if (mtip_check_surprise_removal( 817 dd->pdev)) { 818 return; 819 } 820 } 821 } 822 completed >>= 1; 823 } 824 825 /* If last, re-enable interrupts */ 826 if (atomic_dec_return(&dd->irq_workers_active) == 0) 827 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); 828 } 829 830 /* 831 * Process legacy pio and d2h interrupts 832 */ 833 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) 834 { 835 struct mtip_port *port = dd->port; 836 struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); 837 838 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && 839 (cmd != NULL) && !(readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 840 & (1 << MTIP_TAG_INTERNAL))) { 841 if (cmd->comp_func) { 842 cmd->comp_func(port, MTIP_TAG_INTERNAL, cmd, 0); 843 return; 844 } 845 } 846 847 return; 848 } 849 850 /* 851 * Demux and handle errors 852 */ 853 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) 854 { 855 856 if (unlikely(port_stat & PORT_IRQ_CONNECT)) { 857 dev_warn(&dd->pdev->dev, 858 "Clearing PxSERR.DIAG.x\n"); 859 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); 860 } 861 862 if (unlikely(port_stat & PORT_IRQ_PHYRDY)) { 863 dev_warn(&dd->pdev->dev, 864 "Clearing PxSERR.DIAG.n\n"); 865 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); 866 } 867 868 if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) { 869 dev_warn(&dd->pdev->dev, 870 "Port stat errors %x unhandled\n", 871 (port_stat & ~PORT_IRQ_HANDLED)); 872 if (mtip_check_surprise_removal(dd->pdev)) 873 return; 874 } 875 if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) { 876 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); 877 wake_up_interruptible(&dd->port->svc_wait); 878 } 879 } 880 881 static inline irqreturn_t mtip_handle_irq(struct driver_data *data) 882 { 883 struct driver_data *dd = (struct driver_data *) data; 884 struct mtip_port *port = dd->port; 885 u32 hba_stat, port_stat; 886 int rv = IRQ_NONE; 887 int do_irq_enable = 1, i, workers; 888 struct mtip_work *twork; 889 890 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); 891 if (hba_stat) { 892 rv = IRQ_HANDLED; 893 894 /* Acknowledge the interrupt status on the port.*/ 895 port_stat = readl(port->mmio + PORT_IRQ_STAT); 896 writel(port_stat, port->mmio + PORT_IRQ_STAT); 897 898 /* Demux port status */ 899 if (likely(port_stat & PORT_IRQ_SDB_FIS)) { 900 do_irq_enable = 0; 901 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); 902 903 /* Start at 1: group zero is always local? */ 904 for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS; 905 i++) { 906 twork = &dd->work[i]; 907 twork->completed = readl(port->completed[i]); 908 if (twork->completed) 909 workers++; 910 } 911 912 atomic_set(&dd->irq_workers_active, workers); 913 if (workers) { 914 for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) { 915 twork = &dd->work[i]; 916 if (twork->completed) 917 queue_work_on( 918 twork->cpu_binding, 919 dd->isr_workq, 920 &twork->work); 921 } 922 923 if (likely(dd->work[0].completed)) 924 mtip_workq_sdbfx(port, 0, 925 dd->work[0].completed); 926 927 } else { 928 /* 929 * Chip quirk: SDB interrupt but nothing 930 * to complete 931 */ 932 do_irq_enable = 1; 933 } 934 } 935 936 if (unlikely(port_stat & PORT_IRQ_ERR)) { 937 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 938 /* don't proceed further */ 939 return IRQ_HANDLED; 940 } 941 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 942 &dd->dd_flag)) 943 return rv; 944 945 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); 946 } 947 948 if (unlikely(port_stat & PORT_IRQ_LEGACY)) 949 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); 950 } 951 952 /* acknowledge interrupt */ 953 if (unlikely(do_irq_enable)) 954 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); 955 956 return rv; 957 } 958 959 /* 960 * HBA interrupt subroutine. 961 * 962 * @irq IRQ number. 963 * @instance Pointer to the driver data structure. 964 * 965 * return value 966 * IRQ_HANDLED A HBA interrupt was pending and handled. 967 * IRQ_NONE This interrupt was not for the HBA. 968 */ 969 static irqreturn_t mtip_irq_handler(int irq, void *instance) 970 { 971 struct driver_data *dd = instance; 972 973 return mtip_handle_irq(dd); 974 } 975 976 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) 977 { 978 writel(1 << MTIP_TAG_BIT(tag), 979 port->cmd_issue[MTIP_TAG_INDEX(tag)]); 980 } 981 982 static bool mtip_pause_ncq(struct mtip_port *port, 983 struct host_to_dev_fis *fis) 984 { 985 struct host_to_dev_fis *reply; 986 unsigned long task_file_data; 987 988 reply = port->rxfis + RX_FIS_D2H_REG; 989 task_file_data = readl(port->mmio+PORT_TFDATA); 990 991 if (fis->command == ATA_CMD_SEC_ERASE_UNIT) 992 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 993 994 if ((task_file_data & 1)) 995 return false; 996 997 if (fis->command == ATA_CMD_SEC_ERASE_PREP) { 998 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 999 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1000 port->ic_pause_timer = jiffies; 1001 return true; 1002 } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) && 1003 (fis->features == 0x03)) { 1004 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 1005 port->ic_pause_timer = jiffies; 1006 return true; 1007 } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) || 1008 ((fis->command == 0xFC) && 1009 (fis->features == 0x27 || fis->features == 0x72 || 1010 fis->features == 0x62 || fis->features == 0x26))) { 1011 /* Com reset after secure erase or lowlevel format */ 1012 mtip_restart_port(port); 1013 return false; 1014 } 1015 1016 return false; 1017 } 1018 1019 /* 1020 * Wait for port to quiesce 1021 * 1022 * @port Pointer to port data structure 1023 * @timeout Max duration to wait (ms) 1024 * 1025 * return value 1026 * 0 Success 1027 * -EBUSY Commands still active 1028 */ 1029 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) 1030 { 1031 unsigned long to; 1032 unsigned int n; 1033 unsigned int active = 1; 1034 1035 blk_mq_stop_hw_queues(port->dd->queue); 1036 1037 to = jiffies + msecs_to_jiffies(timeout); 1038 do { 1039 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && 1040 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 1041 msleep(20); 1042 continue; /* svc thd is actively issuing commands */ 1043 } 1044 1045 msleep(100); 1046 if (mtip_check_surprise_removal(port->dd->pdev)) 1047 goto err_fault; 1048 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 1049 goto err_fault; 1050 1051 /* 1052 * Ignore s_active bit 0 of array element 0. 1053 * This bit will always be set 1054 */ 1055 active = readl(port->s_active[0]) & 0xFFFFFFFE; 1056 for (n = 1; n < port->dd->slot_groups; n++) 1057 active |= readl(port->s_active[n]); 1058 1059 if (!active) 1060 break; 1061 } while (time_before(jiffies, to)); 1062 1063 blk_mq_start_stopped_hw_queues(port->dd->queue, true); 1064 return active ? -EBUSY : 0; 1065 err_fault: 1066 blk_mq_start_stopped_hw_queues(port->dd->queue, true); 1067 return -EFAULT; 1068 } 1069 1070 /* 1071 * Execute an internal command and wait for the completion. 1072 * 1073 * @port Pointer to the port data structure. 1074 * @fis Pointer to the FIS that describes the command. 1075 * @fis_len Length in WORDS of the FIS. 1076 * @buffer DMA accessible for command data. 1077 * @buf_len Length, in bytes, of the data buffer. 1078 * @opts Command header options, excluding the FIS length 1079 * and the number of PRD entries. 1080 * @timeout Time in ms to wait for the command to complete. 1081 * 1082 * return value 1083 * 0 Command completed successfully. 1084 * -EFAULT The buffer address is not correctly aligned. 1085 * -EBUSY Internal command or other IO in progress. 1086 * -EAGAIN Time out waiting for command to complete. 1087 */ 1088 static int mtip_exec_internal_command(struct mtip_port *port, 1089 struct host_to_dev_fis *fis, 1090 int fis_len, 1091 dma_addr_t buffer, 1092 int buf_len, 1093 u32 opts, 1094 gfp_t atomic, 1095 unsigned long timeout) 1096 { 1097 struct mtip_cmd_sg *command_sg; 1098 DECLARE_COMPLETION_ONSTACK(wait); 1099 struct mtip_cmd *int_cmd; 1100 struct driver_data *dd = port->dd; 1101 int rv = 0; 1102 1103 /* Make sure the buffer is 8 byte aligned. This is asic specific. */ 1104 if (buffer & 0x00000007) { 1105 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); 1106 return -EFAULT; 1107 } 1108 1109 int_cmd = mtip_get_int_command(dd); 1110 1111 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1112 port->ic_pause_timer = 0; 1113 1114 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 1115 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 1116 1117 if (atomic == GFP_KERNEL) { 1118 if (fis->command != ATA_CMD_STANDBYNOW1) { 1119 /* wait for io to complete if non atomic */ 1120 if (mtip_quiesce_io(port, 1121 MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) { 1122 dev_warn(&dd->pdev->dev, 1123 "Failed to quiesce IO\n"); 1124 mtip_put_int_command(dd, int_cmd); 1125 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1126 wake_up_interruptible(&port->svc_wait); 1127 return -EBUSY; 1128 } 1129 } 1130 1131 /* Set the completion function and data for the command. */ 1132 int_cmd->comp_data = &wait; 1133 int_cmd->comp_func = mtip_completion; 1134 1135 } else { 1136 /* Clear completion - we're going to poll */ 1137 int_cmd->comp_data = NULL; 1138 int_cmd->comp_func = mtip_null_completion; 1139 } 1140 1141 /* Copy the command to the command table */ 1142 memcpy(int_cmd->command, fis, fis_len*4); 1143 1144 /* Populate the SG list */ 1145 int_cmd->command_header->opts = 1146 __force_bit2int cpu_to_le32(opts | fis_len); 1147 if (buf_len) { 1148 command_sg = int_cmd->command + AHCI_CMD_TBL_HDR_SZ; 1149 1150 command_sg->info = 1151 __force_bit2int cpu_to_le32((buf_len-1) & 0x3FFFFF); 1152 command_sg->dba = 1153 __force_bit2int cpu_to_le32(buffer & 0xFFFFFFFF); 1154 command_sg->dba_upper = 1155 __force_bit2int cpu_to_le32((buffer >> 16) >> 16); 1156 1157 int_cmd->command_header->opts |= 1158 __force_bit2int cpu_to_le32((1 << 16)); 1159 } 1160 1161 /* Populate the command header */ 1162 int_cmd->command_header->byte_count = 0; 1163 1164 /* Issue the command to the hardware */ 1165 mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL); 1166 1167 if (atomic == GFP_KERNEL) { 1168 /* Wait for the command to complete or timeout. */ 1169 if ((rv = wait_for_completion_interruptible_timeout( 1170 &wait, 1171 msecs_to_jiffies(timeout))) <= 0) { 1172 if (rv == -ERESTARTSYS) { /* interrupted */ 1173 dev_err(&dd->pdev->dev, 1174 "Internal command [%02X] was interrupted after %lu ms\n", 1175 fis->command, timeout); 1176 rv = -EINTR; 1177 goto exec_ic_exit; 1178 } else if (rv == 0) /* timeout */ 1179 dev_err(&dd->pdev->dev, 1180 "Internal command did not complete [%02X] within timeout of %lu ms\n", 1181 fis->command, timeout); 1182 else 1183 dev_err(&dd->pdev->dev, 1184 "Internal command [%02X] wait returned code [%d] after %lu ms - unhandled\n", 1185 fis->command, rv, timeout); 1186 1187 if (mtip_check_surprise_removal(dd->pdev) || 1188 test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1189 &dd->dd_flag)) { 1190 dev_err(&dd->pdev->dev, 1191 "Internal command [%02X] wait returned due to SR\n", 1192 fis->command); 1193 rv = -ENXIO; 1194 goto exec_ic_exit; 1195 } 1196 mtip_device_reset(dd); /* recover from timeout issue */ 1197 rv = -EAGAIN; 1198 goto exec_ic_exit; 1199 } 1200 } else { 1201 u32 hba_stat, port_stat; 1202 1203 /* Spin for <timeout> checking if command still outstanding */ 1204 timeout = jiffies + msecs_to_jiffies(timeout); 1205 while ((readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 1206 & (1 << MTIP_TAG_INTERNAL)) 1207 && time_before(jiffies, timeout)) { 1208 if (mtip_check_surprise_removal(dd->pdev)) { 1209 rv = -ENXIO; 1210 goto exec_ic_exit; 1211 } 1212 if ((fis->command != ATA_CMD_STANDBYNOW1) && 1213 test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1214 &dd->dd_flag)) { 1215 rv = -ENXIO; 1216 goto exec_ic_exit; 1217 } 1218 port_stat = readl(port->mmio + PORT_IRQ_STAT); 1219 if (!port_stat) 1220 continue; 1221 1222 if (port_stat & PORT_IRQ_ERR) { 1223 dev_err(&dd->pdev->dev, 1224 "Internal command [%02X] failed\n", 1225 fis->command); 1226 mtip_device_reset(dd); 1227 rv = -EIO; 1228 goto exec_ic_exit; 1229 } else { 1230 writel(port_stat, port->mmio + PORT_IRQ_STAT); 1231 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); 1232 if (hba_stat) 1233 writel(hba_stat, 1234 dd->mmio + HOST_IRQ_STAT); 1235 } 1236 break; 1237 } 1238 } 1239 1240 if (readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 1241 & (1 << MTIP_TAG_INTERNAL)) { 1242 rv = -ENXIO; 1243 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 1244 mtip_device_reset(dd); 1245 rv = -EAGAIN; 1246 } 1247 } 1248 exec_ic_exit: 1249 /* Clear the allocated and active bits for the internal command. */ 1250 mtip_put_int_command(dd, int_cmd); 1251 if (rv >= 0 && mtip_pause_ncq(port, fis)) { 1252 /* NCQ paused */ 1253 return rv; 1254 } 1255 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1256 wake_up_interruptible(&port->svc_wait); 1257 1258 return rv; 1259 } 1260 1261 /* 1262 * Byte-swap ATA ID strings. 1263 * 1264 * ATA identify data contains strings in byte-swapped 16-bit words. 1265 * They must be swapped (on all architectures) to be usable as C strings. 1266 * This function swaps bytes in-place. 1267 * 1268 * @buf The buffer location of the string 1269 * @len The number of bytes to swap 1270 * 1271 * return value 1272 * None 1273 */ 1274 static inline void ata_swap_string(u16 *buf, unsigned int len) 1275 { 1276 int i; 1277 for (i = 0; i < (len/2); i++) 1278 be16_to_cpus(&buf[i]); 1279 } 1280 1281 static void mtip_set_timeout(struct driver_data *dd, 1282 struct host_to_dev_fis *fis, 1283 unsigned int *timeout, u8 erasemode) 1284 { 1285 switch (fis->command) { 1286 case ATA_CMD_DOWNLOAD_MICRO: 1287 *timeout = 120000; /* 2 minutes */ 1288 break; 1289 case ATA_CMD_SEC_ERASE_UNIT: 1290 case 0xFC: 1291 if (erasemode) 1292 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); 1293 else 1294 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); 1295 break; 1296 case ATA_CMD_STANDBYNOW1: 1297 *timeout = 120000; /* 2 minutes */ 1298 break; 1299 case 0xF7: 1300 case 0xFA: 1301 *timeout = 60000; /* 60 seconds */ 1302 break; 1303 case ATA_CMD_SMART: 1304 *timeout = 15000; /* 15 seconds */ 1305 break; 1306 default: 1307 *timeout = MTIP_IOCTL_CMD_TIMEOUT_MS; 1308 break; 1309 } 1310 } 1311 1312 /* 1313 * Request the device identity information. 1314 * 1315 * If a user space buffer is not specified, i.e. is NULL, the 1316 * identify information is still read from the drive and placed 1317 * into the identify data buffer (@e port->identify) in the 1318 * port data structure. 1319 * When the identify buffer contains valid identify information @e 1320 * port->identify_valid is non-zero. 1321 * 1322 * @port Pointer to the port structure. 1323 * @user_buffer A user space buffer where the identify data should be 1324 * copied. 1325 * 1326 * return value 1327 * 0 Command completed successfully. 1328 * -EFAULT An error occurred while coping data to the user buffer. 1329 * -1 Command failed. 1330 */ 1331 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer) 1332 { 1333 int rv = 0; 1334 struct host_to_dev_fis fis; 1335 1336 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 1337 return -EFAULT; 1338 1339 /* Build the FIS. */ 1340 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1341 fis.type = 0x27; 1342 fis.opts = 1 << 7; 1343 fis.command = ATA_CMD_ID_ATA; 1344 1345 /* Set the identify information as invalid. */ 1346 port->identify_valid = 0; 1347 1348 /* Clear the identify information. */ 1349 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); 1350 1351 /* Execute the command. */ 1352 if (mtip_exec_internal_command(port, 1353 &fis, 1354 5, 1355 port->identify_dma, 1356 sizeof(u16) * ATA_ID_WORDS, 1357 0, 1358 GFP_KERNEL, 1359 MTIP_INT_CMD_TIMEOUT_MS) 1360 < 0) { 1361 rv = -1; 1362 goto out; 1363 } 1364 1365 /* 1366 * Perform any necessary byte-swapping. Yes, the kernel does in fact 1367 * perform field-sensitive swapping on the string fields. 1368 * See the kernel use of ata_id_string() for proof of this. 1369 */ 1370 #ifdef __LITTLE_ENDIAN 1371 ata_swap_string(port->identify + 27, 40); /* model string*/ 1372 ata_swap_string(port->identify + 23, 8); /* firmware string*/ 1373 ata_swap_string(port->identify + 10, 20); /* serial# string*/ 1374 #else 1375 { 1376 int i; 1377 for (i = 0; i < ATA_ID_WORDS; i++) 1378 port->identify[i] = le16_to_cpu(port->identify[i]); 1379 } 1380 #endif 1381 1382 /* Check security locked state */ 1383 if (port->identify[128] & 0x4) 1384 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1385 else 1386 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1387 1388 #ifdef MTIP_TRIM /* Disabling TRIM support temporarily */ 1389 /* Demux ID.DRAT & ID.RZAT to determine trim support */ 1390 if (port->identify[69] & (1 << 14) && port->identify[69] & (1 << 5)) 1391 port->dd->trim_supp = true; 1392 else 1393 #endif 1394 port->dd->trim_supp = false; 1395 1396 /* Set the identify buffer as valid. */ 1397 port->identify_valid = 1; 1398 1399 if (user_buffer) { 1400 if (copy_to_user( 1401 user_buffer, 1402 port->identify, 1403 ATA_ID_WORDS * sizeof(u16))) { 1404 rv = -EFAULT; 1405 goto out; 1406 } 1407 } 1408 1409 out: 1410 return rv; 1411 } 1412 1413 /* 1414 * Issue a standby immediate command to the device. 1415 * 1416 * @port Pointer to the port structure. 1417 * 1418 * return value 1419 * 0 Command was executed successfully. 1420 * -1 An error occurred while executing the command. 1421 */ 1422 static int mtip_standby_immediate(struct mtip_port *port) 1423 { 1424 int rv; 1425 struct host_to_dev_fis fis; 1426 unsigned long start; 1427 unsigned int timeout; 1428 1429 /* Build the FIS. */ 1430 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1431 fis.type = 0x27; 1432 fis.opts = 1 << 7; 1433 fis.command = ATA_CMD_STANDBYNOW1; 1434 1435 mtip_set_timeout(port->dd, &fis, &timeout, 0); 1436 1437 start = jiffies; 1438 rv = mtip_exec_internal_command(port, 1439 &fis, 1440 5, 1441 0, 1442 0, 1443 0, 1444 GFP_ATOMIC, 1445 timeout); 1446 dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n", 1447 jiffies_to_msecs(jiffies - start)); 1448 if (rv) 1449 dev_warn(&port->dd->pdev->dev, 1450 "STANDBY IMMEDIATE command failed.\n"); 1451 1452 return rv; 1453 } 1454 1455 /* 1456 * Issue a READ LOG EXT command to the device. 1457 * 1458 * @port pointer to the port structure. 1459 * @page page number to fetch 1460 * @buffer pointer to buffer 1461 * @buffer_dma dma address corresponding to @buffer 1462 * @sectors page length to fetch, in sectors 1463 * 1464 * return value 1465 * @rv return value from mtip_exec_internal_command() 1466 */ 1467 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 1468 dma_addr_t buffer_dma, unsigned int sectors) 1469 { 1470 struct host_to_dev_fis fis; 1471 1472 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1473 fis.type = 0x27; 1474 fis.opts = 1 << 7; 1475 fis.command = ATA_CMD_READ_LOG_EXT; 1476 fis.sect_count = sectors & 0xFF; 1477 fis.sect_cnt_ex = (sectors >> 8) & 0xFF; 1478 fis.lba_low = page; 1479 fis.lba_mid = 0; 1480 fis.device = ATA_DEVICE_OBS; 1481 1482 memset(buffer, 0, sectors * ATA_SECT_SIZE); 1483 1484 return mtip_exec_internal_command(port, 1485 &fis, 1486 5, 1487 buffer_dma, 1488 sectors * ATA_SECT_SIZE, 1489 0, 1490 GFP_ATOMIC, 1491 MTIP_INT_CMD_TIMEOUT_MS); 1492 } 1493 1494 /* 1495 * Issue a SMART READ DATA command to the device. 1496 * 1497 * @port pointer to the port structure. 1498 * @buffer pointer to buffer 1499 * @buffer_dma dma address corresponding to @buffer 1500 * 1501 * return value 1502 * @rv return value from mtip_exec_internal_command() 1503 */ 1504 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, 1505 dma_addr_t buffer_dma) 1506 { 1507 struct host_to_dev_fis fis; 1508 1509 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1510 fis.type = 0x27; 1511 fis.opts = 1 << 7; 1512 fis.command = ATA_CMD_SMART; 1513 fis.features = 0xD0; 1514 fis.sect_count = 1; 1515 fis.lba_mid = 0x4F; 1516 fis.lba_hi = 0xC2; 1517 fis.device = ATA_DEVICE_OBS; 1518 1519 return mtip_exec_internal_command(port, 1520 &fis, 1521 5, 1522 buffer_dma, 1523 ATA_SECT_SIZE, 1524 0, 1525 GFP_ATOMIC, 1526 15000); 1527 } 1528 1529 /* 1530 * Get the value of a smart attribute 1531 * 1532 * @port pointer to the port structure 1533 * @id attribute number 1534 * @attrib pointer to return attrib information corresponding to @id 1535 * 1536 * return value 1537 * -EINVAL NULL buffer passed or unsupported attribute @id. 1538 * -EPERM Identify data not valid, SMART not supported or not enabled 1539 */ 1540 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 1541 struct smart_attr *attrib) 1542 { 1543 int rv, i; 1544 struct smart_attr *pattr; 1545 1546 if (!attrib) 1547 return -EINVAL; 1548 1549 if (!port->identify_valid) { 1550 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); 1551 return -EPERM; 1552 } 1553 if (!(port->identify[82] & 0x1)) { 1554 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); 1555 return -EPERM; 1556 } 1557 if (!(port->identify[85] & 0x1)) { 1558 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); 1559 return -EPERM; 1560 } 1561 1562 memset(port->smart_buf, 0, ATA_SECT_SIZE); 1563 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); 1564 if (rv) { 1565 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); 1566 return rv; 1567 } 1568 1569 pattr = (struct smart_attr *)(port->smart_buf + 2); 1570 for (i = 0; i < 29; i++, pattr++) 1571 if (pattr->attr_id == id) { 1572 memcpy(attrib, pattr, sizeof(struct smart_attr)); 1573 break; 1574 } 1575 1576 if (i == 29) { 1577 dev_warn(&port->dd->pdev->dev, 1578 "Query for invalid SMART attribute ID\n"); 1579 rv = -EINVAL; 1580 } 1581 1582 return rv; 1583 } 1584 1585 /* 1586 * Trim unused sectors 1587 * 1588 * @dd pointer to driver_data structure 1589 * @lba starting lba 1590 * @len # of 512b sectors to trim 1591 * 1592 * return value 1593 * -ENOMEM Out of dma memory 1594 * -EINVAL Invalid parameters passed in, trim not supported 1595 * -EIO Error submitting trim request to hw 1596 */ 1597 static int mtip_send_trim(struct driver_data *dd, unsigned int lba, 1598 unsigned int len) 1599 { 1600 int i, rv = 0; 1601 u64 tlba, tlen, sect_left; 1602 struct mtip_trim_entry *buf; 1603 dma_addr_t dma_addr; 1604 struct host_to_dev_fis fis; 1605 1606 if (!len || dd->trim_supp == false) 1607 return -EINVAL; 1608 1609 /* Trim request too big */ 1610 WARN_ON(len > (MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES)); 1611 1612 /* Trim request not aligned on 4k boundary */ 1613 WARN_ON(len % 8 != 0); 1614 1615 /* Warn if vu_trim structure is too big */ 1616 WARN_ON(sizeof(struct mtip_trim) > ATA_SECT_SIZE); 1617 1618 /* Allocate a DMA buffer for the trim structure */ 1619 buf = dmam_alloc_coherent(&dd->pdev->dev, ATA_SECT_SIZE, &dma_addr, 1620 GFP_KERNEL); 1621 if (!buf) 1622 return -ENOMEM; 1623 memset(buf, 0, ATA_SECT_SIZE); 1624 1625 for (i = 0, sect_left = len, tlba = lba; 1626 i < MTIP_MAX_TRIM_ENTRIES && sect_left; 1627 i++) { 1628 tlen = (sect_left >= MTIP_MAX_TRIM_ENTRY_LEN ? 1629 MTIP_MAX_TRIM_ENTRY_LEN : 1630 sect_left); 1631 buf[i].lba = __force_bit2int cpu_to_le32(tlba); 1632 buf[i].range = __force_bit2int cpu_to_le16(tlen); 1633 tlba += tlen; 1634 sect_left -= tlen; 1635 } 1636 WARN_ON(sect_left != 0); 1637 1638 /* Build the fis */ 1639 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1640 fis.type = 0x27; 1641 fis.opts = 1 << 7; 1642 fis.command = 0xfb; 1643 fis.features = 0x60; 1644 fis.sect_count = 1; 1645 fis.device = ATA_DEVICE_OBS; 1646 1647 if (mtip_exec_internal_command(dd->port, 1648 &fis, 1649 5, 1650 dma_addr, 1651 ATA_SECT_SIZE, 1652 0, 1653 GFP_KERNEL, 1654 MTIP_TRIM_TIMEOUT_MS) < 0) 1655 rv = -EIO; 1656 1657 dmam_free_coherent(&dd->pdev->dev, ATA_SECT_SIZE, buf, dma_addr); 1658 return rv; 1659 } 1660 1661 /* 1662 * Get the drive capacity. 1663 * 1664 * @dd Pointer to the device data structure. 1665 * @sectors Pointer to the variable that will receive the sector count. 1666 * 1667 * return value 1668 * 1 Capacity was returned successfully. 1669 * 0 The identify information is invalid. 1670 */ 1671 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) 1672 { 1673 struct mtip_port *port = dd->port; 1674 u64 total, raw0, raw1, raw2, raw3; 1675 raw0 = port->identify[100]; 1676 raw1 = port->identify[101]; 1677 raw2 = port->identify[102]; 1678 raw3 = port->identify[103]; 1679 total = raw0 | raw1<<16 | raw2<<32 | raw3<<48; 1680 *sectors = total; 1681 return (bool) !!port->identify_valid; 1682 } 1683 1684 /* 1685 * Display the identify command data. 1686 * 1687 * @port Pointer to the port data structure. 1688 * 1689 * return value 1690 * None 1691 */ 1692 static void mtip_dump_identify(struct mtip_port *port) 1693 { 1694 sector_t sectors; 1695 unsigned short revid; 1696 char cbuf[42]; 1697 1698 if (!port->identify_valid) 1699 return; 1700 1701 strlcpy(cbuf, (char *)(port->identify+10), 21); 1702 dev_info(&port->dd->pdev->dev, 1703 "Serial No.: %s\n", cbuf); 1704 1705 strlcpy(cbuf, (char *)(port->identify+23), 9); 1706 dev_info(&port->dd->pdev->dev, 1707 "Firmware Ver.: %s\n", cbuf); 1708 1709 strlcpy(cbuf, (char *)(port->identify+27), 41); 1710 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); 1711 1712 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", 1713 port->identify[128], 1714 port->identify[128] & 0x4 ? "(LOCKED)" : ""); 1715 1716 if (mtip_hw_get_capacity(port->dd, §ors)) 1717 dev_info(&port->dd->pdev->dev, 1718 "Capacity: %llu sectors (%llu MB)\n", 1719 (u64)sectors, 1720 ((u64)sectors) * ATA_SECT_SIZE >> 20); 1721 1722 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); 1723 switch (revid & 0xFF) { 1724 case 0x1: 1725 strlcpy(cbuf, "A0", 3); 1726 break; 1727 case 0x3: 1728 strlcpy(cbuf, "A2", 3); 1729 break; 1730 default: 1731 strlcpy(cbuf, "?", 2); 1732 break; 1733 } 1734 dev_info(&port->dd->pdev->dev, 1735 "Card Type: %s\n", cbuf); 1736 } 1737 1738 /* 1739 * Map the commands scatter list into the command table. 1740 * 1741 * @command Pointer to the command. 1742 * @nents Number of scatter list entries. 1743 * 1744 * return value 1745 * None 1746 */ 1747 static inline void fill_command_sg(struct driver_data *dd, 1748 struct mtip_cmd *command, 1749 int nents) 1750 { 1751 int n; 1752 unsigned int dma_len; 1753 struct mtip_cmd_sg *command_sg; 1754 struct scatterlist *sg = command->sg; 1755 1756 command_sg = command->command + AHCI_CMD_TBL_HDR_SZ; 1757 1758 for (n = 0; n < nents; n++) { 1759 dma_len = sg_dma_len(sg); 1760 if (dma_len > 0x400000) 1761 dev_err(&dd->pdev->dev, 1762 "DMA segment length truncated\n"); 1763 command_sg->info = __force_bit2int 1764 cpu_to_le32((dma_len-1) & 0x3FFFFF); 1765 command_sg->dba = __force_bit2int 1766 cpu_to_le32(sg_dma_address(sg)); 1767 command_sg->dba_upper = __force_bit2int 1768 cpu_to_le32((sg_dma_address(sg) >> 16) >> 16); 1769 command_sg++; 1770 sg++; 1771 } 1772 } 1773 1774 /* 1775 * @brief Execute a drive command. 1776 * 1777 * return value 0 The command completed successfully. 1778 * return value -1 An error occurred while executing the command. 1779 */ 1780 static int exec_drive_task(struct mtip_port *port, u8 *command) 1781 { 1782 struct host_to_dev_fis fis; 1783 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); 1784 unsigned int to; 1785 1786 /* Build the FIS. */ 1787 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1788 fis.type = 0x27; 1789 fis.opts = 1 << 7; 1790 fis.command = command[0]; 1791 fis.features = command[1]; 1792 fis.sect_count = command[2]; 1793 fis.sector = command[3]; 1794 fis.cyl_low = command[4]; 1795 fis.cyl_hi = command[5]; 1796 fis.device = command[6] & ~0x10; /* Clear the dev bit*/ 1797 1798 mtip_set_timeout(port->dd, &fis, &to, 0); 1799 1800 dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n", 1801 __func__, 1802 command[0], 1803 command[1], 1804 command[2], 1805 command[3], 1806 command[4], 1807 command[5], 1808 command[6]); 1809 1810 /* Execute the command. */ 1811 if (mtip_exec_internal_command(port, 1812 &fis, 1813 5, 1814 0, 1815 0, 1816 0, 1817 GFP_KERNEL, 1818 to) < 0) { 1819 return -1; 1820 } 1821 1822 command[0] = reply->command; /* Status*/ 1823 command[1] = reply->features; /* Error*/ 1824 command[4] = reply->cyl_low; 1825 command[5] = reply->cyl_hi; 1826 1827 dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n", 1828 __func__, 1829 command[0], 1830 command[1], 1831 command[4], 1832 command[5]); 1833 1834 return 0; 1835 } 1836 1837 /* 1838 * @brief Execute a drive command. 1839 * 1840 * @param port Pointer to the port data structure. 1841 * @param command Pointer to the user specified command parameters. 1842 * @param user_buffer Pointer to the user space buffer where read sector 1843 * data should be copied. 1844 * 1845 * return value 0 The command completed successfully. 1846 * return value -EFAULT An error occurred while copying the completion 1847 * data to the user space buffer. 1848 * return value -1 An error occurred while executing the command. 1849 */ 1850 static int exec_drive_command(struct mtip_port *port, u8 *command, 1851 void __user *user_buffer) 1852 { 1853 struct host_to_dev_fis fis; 1854 struct host_to_dev_fis *reply; 1855 u8 *buf = NULL; 1856 dma_addr_t dma_addr = 0; 1857 int rv = 0, xfer_sz = command[3]; 1858 unsigned int to; 1859 1860 if (xfer_sz) { 1861 if (!user_buffer) 1862 return -EFAULT; 1863 1864 buf = dmam_alloc_coherent(&port->dd->pdev->dev, 1865 ATA_SECT_SIZE * xfer_sz, 1866 &dma_addr, 1867 GFP_KERNEL); 1868 if (!buf) { 1869 dev_err(&port->dd->pdev->dev, 1870 "Memory allocation failed (%d bytes)\n", 1871 ATA_SECT_SIZE * xfer_sz); 1872 return -ENOMEM; 1873 } 1874 memset(buf, 0, ATA_SECT_SIZE * xfer_sz); 1875 } 1876 1877 /* Build the FIS. */ 1878 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1879 fis.type = 0x27; 1880 fis.opts = 1 << 7; 1881 fis.command = command[0]; 1882 fis.features = command[2]; 1883 fis.sect_count = command[3]; 1884 if (fis.command == ATA_CMD_SMART) { 1885 fis.sector = command[1]; 1886 fis.cyl_low = 0x4F; 1887 fis.cyl_hi = 0xC2; 1888 } 1889 1890 mtip_set_timeout(port->dd, &fis, &to, 0); 1891 1892 if (xfer_sz) 1893 reply = (port->rxfis + RX_FIS_PIO_SETUP); 1894 else 1895 reply = (port->rxfis + RX_FIS_D2H_REG); 1896 1897 dbg_printk(MTIP_DRV_NAME 1898 " %s: User Command: cmd %x, sect %x, " 1899 "feat %x, sectcnt %x\n", 1900 __func__, 1901 command[0], 1902 command[1], 1903 command[2], 1904 command[3]); 1905 1906 /* Execute the command. */ 1907 if (mtip_exec_internal_command(port, 1908 &fis, 1909 5, 1910 (xfer_sz ? dma_addr : 0), 1911 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0), 1912 0, 1913 GFP_KERNEL, 1914 to) 1915 < 0) { 1916 rv = -EFAULT; 1917 goto exit_drive_command; 1918 } 1919 1920 /* Collect the completion status. */ 1921 command[0] = reply->command; /* Status*/ 1922 command[1] = reply->features; /* Error*/ 1923 command[2] = reply->sect_count; 1924 1925 dbg_printk(MTIP_DRV_NAME 1926 " %s: Completion Status: stat %x, " 1927 "err %x, nsect %x\n", 1928 __func__, 1929 command[0], 1930 command[1], 1931 command[2]); 1932 1933 if (xfer_sz) { 1934 if (copy_to_user(user_buffer, 1935 buf, 1936 ATA_SECT_SIZE * command[3])) { 1937 rv = -EFAULT; 1938 goto exit_drive_command; 1939 } 1940 } 1941 exit_drive_command: 1942 if (buf) 1943 dmam_free_coherent(&port->dd->pdev->dev, 1944 ATA_SECT_SIZE * xfer_sz, buf, dma_addr); 1945 return rv; 1946 } 1947 1948 /* 1949 * Indicates whether a command has a single sector payload. 1950 * 1951 * @command passed to the device to perform the certain event. 1952 * @features passed to the device to perform the certain event. 1953 * 1954 * return value 1955 * 1 command is one that always has a single sector payload, 1956 * regardless of the value in the Sector Count field. 1957 * 0 otherwise 1958 * 1959 */ 1960 static unsigned int implicit_sector(unsigned char command, 1961 unsigned char features) 1962 { 1963 unsigned int rv = 0; 1964 1965 /* list of commands that have an implicit sector count of 1 */ 1966 switch (command) { 1967 case ATA_CMD_SEC_SET_PASS: 1968 case ATA_CMD_SEC_UNLOCK: 1969 case ATA_CMD_SEC_ERASE_PREP: 1970 case ATA_CMD_SEC_ERASE_UNIT: 1971 case ATA_CMD_SEC_FREEZE_LOCK: 1972 case ATA_CMD_SEC_DISABLE_PASS: 1973 case ATA_CMD_PMP_READ: 1974 case ATA_CMD_PMP_WRITE: 1975 rv = 1; 1976 break; 1977 case ATA_CMD_SET_MAX: 1978 if (features == ATA_SET_MAX_UNLOCK) 1979 rv = 1; 1980 break; 1981 case ATA_CMD_SMART: 1982 if ((features == ATA_SMART_READ_VALUES) || 1983 (features == ATA_SMART_READ_THRESHOLDS)) 1984 rv = 1; 1985 break; 1986 case ATA_CMD_CONF_OVERLAY: 1987 if ((features == ATA_DCO_IDENTIFY) || 1988 (features == ATA_DCO_SET)) 1989 rv = 1; 1990 break; 1991 } 1992 return rv; 1993 } 1994 1995 /* 1996 * Executes a taskfile 1997 * See ide_taskfile_ioctl() for derivation 1998 */ 1999 static int exec_drive_taskfile(struct driver_data *dd, 2000 void __user *buf, 2001 ide_task_request_t *req_task, 2002 int outtotal) 2003 { 2004 struct host_to_dev_fis fis; 2005 struct host_to_dev_fis *reply; 2006 u8 *outbuf = NULL; 2007 u8 *inbuf = NULL; 2008 dma_addr_t outbuf_dma = 0; 2009 dma_addr_t inbuf_dma = 0; 2010 dma_addr_t dma_buffer = 0; 2011 int err = 0; 2012 unsigned int taskin = 0; 2013 unsigned int taskout = 0; 2014 u8 nsect = 0; 2015 unsigned int timeout; 2016 unsigned int force_single_sector; 2017 unsigned int transfer_size; 2018 unsigned long task_file_data; 2019 int intotal = outtotal + req_task->out_size; 2020 int erasemode = 0; 2021 2022 taskout = req_task->out_size; 2023 taskin = req_task->in_size; 2024 /* 130560 = 512 * 0xFF*/ 2025 if (taskin > 130560 || taskout > 130560) { 2026 err = -EINVAL; 2027 goto abort; 2028 } 2029 2030 if (taskout) { 2031 outbuf = kzalloc(taskout, GFP_KERNEL); 2032 if (outbuf == NULL) { 2033 err = -ENOMEM; 2034 goto abort; 2035 } 2036 if (copy_from_user(outbuf, buf + outtotal, taskout)) { 2037 err = -EFAULT; 2038 goto abort; 2039 } 2040 outbuf_dma = pci_map_single(dd->pdev, 2041 outbuf, 2042 taskout, 2043 DMA_TO_DEVICE); 2044 if (outbuf_dma == 0) { 2045 err = -ENOMEM; 2046 goto abort; 2047 } 2048 dma_buffer = outbuf_dma; 2049 } 2050 2051 if (taskin) { 2052 inbuf = kzalloc(taskin, GFP_KERNEL); 2053 if (inbuf == NULL) { 2054 err = -ENOMEM; 2055 goto abort; 2056 } 2057 2058 if (copy_from_user(inbuf, buf + intotal, taskin)) { 2059 err = -EFAULT; 2060 goto abort; 2061 } 2062 inbuf_dma = pci_map_single(dd->pdev, 2063 inbuf, 2064 taskin, DMA_FROM_DEVICE); 2065 if (inbuf_dma == 0) { 2066 err = -ENOMEM; 2067 goto abort; 2068 } 2069 dma_buffer = inbuf_dma; 2070 } 2071 2072 /* only supports PIO and non-data commands from this ioctl. */ 2073 switch (req_task->data_phase) { 2074 case TASKFILE_OUT: 2075 nsect = taskout / ATA_SECT_SIZE; 2076 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 2077 break; 2078 case TASKFILE_IN: 2079 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 2080 break; 2081 case TASKFILE_NO_DATA: 2082 reply = (dd->port->rxfis + RX_FIS_D2H_REG); 2083 break; 2084 default: 2085 err = -EINVAL; 2086 goto abort; 2087 } 2088 2089 /* Build the FIS. */ 2090 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 2091 2092 fis.type = 0x27; 2093 fis.opts = 1 << 7; 2094 fis.command = req_task->io_ports[7]; 2095 fis.features = req_task->io_ports[1]; 2096 fis.sect_count = req_task->io_ports[2]; 2097 fis.lba_low = req_task->io_ports[3]; 2098 fis.lba_mid = req_task->io_ports[4]; 2099 fis.lba_hi = req_task->io_ports[5]; 2100 /* Clear the dev bit*/ 2101 fis.device = req_task->io_ports[6] & ~0x10; 2102 2103 if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) { 2104 req_task->in_flags.all = 2105 IDE_TASKFILE_STD_IN_FLAGS | 2106 (IDE_HOB_STD_IN_FLAGS << 8); 2107 fis.lba_low_ex = req_task->hob_ports[3]; 2108 fis.lba_mid_ex = req_task->hob_ports[4]; 2109 fis.lba_hi_ex = req_task->hob_ports[5]; 2110 fis.features_ex = req_task->hob_ports[1]; 2111 fis.sect_cnt_ex = req_task->hob_ports[2]; 2112 2113 } else { 2114 req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS; 2115 } 2116 2117 force_single_sector = implicit_sector(fis.command, fis.features); 2118 2119 if ((taskin || taskout) && (!fis.sect_count)) { 2120 if (nsect) 2121 fis.sect_count = nsect; 2122 else { 2123 if (!force_single_sector) { 2124 dev_warn(&dd->pdev->dev, 2125 "data movement but " 2126 "sect_count is 0\n"); 2127 err = -EINVAL; 2128 goto abort; 2129 } 2130 } 2131 } 2132 2133 dbg_printk(MTIP_DRV_NAME 2134 " %s: cmd %x, feat %x, nsect %x," 2135 " sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x," 2136 " head/dev %x\n", 2137 __func__, 2138 fis.command, 2139 fis.features, 2140 fis.sect_count, 2141 fis.lba_low, 2142 fis.lba_mid, 2143 fis.lba_hi, 2144 fis.device); 2145 2146 /* check for erase mode support during secure erase.*/ 2147 if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf && 2148 (outbuf[0] & MTIP_SEC_ERASE_MODE)) { 2149 erasemode = 1; 2150 } 2151 2152 mtip_set_timeout(dd, &fis, &timeout, erasemode); 2153 2154 /* Determine the correct transfer size.*/ 2155 if (force_single_sector) 2156 transfer_size = ATA_SECT_SIZE; 2157 else 2158 transfer_size = ATA_SECT_SIZE * fis.sect_count; 2159 2160 /* Execute the command.*/ 2161 if (mtip_exec_internal_command(dd->port, 2162 &fis, 2163 5, 2164 dma_buffer, 2165 transfer_size, 2166 0, 2167 GFP_KERNEL, 2168 timeout) < 0) { 2169 err = -EIO; 2170 goto abort; 2171 } 2172 2173 task_file_data = readl(dd->port->mmio+PORT_TFDATA); 2174 2175 if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) { 2176 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; 2177 req_task->io_ports[7] = reply->control; 2178 } else { 2179 reply = dd->port->rxfis + RX_FIS_D2H_REG; 2180 req_task->io_ports[7] = reply->command; 2181 } 2182 2183 /* reclaim the DMA buffers.*/ 2184 if (inbuf_dma) 2185 pci_unmap_single(dd->pdev, inbuf_dma, 2186 taskin, DMA_FROM_DEVICE); 2187 if (outbuf_dma) 2188 pci_unmap_single(dd->pdev, outbuf_dma, 2189 taskout, DMA_TO_DEVICE); 2190 inbuf_dma = 0; 2191 outbuf_dma = 0; 2192 2193 /* return the ATA registers to the caller.*/ 2194 req_task->io_ports[1] = reply->features; 2195 req_task->io_ports[2] = reply->sect_count; 2196 req_task->io_ports[3] = reply->lba_low; 2197 req_task->io_ports[4] = reply->lba_mid; 2198 req_task->io_ports[5] = reply->lba_hi; 2199 req_task->io_ports[6] = reply->device; 2200 2201 if (req_task->out_flags.all & 1) { 2202 2203 req_task->hob_ports[3] = reply->lba_low_ex; 2204 req_task->hob_ports[4] = reply->lba_mid_ex; 2205 req_task->hob_ports[5] = reply->lba_hi_ex; 2206 req_task->hob_ports[1] = reply->features_ex; 2207 req_task->hob_ports[2] = reply->sect_cnt_ex; 2208 } 2209 dbg_printk(MTIP_DRV_NAME 2210 " %s: Completion: stat %x," 2211 "err %x, sect_cnt %x, lbalo %x," 2212 "lbamid %x, lbahi %x, dev %x\n", 2213 __func__, 2214 req_task->io_ports[7], 2215 req_task->io_ports[1], 2216 req_task->io_ports[2], 2217 req_task->io_ports[3], 2218 req_task->io_ports[4], 2219 req_task->io_ports[5], 2220 req_task->io_ports[6]); 2221 2222 if (taskout) { 2223 if (copy_to_user(buf + outtotal, outbuf, taskout)) { 2224 err = -EFAULT; 2225 goto abort; 2226 } 2227 } 2228 if (taskin) { 2229 if (copy_to_user(buf + intotal, inbuf, taskin)) { 2230 err = -EFAULT; 2231 goto abort; 2232 } 2233 } 2234 abort: 2235 if (inbuf_dma) 2236 pci_unmap_single(dd->pdev, inbuf_dma, 2237 taskin, DMA_FROM_DEVICE); 2238 if (outbuf_dma) 2239 pci_unmap_single(dd->pdev, outbuf_dma, 2240 taskout, DMA_TO_DEVICE); 2241 kfree(outbuf); 2242 kfree(inbuf); 2243 2244 return err; 2245 } 2246 2247 /* 2248 * Handle IOCTL calls from the Block Layer. 2249 * 2250 * This function is called by the Block Layer when it receives an IOCTL 2251 * command that it does not understand. If the IOCTL command is not supported 2252 * this function returns -ENOTTY. 2253 * 2254 * @dd Pointer to the driver data structure. 2255 * @cmd IOCTL command passed from the Block Layer. 2256 * @arg IOCTL argument passed from the Block Layer. 2257 * 2258 * return value 2259 * 0 The IOCTL completed successfully. 2260 * -ENOTTY The specified command is not supported. 2261 * -EFAULT An error occurred copying data to a user space buffer. 2262 * -EIO An error occurred while executing the command. 2263 */ 2264 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, 2265 unsigned long arg) 2266 { 2267 switch (cmd) { 2268 case HDIO_GET_IDENTITY: 2269 { 2270 if (copy_to_user((void __user *)arg, dd->port->identify, 2271 sizeof(u16) * ATA_ID_WORDS)) 2272 return -EFAULT; 2273 break; 2274 } 2275 case HDIO_DRIVE_CMD: 2276 { 2277 u8 drive_command[4]; 2278 2279 /* Copy the user command info to our buffer. */ 2280 if (copy_from_user(drive_command, 2281 (void __user *) arg, 2282 sizeof(drive_command))) 2283 return -EFAULT; 2284 2285 /* Execute the drive command. */ 2286 if (exec_drive_command(dd->port, 2287 drive_command, 2288 (void __user *) (arg+4))) 2289 return -EIO; 2290 2291 /* Copy the status back to the users buffer. */ 2292 if (copy_to_user((void __user *) arg, 2293 drive_command, 2294 sizeof(drive_command))) 2295 return -EFAULT; 2296 2297 break; 2298 } 2299 case HDIO_DRIVE_TASK: 2300 { 2301 u8 drive_command[7]; 2302 2303 /* Copy the user command info to our buffer. */ 2304 if (copy_from_user(drive_command, 2305 (void __user *) arg, 2306 sizeof(drive_command))) 2307 return -EFAULT; 2308 2309 /* Execute the drive command. */ 2310 if (exec_drive_task(dd->port, drive_command)) 2311 return -EIO; 2312 2313 /* Copy the status back to the users buffer. */ 2314 if (copy_to_user((void __user *) arg, 2315 drive_command, 2316 sizeof(drive_command))) 2317 return -EFAULT; 2318 2319 break; 2320 } 2321 case HDIO_DRIVE_TASKFILE: { 2322 ide_task_request_t req_task; 2323 int ret, outtotal; 2324 2325 if (copy_from_user(&req_task, (void __user *) arg, 2326 sizeof(req_task))) 2327 return -EFAULT; 2328 2329 outtotal = sizeof(req_task); 2330 2331 ret = exec_drive_taskfile(dd, (void __user *) arg, 2332 &req_task, outtotal); 2333 2334 if (copy_to_user((void __user *) arg, &req_task, 2335 sizeof(req_task))) 2336 return -EFAULT; 2337 2338 return ret; 2339 } 2340 2341 default: 2342 return -EINVAL; 2343 } 2344 return 0; 2345 } 2346 2347 /* 2348 * Submit an IO to the hw 2349 * 2350 * This function is called by the block layer to issue an io 2351 * to the device. Upon completion, the callback function will 2352 * be called with the data parameter passed as the callback data. 2353 * 2354 * @dd Pointer to the driver data structure. 2355 * @start First sector to read. 2356 * @nsect Number of sectors to read. 2357 * @nents Number of entries in scatter list for the read command. 2358 * @tag The tag of this read command. 2359 * @callback Pointer to the function that should be called 2360 * when the read completes. 2361 * @data Callback data passed to the callback function 2362 * when the read completes. 2363 * @dir Direction (read or write) 2364 * 2365 * return value 2366 * None 2367 */ 2368 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, 2369 struct mtip_cmd *command, int nents, 2370 struct blk_mq_hw_ctx *hctx) 2371 { 2372 struct host_to_dev_fis *fis; 2373 struct mtip_port *port = dd->port; 2374 int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 2375 u64 start = blk_rq_pos(rq); 2376 unsigned int nsect = blk_rq_sectors(rq); 2377 2378 /* Map the scatter list for DMA access */ 2379 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); 2380 2381 command->scatter_ents = nents; 2382 2383 /* 2384 * The number of retries for this command before it is 2385 * reported as a failure to the upper layers. 2386 */ 2387 command->retries = MTIP_MAX_RETRIES; 2388 2389 /* Fill out fis */ 2390 fis = command->command; 2391 fis->type = 0x27; 2392 fis->opts = 1 << 7; 2393 if (rq_data_dir(rq) == READ) 2394 fis->command = ATA_CMD_FPDMA_READ; 2395 else 2396 fis->command = ATA_CMD_FPDMA_WRITE; 2397 fis->lba_low = start & 0xFF; 2398 fis->lba_mid = (start >> 8) & 0xFF; 2399 fis->lba_hi = (start >> 16) & 0xFF; 2400 fis->lba_low_ex = (start >> 24) & 0xFF; 2401 fis->lba_mid_ex = (start >> 32) & 0xFF; 2402 fis->lba_hi_ex = (start >> 40) & 0xFF; 2403 fis->device = 1 << 6; 2404 fis->features = nsect & 0xFF; 2405 fis->features_ex = (nsect >> 8) & 0xFF; 2406 fis->sect_count = ((rq->tag << 3) | (rq->tag >> 5)); 2407 fis->sect_cnt_ex = 0; 2408 fis->control = 0; 2409 fis->res2 = 0; 2410 fis->res3 = 0; 2411 fill_command_sg(dd, command, nents); 2412 2413 if (command->unaligned) 2414 fis->device |= 1 << 7; 2415 2416 /* Populate the command header */ 2417 command->command_header->opts = 2418 __force_bit2int cpu_to_le32( 2419 (nents << 16) | 5 | AHCI_CMD_PREFETCH); 2420 command->command_header->byte_count = 0; 2421 2422 /* 2423 * Set the completion function and data for the command 2424 * within this layer. 2425 */ 2426 command->comp_data = dd; 2427 command->comp_func = mtip_async_complete; 2428 command->direction = dma_dir; 2429 2430 /* 2431 * To prevent this command from being issued 2432 * if an internal command is in progress or error handling is active. 2433 */ 2434 if (port->flags & MTIP_PF_PAUSE_IO) { 2435 set_bit(rq->tag, port->cmds_to_issue); 2436 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2437 return; 2438 } 2439 2440 /* Issue the command to the hardware */ 2441 mtip_issue_ncq_command(port, rq->tag); 2442 } 2443 2444 /* 2445 * Sysfs status dump. 2446 * 2447 * @dev Pointer to the device structure, passed by the kernrel. 2448 * @attr Pointer to the device_attribute structure passed by the kernel. 2449 * @buf Pointer to the char buffer that will receive the stats info. 2450 * 2451 * return value 2452 * The size, in bytes, of the data copied into buf. 2453 */ 2454 static ssize_t mtip_hw_show_status(struct device *dev, 2455 struct device_attribute *attr, 2456 char *buf) 2457 { 2458 struct driver_data *dd = dev_to_disk(dev)->private_data; 2459 int size = 0; 2460 2461 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) 2462 size += sprintf(buf, "%s", "thermal_shutdown\n"); 2463 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) 2464 size += sprintf(buf, "%s", "write_protect\n"); 2465 else 2466 size += sprintf(buf, "%s", "online\n"); 2467 2468 return size; 2469 } 2470 2471 static DEVICE_ATTR(status, S_IRUGO, mtip_hw_show_status, NULL); 2472 2473 /* debugsfs entries */ 2474 2475 static ssize_t show_device_status(struct device_driver *drv, char *buf) 2476 { 2477 int size = 0; 2478 struct driver_data *dd, *tmp; 2479 unsigned long flags; 2480 char id_buf[42]; 2481 u16 status = 0; 2482 2483 spin_lock_irqsave(&dev_lock, flags); 2484 size += sprintf(&buf[size], "Devices Present:\n"); 2485 list_for_each_entry_safe(dd, tmp, &online_list, online_list) { 2486 if (dd->pdev) { 2487 if (dd->port && 2488 dd->port->identify && 2489 dd->port->identify_valid) { 2490 strlcpy(id_buf, 2491 (char *) (dd->port->identify + 10), 21); 2492 status = *(dd->port->identify + 141); 2493 } else { 2494 memset(id_buf, 0, 42); 2495 status = 0; 2496 } 2497 2498 if (dd->port && 2499 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { 2500 size += sprintf(&buf[size], 2501 " device %s %s (ftl rebuild %d %%)\n", 2502 dev_name(&dd->pdev->dev), 2503 id_buf, 2504 status); 2505 } else { 2506 size += sprintf(&buf[size], 2507 " device %s %s\n", 2508 dev_name(&dd->pdev->dev), 2509 id_buf); 2510 } 2511 } 2512 } 2513 2514 size += sprintf(&buf[size], "Devices Being Removed:\n"); 2515 list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) { 2516 if (dd->pdev) { 2517 if (dd->port && 2518 dd->port->identify && 2519 dd->port->identify_valid) { 2520 strlcpy(id_buf, 2521 (char *) (dd->port->identify+10), 21); 2522 status = *(dd->port->identify + 141); 2523 } else { 2524 memset(id_buf, 0, 42); 2525 status = 0; 2526 } 2527 2528 if (dd->port && 2529 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { 2530 size += sprintf(&buf[size], 2531 " device %s %s (ftl rebuild %d %%)\n", 2532 dev_name(&dd->pdev->dev), 2533 id_buf, 2534 status); 2535 } else { 2536 size += sprintf(&buf[size], 2537 " device %s %s\n", 2538 dev_name(&dd->pdev->dev), 2539 id_buf); 2540 } 2541 } 2542 } 2543 spin_unlock_irqrestore(&dev_lock, flags); 2544 2545 return size; 2546 } 2547 2548 static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf, 2549 size_t len, loff_t *offset) 2550 { 2551 struct driver_data *dd = (struct driver_data *)f->private_data; 2552 int size = *offset; 2553 char *buf; 2554 int rv = 0; 2555 2556 if (!len || *offset) 2557 return 0; 2558 2559 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2560 if (!buf) { 2561 dev_err(&dd->pdev->dev, 2562 "Memory allocation: status buffer\n"); 2563 return -ENOMEM; 2564 } 2565 2566 size += show_device_status(NULL, buf); 2567 2568 *offset = size <= len ? size : len; 2569 size = copy_to_user(ubuf, buf, *offset); 2570 if (size) 2571 rv = -EFAULT; 2572 2573 kfree(buf); 2574 return rv ? rv : *offset; 2575 } 2576 2577 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf, 2578 size_t len, loff_t *offset) 2579 { 2580 struct driver_data *dd = (struct driver_data *)f->private_data; 2581 char *buf; 2582 u32 group_allocated; 2583 int size = *offset; 2584 int n, rv = 0; 2585 2586 if (!len || size) 2587 return 0; 2588 2589 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2590 if (!buf) { 2591 dev_err(&dd->pdev->dev, 2592 "Memory allocation: register buffer\n"); 2593 return -ENOMEM; 2594 } 2595 2596 size += sprintf(&buf[size], "H/ S ACTive : [ 0x"); 2597 2598 for (n = dd->slot_groups-1; n >= 0; n--) 2599 size += sprintf(&buf[size], "%08X ", 2600 readl(dd->port->s_active[n])); 2601 2602 size += sprintf(&buf[size], "]\n"); 2603 size += sprintf(&buf[size], "H/ Command Issue : [ 0x"); 2604 2605 for (n = dd->slot_groups-1; n >= 0; n--) 2606 size += sprintf(&buf[size], "%08X ", 2607 readl(dd->port->cmd_issue[n])); 2608 2609 size += sprintf(&buf[size], "]\n"); 2610 size += sprintf(&buf[size], "H/ Completed : [ 0x"); 2611 2612 for (n = dd->slot_groups-1; n >= 0; n--) 2613 size += sprintf(&buf[size], "%08X ", 2614 readl(dd->port->completed[n])); 2615 2616 size += sprintf(&buf[size], "]\n"); 2617 size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n", 2618 readl(dd->port->mmio + PORT_IRQ_STAT)); 2619 size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n", 2620 readl(dd->mmio + HOST_IRQ_STAT)); 2621 size += sprintf(&buf[size], "\n"); 2622 2623 size += sprintf(&buf[size], "L/ Allocated : [ 0x"); 2624 2625 for (n = dd->slot_groups-1; n >= 0; n--) { 2626 if (sizeof(long) > sizeof(u32)) 2627 group_allocated = 2628 dd->port->allocated[n/2] >> (32*(n&1)); 2629 else 2630 group_allocated = dd->port->allocated[n]; 2631 size += sprintf(&buf[size], "%08X ", group_allocated); 2632 } 2633 size += sprintf(&buf[size], "]\n"); 2634 2635 size += sprintf(&buf[size], "L/ Commands in Q : [ 0x"); 2636 2637 for (n = dd->slot_groups-1; n >= 0; n--) { 2638 if (sizeof(long) > sizeof(u32)) 2639 group_allocated = 2640 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); 2641 else 2642 group_allocated = dd->port->cmds_to_issue[n]; 2643 size += sprintf(&buf[size], "%08X ", group_allocated); 2644 } 2645 size += sprintf(&buf[size], "]\n"); 2646 2647 *offset = size <= len ? size : len; 2648 size = copy_to_user(ubuf, buf, *offset); 2649 if (size) 2650 rv = -EFAULT; 2651 2652 kfree(buf); 2653 return rv ? rv : *offset; 2654 } 2655 2656 static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf, 2657 size_t len, loff_t *offset) 2658 { 2659 struct driver_data *dd = (struct driver_data *)f->private_data; 2660 char *buf; 2661 int size = *offset; 2662 int rv = 0; 2663 2664 if (!len || size) 2665 return 0; 2666 2667 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2668 if (!buf) { 2669 dev_err(&dd->pdev->dev, 2670 "Memory allocation: flag buffer\n"); 2671 return -ENOMEM; 2672 } 2673 2674 size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n", 2675 dd->port->flags); 2676 size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n", 2677 dd->dd_flag); 2678 2679 *offset = size <= len ? size : len; 2680 size = copy_to_user(ubuf, buf, *offset); 2681 if (size) 2682 rv = -EFAULT; 2683 2684 kfree(buf); 2685 return rv ? rv : *offset; 2686 } 2687 2688 static const struct file_operations mtip_device_status_fops = { 2689 .owner = THIS_MODULE, 2690 .open = simple_open, 2691 .read = mtip_hw_read_device_status, 2692 .llseek = no_llseek, 2693 }; 2694 2695 static const struct file_operations mtip_regs_fops = { 2696 .owner = THIS_MODULE, 2697 .open = simple_open, 2698 .read = mtip_hw_read_registers, 2699 .llseek = no_llseek, 2700 }; 2701 2702 static const struct file_operations mtip_flags_fops = { 2703 .owner = THIS_MODULE, 2704 .open = simple_open, 2705 .read = mtip_hw_read_flags, 2706 .llseek = no_llseek, 2707 }; 2708 2709 /* 2710 * Create the sysfs related attributes. 2711 * 2712 * @dd Pointer to the driver data structure. 2713 * @kobj Pointer to the kobj for the block device. 2714 * 2715 * return value 2716 * 0 Operation completed successfully. 2717 * -EINVAL Invalid parameter. 2718 */ 2719 static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj) 2720 { 2721 if (!kobj || !dd) 2722 return -EINVAL; 2723 2724 if (sysfs_create_file(kobj, &dev_attr_status.attr)) 2725 dev_warn(&dd->pdev->dev, 2726 "Error creating 'status' sysfs entry\n"); 2727 return 0; 2728 } 2729 2730 /* 2731 * Remove the sysfs related attributes. 2732 * 2733 * @dd Pointer to the driver data structure. 2734 * @kobj Pointer to the kobj for the block device. 2735 * 2736 * return value 2737 * 0 Operation completed successfully. 2738 * -EINVAL Invalid parameter. 2739 */ 2740 static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj) 2741 { 2742 if (!kobj || !dd) 2743 return -EINVAL; 2744 2745 sysfs_remove_file(kobj, &dev_attr_status.attr); 2746 2747 return 0; 2748 } 2749 2750 static int mtip_hw_debugfs_init(struct driver_data *dd) 2751 { 2752 if (!dfs_parent) 2753 return -1; 2754 2755 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); 2756 if (IS_ERR_OR_NULL(dd->dfs_node)) { 2757 dev_warn(&dd->pdev->dev, 2758 "Error creating node %s under debugfs\n", 2759 dd->disk->disk_name); 2760 dd->dfs_node = NULL; 2761 return -1; 2762 } 2763 2764 debugfs_create_file("flags", S_IRUGO, dd->dfs_node, dd, 2765 &mtip_flags_fops); 2766 debugfs_create_file("registers", S_IRUGO, dd->dfs_node, dd, 2767 &mtip_regs_fops); 2768 2769 return 0; 2770 } 2771 2772 static void mtip_hw_debugfs_exit(struct driver_data *dd) 2773 { 2774 if (dd->dfs_node) 2775 debugfs_remove_recursive(dd->dfs_node); 2776 } 2777 2778 static int mtip_free_orphan(struct driver_data *dd) 2779 { 2780 struct kobject *kobj; 2781 2782 if (dd->bdev) { 2783 if (dd->bdev->bd_holders >= 1) 2784 return -2; 2785 2786 bdput(dd->bdev); 2787 dd->bdev = NULL; 2788 } 2789 2790 mtip_hw_debugfs_exit(dd); 2791 2792 spin_lock(&rssd_index_lock); 2793 ida_remove(&rssd_index_ida, dd->index); 2794 spin_unlock(&rssd_index_lock); 2795 2796 if (!test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag) && 2797 test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) { 2798 put_disk(dd->disk); 2799 } else { 2800 if (dd->disk) { 2801 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 2802 if (kobj) { 2803 mtip_hw_sysfs_exit(dd, kobj); 2804 kobject_put(kobj); 2805 } 2806 del_gendisk(dd->disk); 2807 dd->disk = NULL; 2808 } 2809 if (dd->queue) { 2810 dd->queue->queuedata = NULL; 2811 blk_cleanup_queue(dd->queue); 2812 blk_mq_free_tag_set(&dd->tags); 2813 dd->queue = NULL; 2814 } 2815 } 2816 kfree(dd); 2817 return 0; 2818 } 2819 2820 /* 2821 * Perform any init/resume time hardware setup 2822 * 2823 * @dd Pointer to the driver data structure. 2824 * 2825 * return value 2826 * None 2827 */ 2828 static inline void hba_setup(struct driver_data *dd) 2829 { 2830 u32 hwdata; 2831 hwdata = readl(dd->mmio + HOST_HSORG); 2832 2833 /* interrupt bug workaround: use only 1 IS bit.*/ 2834 writel(hwdata | 2835 HSORG_DISABLE_SLOTGRP_INTR | 2836 HSORG_DISABLE_SLOTGRP_PXIS, 2837 dd->mmio + HOST_HSORG); 2838 } 2839 2840 static int mtip_device_unaligned_constrained(struct driver_data *dd) 2841 { 2842 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); 2843 } 2844 2845 /* 2846 * Detect the details of the product, and store anything needed 2847 * into the driver data structure. This includes product type and 2848 * version and number of slot groups. 2849 * 2850 * @dd Pointer to the driver data structure. 2851 * 2852 * return value 2853 * None 2854 */ 2855 static void mtip_detect_product(struct driver_data *dd) 2856 { 2857 u32 hwdata; 2858 unsigned int rev, slotgroups; 2859 2860 /* 2861 * HBA base + 0xFC [15:0] - vendor-specific hardware interface 2862 * info register: 2863 * [15:8] hardware/software interface rev# 2864 * [ 3] asic-style interface 2865 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style). 2866 */ 2867 hwdata = readl(dd->mmio + HOST_HSORG); 2868 2869 dd->product_type = MTIP_PRODUCT_UNKNOWN; 2870 dd->slot_groups = 1; 2871 2872 if (hwdata & 0x8) { 2873 dd->product_type = MTIP_PRODUCT_ASICFPGA; 2874 rev = (hwdata & HSORG_HWREV) >> 8; 2875 slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1; 2876 dev_info(&dd->pdev->dev, 2877 "ASIC-FPGA design, HS rev 0x%x, " 2878 "%i slot groups [%i slots]\n", 2879 rev, 2880 slotgroups, 2881 slotgroups * 32); 2882 2883 if (slotgroups > MTIP_MAX_SLOT_GROUPS) { 2884 dev_warn(&dd->pdev->dev, 2885 "Warning: driver only supports " 2886 "%i slot groups.\n", MTIP_MAX_SLOT_GROUPS); 2887 slotgroups = MTIP_MAX_SLOT_GROUPS; 2888 } 2889 dd->slot_groups = slotgroups; 2890 return; 2891 } 2892 2893 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); 2894 } 2895 2896 /* 2897 * Blocking wait for FTL rebuild to complete 2898 * 2899 * @dd Pointer to the DRIVER_DATA structure. 2900 * 2901 * return value 2902 * 0 FTL rebuild completed successfully 2903 * -EFAULT FTL rebuild error/timeout/interruption 2904 */ 2905 static int mtip_ftl_rebuild_poll(struct driver_data *dd) 2906 { 2907 unsigned long timeout, cnt = 0, start; 2908 2909 dev_warn(&dd->pdev->dev, 2910 "FTL rebuild in progress. Polling for completion.\n"); 2911 2912 start = jiffies; 2913 timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS); 2914 2915 do { 2916 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2917 &dd->dd_flag))) 2918 return -EFAULT; 2919 if (mtip_check_surprise_removal(dd->pdev)) 2920 return -EFAULT; 2921 2922 if (mtip_get_identify(dd->port, NULL) < 0) 2923 return -EFAULT; 2924 2925 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 2926 MTIP_FTL_REBUILD_MAGIC) { 2927 ssleep(1); 2928 /* Print message every 3 minutes */ 2929 if (cnt++ >= 180) { 2930 dev_warn(&dd->pdev->dev, 2931 "FTL rebuild in progress (%d secs).\n", 2932 jiffies_to_msecs(jiffies - start) / 1000); 2933 cnt = 0; 2934 } 2935 } else { 2936 dev_warn(&dd->pdev->dev, 2937 "FTL rebuild complete (%d secs).\n", 2938 jiffies_to_msecs(jiffies - start) / 1000); 2939 mtip_block_initialize(dd); 2940 return 0; 2941 } 2942 ssleep(10); 2943 } while (time_before(jiffies, timeout)); 2944 2945 /* Check for timeout */ 2946 dev_err(&dd->pdev->dev, 2947 "Timed out waiting for FTL rebuild to complete (%d secs).\n", 2948 jiffies_to_msecs(jiffies - start) / 1000); 2949 return -EFAULT; 2950 } 2951 2952 /* 2953 * service thread to issue queued commands 2954 * 2955 * @data Pointer to the driver data structure. 2956 * 2957 * return value 2958 * 0 2959 */ 2960 2961 static int mtip_service_thread(void *data) 2962 { 2963 struct driver_data *dd = (struct driver_data *)data; 2964 unsigned long slot, slot_start, slot_wrap; 2965 unsigned int num_cmd_slots = dd->slot_groups * 32; 2966 struct mtip_port *port = dd->port; 2967 int ret; 2968 2969 while (1) { 2970 if (kthread_should_stop() || 2971 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 2972 goto st_out; 2973 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 2974 2975 /* 2976 * the condition is to check neither an internal command is 2977 * is in progress nor error handling is active 2978 */ 2979 wait_event_interruptible(port->svc_wait, (port->flags) && 2980 !(port->flags & MTIP_PF_PAUSE_IO)); 2981 2982 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 2983 2984 if (kthread_should_stop() || 2985 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 2986 goto st_out; 2987 2988 /* If I am an orphan, start self cleanup */ 2989 if (test_bit(MTIP_PF_SR_CLEANUP_BIT, &port->flags)) 2990 break; 2991 2992 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2993 &dd->dd_flag))) 2994 goto st_out; 2995 2996 restart_eh: 2997 /* Demux bits: start with error handling */ 2998 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { 2999 mtip_handle_tfe(dd); 3000 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 3001 } 3002 3003 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) 3004 goto restart_eh; 3005 3006 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 3007 slot = 1; 3008 /* used to restrict the loop to one iteration */ 3009 slot_start = num_cmd_slots; 3010 slot_wrap = 0; 3011 while (1) { 3012 slot = find_next_bit(port->cmds_to_issue, 3013 num_cmd_slots, slot); 3014 if (slot_wrap == 1) { 3015 if ((slot_start >= slot) || 3016 (slot >= num_cmd_slots)) 3017 break; 3018 } 3019 if (unlikely(slot_start == num_cmd_slots)) 3020 slot_start = slot; 3021 3022 if (unlikely(slot == num_cmd_slots)) { 3023 slot = 1; 3024 slot_wrap = 1; 3025 continue; 3026 } 3027 3028 /* Issue the command to the hardware */ 3029 mtip_issue_ncq_command(port, slot); 3030 3031 clear_bit(slot, port->cmds_to_issue); 3032 } 3033 3034 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 3035 } 3036 3037 if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { 3038 if (mtip_ftl_rebuild_poll(dd) < 0) 3039 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, 3040 &dd->dd_flag); 3041 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); 3042 } 3043 } 3044 3045 /* wait for pci remove to exit */ 3046 while (1) { 3047 if (test_bit(MTIP_DDF_REMOVE_DONE_BIT, &dd->dd_flag)) 3048 break; 3049 msleep_interruptible(1000); 3050 if (kthread_should_stop()) 3051 goto st_out; 3052 } 3053 3054 while (1) { 3055 ret = mtip_free_orphan(dd); 3056 if (!ret) { 3057 /* NOTE: All data structures are invalid, do not 3058 * access any here */ 3059 return 0; 3060 } 3061 msleep_interruptible(1000); 3062 if (kthread_should_stop()) 3063 goto st_out; 3064 } 3065 st_out: 3066 return 0; 3067 } 3068 3069 /* 3070 * DMA region teardown 3071 * 3072 * @dd Pointer to driver_data structure 3073 * 3074 * return value 3075 * None 3076 */ 3077 static void mtip_dma_free(struct driver_data *dd) 3078 { 3079 struct mtip_port *port = dd->port; 3080 3081 if (port->block1) 3082 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 3083 port->block1, port->block1_dma); 3084 3085 if (port->command_list) { 3086 dmam_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, 3087 port->command_list, port->command_list_dma); 3088 } 3089 } 3090 3091 /* 3092 * DMA region setup 3093 * 3094 * @dd Pointer to driver_data structure 3095 * 3096 * return value 3097 * -ENOMEM Not enough free DMA region space to initialize driver 3098 */ 3099 static int mtip_dma_alloc(struct driver_data *dd) 3100 { 3101 struct mtip_port *port = dd->port; 3102 3103 /* Allocate dma memory for RX Fis, Identify, and Sector Bufffer */ 3104 port->block1 = 3105 dmam_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 3106 &port->block1_dma, GFP_KERNEL); 3107 if (!port->block1) 3108 return -ENOMEM; 3109 memset(port->block1, 0, BLOCK_DMA_ALLOC_SZ); 3110 3111 /* Allocate dma memory for command list */ 3112 port->command_list = 3113 dmam_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, 3114 &port->command_list_dma, GFP_KERNEL); 3115 if (!port->command_list) { 3116 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 3117 port->block1, port->block1_dma); 3118 port->block1 = NULL; 3119 port->block1_dma = 0; 3120 return -ENOMEM; 3121 } 3122 memset(port->command_list, 0, AHCI_CMD_TBL_SZ); 3123 3124 /* Setup all pointers into first DMA region */ 3125 port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET; 3126 port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET; 3127 port->identify = port->block1 + AHCI_IDFY_OFFSET; 3128 port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET; 3129 port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET; 3130 port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET; 3131 port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET; 3132 port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET; 3133 3134 return 0; 3135 } 3136 3137 static int mtip_hw_get_identify(struct driver_data *dd) 3138 { 3139 struct smart_attr attr242; 3140 unsigned char *buf; 3141 int rv; 3142 3143 if (mtip_get_identify(dd->port, NULL) < 0) 3144 return -EFAULT; 3145 3146 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 3147 MTIP_FTL_REBUILD_MAGIC) { 3148 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); 3149 return MTIP_FTL_REBUILD_MAGIC; 3150 } 3151 mtip_dump_identify(dd->port); 3152 3153 /* check write protect, over temp and rebuild statuses */ 3154 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 3155 dd->port->log_buf, 3156 dd->port->log_buf_dma, 1); 3157 if (rv) { 3158 dev_warn(&dd->pdev->dev, 3159 "Error in READ LOG EXT (10h) command\n"); 3160 /* non-critical error, don't fail the load */ 3161 } else { 3162 buf = (unsigned char *)dd->port->log_buf; 3163 if (buf[259] & 0x1) { 3164 dev_info(&dd->pdev->dev, 3165 "Write protect bit is set.\n"); 3166 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 3167 } 3168 if (buf[288] == 0xF7) { 3169 dev_info(&dd->pdev->dev, 3170 "Exceeded Tmax, drive in thermal shutdown.\n"); 3171 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 3172 } 3173 if (buf[288] == 0xBF) { 3174 dev_info(&dd->pdev->dev, 3175 "Drive indicates rebuild has failed.\n"); 3176 /* TODO */ 3177 } 3178 } 3179 3180 /* get write protect progess */ 3181 memset(&attr242, 0, sizeof(struct smart_attr)); 3182 if (mtip_get_smart_attr(dd->port, 242, &attr242)) 3183 dev_warn(&dd->pdev->dev, 3184 "Unable to check write protect progress\n"); 3185 else 3186 dev_info(&dd->pdev->dev, 3187 "Write protect progress: %u%% (%u blocks)\n", 3188 attr242.cur, le32_to_cpu(attr242.data)); 3189 3190 return rv; 3191 } 3192 3193 /* 3194 * Called once for each card. 3195 * 3196 * @dd Pointer to the driver data structure. 3197 * 3198 * return value 3199 * 0 on success, else an error code. 3200 */ 3201 static int mtip_hw_init(struct driver_data *dd) 3202 { 3203 int i; 3204 int rv; 3205 unsigned int num_command_slots; 3206 unsigned long timeout, timetaken; 3207 3208 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; 3209 3210 mtip_detect_product(dd); 3211 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { 3212 rv = -EIO; 3213 goto out1; 3214 } 3215 num_command_slots = dd->slot_groups * 32; 3216 3217 hba_setup(dd); 3218 3219 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, 3220 dd->numa_node); 3221 if (!dd->port) { 3222 dev_err(&dd->pdev->dev, 3223 "Memory allocation: port structure\n"); 3224 return -ENOMEM; 3225 } 3226 3227 /* Continue workqueue setup */ 3228 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3229 dd->work[i].port = dd->port; 3230 3231 /* Enable unaligned IO constraints for some devices */ 3232 if (mtip_device_unaligned_constrained(dd)) 3233 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; 3234 else 3235 dd->unal_qdepth = 0; 3236 3237 sema_init(&dd->port->cmd_slot_unal, dd->unal_qdepth); 3238 3239 /* Spinlock to prevent concurrent issue */ 3240 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3241 spin_lock_init(&dd->port->cmd_issue_lock[i]); 3242 3243 /* Set the port mmio base address. */ 3244 dd->port->mmio = dd->mmio + PORT_OFFSET; 3245 dd->port->dd = dd; 3246 3247 /* DMA allocations */ 3248 rv = mtip_dma_alloc(dd); 3249 if (rv < 0) 3250 goto out1; 3251 3252 /* Setup the pointers to the extended s_active and CI registers. */ 3253 for (i = 0; i < dd->slot_groups; i++) { 3254 dd->port->s_active[i] = 3255 dd->port->mmio + i*0x80 + PORT_SCR_ACT; 3256 dd->port->cmd_issue[i] = 3257 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; 3258 dd->port->completed[i] = 3259 dd->port->mmio + i*0x80 + PORT_SDBV; 3260 } 3261 3262 timetaken = jiffies; 3263 timeout = jiffies + msecs_to_jiffies(30000); 3264 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && 3265 time_before(jiffies, timeout)) { 3266 mdelay(100); 3267 } 3268 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 3269 timetaken = jiffies - timetaken; 3270 dev_warn(&dd->pdev->dev, 3271 "Surprise removal detected at %u ms\n", 3272 jiffies_to_msecs(timetaken)); 3273 rv = -ENODEV; 3274 goto out2 ; 3275 } 3276 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { 3277 timetaken = jiffies - timetaken; 3278 dev_warn(&dd->pdev->dev, 3279 "Removal detected at %u ms\n", 3280 jiffies_to_msecs(timetaken)); 3281 rv = -EFAULT; 3282 goto out2; 3283 } 3284 3285 /* Conditionally reset the HBA. */ 3286 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { 3287 if (mtip_hba_reset(dd) < 0) { 3288 dev_err(&dd->pdev->dev, 3289 "Card did not reset within timeout\n"); 3290 rv = -EIO; 3291 goto out2; 3292 } 3293 } else { 3294 /* Clear any pending interrupts on the HBA */ 3295 writel(readl(dd->mmio + HOST_IRQ_STAT), 3296 dd->mmio + HOST_IRQ_STAT); 3297 } 3298 3299 mtip_init_port(dd->port); 3300 mtip_start_port(dd->port); 3301 3302 /* Setup the ISR and enable interrupts. */ 3303 rv = devm_request_irq(&dd->pdev->dev, 3304 dd->pdev->irq, 3305 mtip_irq_handler, 3306 IRQF_SHARED, 3307 dev_driver_string(&dd->pdev->dev), 3308 dd); 3309 3310 if (rv) { 3311 dev_err(&dd->pdev->dev, 3312 "Unable to allocate IRQ %d\n", dd->pdev->irq); 3313 goto out2; 3314 } 3315 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); 3316 3317 /* Enable interrupts on the HBA. */ 3318 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3319 dd->mmio + HOST_CTL); 3320 3321 init_waitqueue_head(&dd->port->svc_wait); 3322 3323 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 3324 rv = -EFAULT; 3325 goto out3; 3326 } 3327 3328 return rv; 3329 3330 out3: 3331 /* Disable interrupts on the HBA. */ 3332 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3333 dd->mmio + HOST_CTL); 3334 3335 /* Release the IRQ. */ 3336 irq_set_affinity_hint(dd->pdev->irq, NULL); 3337 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3338 3339 out2: 3340 mtip_deinit_port(dd->port); 3341 mtip_dma_free(dd); 3342 3343 out1: 3344 /* Free the memory allocated for the for structure. */ 3345 kfree(dd->port); 3346 3347 return rv; 3348 } 3349 3350 static void mtip_standby_drive(struct driver_data *dd) 3351 { 3352 if (dd->sr) 3353 return; 3354 3355 /* 3356 * Send standby immediate (E0h) to the drive so that it 3357 * saves its state. 3358 */ 3359 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && 3360 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) 3361 if (mtip_standby_immediate(dd->port)) 3362 dev_warn(&dd->pdev->dev, 3363 "STANDBY IMMEDIATE failed\n"); 3364 } 3365 3366 /* 3367 * Called to deinitialize an interface. 3368 * 3369 * @dd Pointer to the driver data structure. 3370 * 3371 * return value 3372 * 0 3373 */ 3374 static int mtip_hw_exit(struct driver_data *dd) 3375 { 3376 /* 3377 * Send standby immediate (E0h) to the drive so that it 3378 * saves its state. 3379 */ 3380 if (!dd->sr) { 3381 /* de-initialize the port. */ 3382 mtip_deinit_port(dd->port); 3383 3384 /* Disable interrupts on the HBA. */ 3385 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3386 dd->mmio + HOST_CTL); 3387 } 3388 3389 /* Release the IRQ. */ 3390 irq_set_affinity_hint(dd->pdev->irq, NULL); 3391 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3392 3393 /* Free dma regions */ 3394 mtip_dma_free(dd); 3395 3396 /* Free the memory allocated for the for structure. */ 3397 kfree(dd->port); 3398 dd->port = NULL; 3399 3400 return 0; 3401 } 3402 3403 /* 3404 * Issue a Standby Immediate command to the device. 3405 * 3406 * This function is called by the Block Layer just before the 3407 * system powers off during a shutdown. 3408 * 3409 * @dd Pointer to the driver data structure. 3410 * 3411 * return value 3412 * 0 3413 */ 3414 static int mtip_hw_shutdown(struct driver_data *dd) 3415 { 3416 /* 3417 * Send standby immediate (E0h) to the drive so that it 3418 * saves its state. 3419 */ 3420 if (!dd->sr && dd->port) 3421 mtip_standby_immediate(dd->port); 3422 3423 return 0; 3424 } 3425 3426 /* 3427 * Suspend function 3428 * 3429 * This function is called by the Block Layer just before the 3430 * system hibernates. 3431 * 3432 * @dd Pointer to the driver data structure. 3433 * 3434 * return value 3435 * 0 Suspend was successful 3436 * -EFAULT Suspend was not successful 3437 */ 3438 static int mtip_hw_suspend(struct driver_data *dd) 3439 { 3440 /* 3441 * Send standby immediate (E0h) to the drive 3442 * so that it saves its state. 3443 */ 3444 if (mtip_standby_immediate(dd->port) != 0) { 3445 dev_err(&dd->pdev->dev, 3446 "Failed standby-immediate command\n"); 3447 return -EFAULT; 3448 } 3449 3450 /* Disable interrupts on the HBA.*/ 3451 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3452 dd->mmio + HOST_CTL); 3453 mtip_deinit_port(dd->port); 3454 3455 return 0; 3456 } 3457 3458 /* 3459 * Resume function 3460 * 3461 * This function is called by the Block Layer as the 3462 * system resumes. 3463 * 3464 * @dd Pointer to the driver data structure. 3465 * 3466 * return value 3467 * 0 Resume was successful 3468 * -EFAULT Resume was not successful 3469 */ 3470 static int mtip_hw_resume(struct driver_data *dd) 3471 { 3472 /* Perform any needed hardware setup steps */ 3473 hba_setup(dd); 3474 3475 /* Reset the HBA */ 3476 if (mtip_hba_reset(dd) != 0) { 3477 dev_err(&dd->pdev->dev, 3478 "Unable to reset the HBA\n"); 3479 return -EFAULT; 3480 } 3481 3482 /* 3483 * Enable the port, DMA engine, and FIS reception specific 3484 * h/w in controller. 3485 */ 3486 mtip_init_port(dd->port); 3487 mtip_start_port(dd->port); 3488 3489 /* Enable interrupts on the HBA.*/ 3490 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3491 dd->mmio + HOST_CTL); 3492 3493 return 0; 3494 } 3495 3496 /* 3497 * Helper function for reusing disk name 3498 * upon hot insertion. 3499 */ 3500 static int rssd_disk_name_format(char *prefix, 3501 int index, 3502 char *buf, 3503 int buflen) 3504 { 3505 const int base = 'z' - 'a' + 1; 3506 char *begin = buf + strlen(prefix); 3507 char *end = buf + buflen; 3508 char *p; 3509 int unit; 3510 3511 p = end - 1; 3512 *p = '\0'; 3513 unit = base; 3514 do { 3515 if (p == begin) 3516 return -EINVAL; 3517 *--p = 'a' + (index % unit); 3518 index = (index / unit) - 1; 3519 } while (index >= 0); 3520 3521 memmove(begin, p, end - p); 3522 memcpy(buf, prefix, strlen(prefix)); 3523 3524 return 0; 3525 } 3526 3527 /* 3528 * Block layer IOCTL handler. 3529 * 3530 * @dev Pointer to the block_device structure. 3531 * @mode ignored 3532 * @cmd IOCTL command passed from the user application. 3533 * @arg Argument passed from the user application. 3534 * 3535 * return value 3536 * 0 IOCTL completed successfully. 3537 * -ENOTTY IOCTL not supported or invalid driver data 3538 * structure pointer. 3539 */ 3540 static int mtip_block_ioctl(struct block_device *dev, 3541 fmode_t mode, 3542 unsigned cmd, 3543 unsigned long arg) 3544 { 3545 struct driver_data *dd = dev->bd_disk->private_data; 3546 3547 if (!capable(CAP_SYS_ADMIN)) 3548 return -EACCES; 3549 3550 if (!dd) 3551 return -ENOTTY; 3552 3553 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3554 return -ENOTTY; 3555 3556 switch (cmd) { 3557 case BLKFLSBUF: 3558 return -ENOTTY; 3559 default: 3560 return mtip_hw_ioctl(dd, cmd, arg); 3561 } 3562 } 3563 3564 #ifdef CONFIG_COMPAT 3565 /* 3566 * Block layer compat IOCTL handler. 3567 * 3568 * @dev Pointer to the block_device structure. 3569 * @mode ignored 3570 * @cmd IOCTL command passed from the user application. 3571 * @arg Argument passed from the user application. 3572 * 3573 * return value 3574 * 0 IOCTL completed successfully. 3575 * -ENOTTY IOCTL not supported or invalid driver data 3576 * structure pointer. 3577 */ 3578 static int mtip_block_compat_ioctl(struct block_device *dev, 3579 fmode_t mode, 3580 unsigned cmd, 3581 unsigned long arg) 3582 { 3583 struct driver_data *dd = dev->bd_disk->private_data; 3584 3585 if (!capable(CAP_SYS_ADMIN)) 3586 return -EACCES; 3587 3588 if (!dd) 3589 return -ENOTTY; 3590 3591 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3592 return -ENOTTY; 3593 3594 switch (cmd) { 3595 case BLKFLSBUF: 3596 return -ENOTTY; 3597 case HDIO_DRIVE_TASKFILE: { 3598 struct mtip_compat_ide_task_request_s __user *compat_req_task; 3599 ide_task_request_t req_task; 3600 int compat_tasksize, outtotal, ret; 3601 3602 compat_tasksize = 3603 sizeof(struct mtip_compat_ide_task_request_s); 3604 3605 compat_req_task = 3606 (struct mtip_compat_ide_task_request_s __user *) arg; 3607 3608 if (copy_from_user(&req_task, (void __user *) arg, 3609 compat_tasksize - (2 * sizeof(compat_long_t)))) 3610 return -EFAULT; 3611 3612 if (get_user(req_task.out_size, &compat_req_task->out_size)) 3613 return -EFAULT; 3614 3615 if (get_user(req_task.in_size, &compat_req_task->in_size)) 3616 return -EFAULT; 3617 3618 outtotal = sizeof(struct mtip_compat_ide_task_request_s); 3619 3620 ret = exec_drive_taskfile(dd, (void __user *) arg, 3621 &req_task, outtotal); 3622 3623 if (copy_to_user((void __user *) arg, &req_task, 3624 compat_tasksize - 3625 (2 * sizeof(compat_long_t)))) 3626 return -EFAULT; 3627 3628 if (put_user(req_task.out_size, &compat_req_task->out_size)) 3629 return -EFAULT; 3630 3631 if (put_user(req_task.in_size, &compat_req_task->in_size)) 3632 return -EFAULT; 3633 3634 return ret; 3635 } 3636 default: 3637 return mtip_hw_ioctl(dd, cmd, arg); 3638 } 3639 } 3640 #endif 3641 3642 /* 3643 * Obtain the geometry of the device. 3644 * 3645 * You may think that this function is obsolete, but some applications, 3646 * fdisk for example still used CHS values. This function describes the 3647 * device as having 224 heads and 56 sectors per cylinder. These values are 3648 * chosen so that each cylinder is aligned on a 4KB boundary. Since a 3649 * partition is described in terms of a start and end cylinder this means 3650 * that each partition is also 4KB aligned. Non-aligned partitions adversely 3651 * affects performance. 3652 * 3653 * @dev Pointer to the block_device strucutre. 3654 * @geo Pointer to a hd_geometry structure. 3655 * 3656 * return value 3657 * 0 Operation completed successfully. 3658 * -ENOTTY An error occurred while reading the drive capacity. 3659 */ 3660 static int mtip_block_getgeo(struct block_device *dev, 3661 struct hd_geometry *geo) 3662 { 3663 struct driver_data *dd = dev->bd_disk->private_data; 3664 sector_t capacity; 3665 3666 if (!dd) 3667 return -ENOTTY; 3668 3669 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3670 dev_warn(&dd->pdev->dev, 3671 "Could not get drive capacity.\n"); 3672 return -ENOTTY; 3673 } 3674 3675 geo->heads = 224; 3676 geo->sectors = 56; 3677 sector_div(capacity, (geo->heads * geo->sectors)); 3678 geo->cylinders = capacity; 3679 return 0; 3680 } 3681 3682 /* 3683 * Block device operation function. 3684 * 3685 * This structure contains pointers to the functions required by the block 3686 * layer. 3687 */ 3688 static const struct block_device_operations mtip_block_ops = { 3689 .ioctl = mtip_block_ioctl, 3690 #ifdef CONFIG_COMPAT 3691 .compat_ioctl = mtip_block_compat_ioctl, 3692 #endif 3693 .getgeo = mtip_block_getgeo, 3694 .owner = THIS_MODULE 3695 }; 3696 3697 /* 3698 * Block layer make request function. 3699 * 3700 * This function is called by the kernel to process a BIO for 3701 * the P320 device. 3702 * 3703 * @queue Pointer to the request queue. Unused other than to obtain 3704 * the driver data structure. 3705 * @rq Pointer to the request. 3706 * 3707 */ 3708 static int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq) 3709 { 3710 struct driver_data *dd = hctx->queue->queuedata; 3711 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3712 unsigned int nents; 3713 3714 if (unlikely(dd->dd_flag & MTIP_DDF_STOP_IO)) { 3715 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 3716 &dd->dd_flag))) { 3717 return -ENXIO; 3718 } 3719 if (unlikely(test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))) { 3720 return -ENODATA; 3721 } 3722 if (unlikely(test_bit(MTIP_DDF_WRITE_PROTECT_BIT, 3723 &dd->dd_flag) && 3724 rq_data_dir(rq))) { 3725 return -ENODATA; 3726 } 3727 if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))) 3728 return -ENODATA; 3729 if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) 3730 return -ENXIO; 3731 } 3732 3733 if (rq->cmd_flags & REQ_DISCARD) { 3734 int err; 3735 3736 err = mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq)); 3737 blk_mq_end_io(rq, err); 3738 return 0; 3739 } 3740 3741 /* Create the scatter list for this request. */ 3742 nents = blk_rq_map_sg(hctx->queue, rq, cmd->sg); 3743 3744 /* Issue the read/write. */ 3745 mtip_hw_submit_io(dd, rq, cmd, nents, hctx); 3746 return 0; 3747 } 3748 3749 static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx, 3750 struct request *rq) 3751 { 3752 struct driver_data *dd = hctx->queue->queuedata; 3753 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3754 3755 if (!dd->unal_qdepth || rq_data_dir(rq) == READ) 3756 return false; 3757 3758 /* 3759 * If unaligned depth must be limited on this controller, mark it 3760 * as unaligned if the IO isn't on a 4k boundary (start of length). 3761 */ 3762 if (blk_rq_sectors(rq) <= 64) { 3763 if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7)) 3764 cmd->unaligned = 1; 3765 } 3766 3767 if (cmd->unaligned && down_trylock(&dd->port->cmd_slot_unal)) 3768 return true; 3769 3770 return false; 3771 } 3772 3773 static int mtip_queue_rq(struct blk_mq_hw_ctx *hctx, struct request *rq) 3774 { 3775 int ret; 3776 3777 if (mtip_check_unal_depth(hctx, rq)) 3778 return BLK_MQ_RQ_QUEUE_BUSY; 3779 3780 ret = mtip_submit_request(hctx, rq); 3781 if (!ret) 3782 return BLK_MQ_RQ_QUEUE_OK; 3783 3784 rq->errors = ret; 3785 return BLK_MQ_RQ_QUEUE_ERROR; 3786 } 3787 3788 static void mtip_free_cmd(void *data, struct request *rq, 3789 unsigned int hctx_idx, unsigned int request_idx) 3790 { 3791 struct driver_data *dd = data; 3792 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3793 3794 if (!cmd->command) 3795 return; 3796 3797 dmam_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, 3798 cmd->command, cmd->command_dma); 3799 } 3800 3801 static int mtip_init_cmd(void *data, struct request *rq, unsigned int hctx_idx, 3802 unsigned int request_idx, unsigned int numa_node) 3803 { 3804 struct driver_data *dd = data; 3805 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3806 u32 host_cap_64 = readl(dd->mmio + HOST_CAP) & HOST_CAP_64; 3807 3808 cmd->command = dmam_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, 3809 &cmd->command_dma, GFP_KERNEL); 3810 if (!cmd->command) 3811 return -ENOMEM; 3812 3813 memset(cmd->command, 0, CMD_DMA_ALLOC_SZ); 3814 3815 /* Point the command headers at the command tables. */ 3816 cmd->command_header = dd->port->command_list + 3817 (sizeof(struct mtip_cmd_hdr) * request_idx); 3818 cmd->command_header_dma = dd->port->command_list_dma + 3819 (sizeof(struct mtip_cmd_hdr) * request_idx); 3820 3821 if (host_cap_64) 3822 cmd->command_header->ctbau = __force_bit2int cpu_to_le32((cmd->command_dma >> 16) >> 16); 3823 3824 cmd->command_header->ctba = __force_bit2int cpu_to_le32(cmd->command_dma & 0xFFFFFFFF); 3825 3826 sg_init_table(cmd->sg, MTIP_MAX_SG); 3827 return 0; 3828 } 3829 3830 static struct blk_mq_ops mtip_mq_ops = { 3831 .queue_rq = mtip_queue_rq, 3832 .map_queue = blk_mq_map_queue, 3833 .init_request = mtip_init_cmd, 3834 .exit_request = mtip_free_cmd, 3835 }; 3836 3837 /* 3838 * Block layer initialization function. 3839 * 3840 * This function is called once by the PCI layer for each P320 3841 * device that is connected to the system. 3842 * 3843 * @dd Pointer to the driver data structure. 3844 * 3845 * return value 3846 * 0 on success else an error code. 3847 */ 3848 static int mtip_block_initialize(struct driver_data *dd) 3849 { 3850 int rv = 0, wait_for_rebuild = 0; 3851 sector_t capacity; 3852 unsigned int index = 0; 3853 struct kobject *kobj; 3854 unsigned char thd_name[16]; 3855 3856 if (dd->disk) 3857 goto skip_create_disk; /* hw init done, before rebuild */ 3858 3859 if (mtip_hw_init(dd)) { 3860 rv = -EINVAL; 3861 goto protocol_init_error; 3862 } 3863 3864 dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node); 3865 if (dd->disk == NULL) { 3866 dev_err(&dd->pdev->dev, 3867 "Unable to allocate gendisk structure\n"); 3868 rv = -EINVAL; 3869 goto alloc_disk_error; 3870 } 3871 3872 /* Generate the disk name, implemented same as in sd.c */ 3873 do { 3874 if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL)) 3875 goto ida_get_error; 3876 3877 spin_lock(&rssd_index_lock); 3878 rv = ida_get_new(&rssd_index_ida, &index); 3879 spin_unlock(&rssd_index_lock); 3880 } while (rv == -EAGAIN); 3881 3882 if (rv) 3883 goto ida_get_error; 3884 3885 rv = rssd_disk_name_format("rssd", 3886 index, 3887 dd->disk->disk_name, 3888 DISK_NAME_LEN); 3889 if (rv) 3890 goto disk_index_error; 3891 3892 dd->disk->driverfs_dev = &dd->pdev->dev; 3893 dd->disk->major = dd->major; 3894 dd->disk->first_minor = dd->instance * MTIP_MAX_MINORS; 3895 dd->disk->fops = &mtip_block_ops; 3896 dd->disk->private_data = dd; 3897 dd->index = index; 3898 3899 mtip_hw_debugfs_init(dd); 3900 3901 skip_create_disk: 3902 memset(&dd->tags, 0, sizeof(dd->tags)); 3903 dd->tags.ops = &mtip_mq_ops; 3904 dd->tags.nr_hw_queues = 1; 3905 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; 3906 dd->tags.reserved_tags = 1; 3907 dd->tags.cmd_size = sizeof(struct mtip_cmd); 3908 dd->tags.numa_node = dd->numa_node; 3909 dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; 3910 dd->tags.driver_data = dd; 3911 3912 rv = blk_mq_alloc_tag_set(&dd->tags); 3913 if (rv) { 3914 dev_err(&dd->pdev->dev, 3915 "Unable to allocate request queue\n"); 3916 rv = -ENOMEM; 3917 goto block_queue_alloc_init_error; 3918 } 3919 3920 /* Allocate the request queue. */ 3921 dd->queue = blk_mq_init_queue(&dd->tags); 3922 if (IS_ERR(dd->queue)) { 3923 dev_err(&dd->pdev->dev, 3924 "Unable to allocate request queue\n"); 3925 rv = -ENOMEM; 3926 goto block_queue_alloc_init_error; 3927 } 3928 3929 dd->disk->queue = dd->queue; 3930 dd->queue->queuedata = dd; 3931 3932 /* Initialize the protocol layer. */ 3933 wait_for_rebuild = mtip_hw_get_identify(dd); 3934 if (wait_for_rebuild < 0) { 3935 dev_err(&dd->pdev->dev, 3936 "Protocol layer initialization failed\n"); 3937 rv = -EINVAL; 3938 goto init_hw_cmds_error; 3939 } 3940 3941 /* 3942 * if rebuild pending, start the service thread, and delay the block 3943 * queue creation and add_disk() 3944 */ 3945 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3946 goto start_service_thread; 3947 3948 /* Set device limits. */ 3949 set_bit(QUEUE_FLAG_NONROT, &dd->queue->queue_flags); 3950 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); 3951 blk_queue_physical_block_size(dd->queue, 4096); 3952 blk_queue_max_hw_sectors(dd->queue, 0xffff); 3953 blk_queue_max_segment_size(dd->queue, 0x400000); 3954 blk_queue_io_min(dd->queue, 4096); 3955 blk_queue_bounce_limit(dd->queue, dd->pdev->dma_mask); 3956 3957 /* 3958 * write back cache is not supported in the device. FUA depends on 3959 * write back cache support, hence setting flush support to zero. 3960 */ 3961 blk_queue_flush(dd->queue, 0); 3962 3963 /* Signal trim support */ 3964 if (dd->trim_supp == true) { 3965 set_bit(QUEUE_FLAG_DISCARD, &dd->queue->queue_flags); 3966 dd->queue->limits.discard_granularity = 4096; 3967 blk_queue_max_discard_sectors(dd->queue, 3968 MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES); 3969 dd->queue->limits.discard_zeroes_data = 0; 3970 } 3971 3972 /* Set the capacity of the device in 512 byte sectors. */ 3973 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3974 dev_warn(&dd->pdev->dev, 3975 "Could not read drive capacity\n"); 3976 rv = -EIO; 3977 goto read_capacity_error; 3978 } 3979 set_capacity(dd->disk, capacity); 3980 3981 /* Enable the block device and add it to /dev */ 3982 add_disk(dd->disk); 3983 3984 dd->bdev = bdget_disk(dd->disk, 0); 3985 /* 3986 * Now that the disk is active, initialize any sysfs attributes 3987 * managed by the protocol layer. 3988 */ 3989 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 3990 if (kobj) { 3991 mtip_hw_sysfs_init(dd, kobj); 3992 kobject_put(kobj); 3993 } 3994 3995 if (dd->mtip_svc_handler) { 3996 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 3997 return rv; /* service thread created for handling rebuild */ 3998 } 3999 4000 start_service_thread: 4001 sprintf(thd_name, "mtip_svc_thd_%02d", index); 4002 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, 4003 dd, dd->numa_node, "%s", 4004 thd_name); 4005 4006 if (IS_ERR(dd->mtip_svc_handler)) { 4007 dev_err(&dd->pdev->dev, "service thread failed to start\n"); 4008 dd->mtip_svc_handler = NULL; 4009 rv = -EFAULT; 4010 goto kthread_run_error; 4011 } 4012 wake_up_process(dd->mtip_svc_handler); 4013 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 4014 rv = wait_for_rebuild; 4015 4016 return rv; 4017 4018 kthread_run_error: 4019 bdput(dd->bdev); 4020 dd->bdev = NULL; 4021 4022 /* Delete our gendisk. This also removes the device from /dev */ 4023 del_gendisk(dd->disk); 4024 4025 read_capacity_error: 4026 init_hw_cmds_error: 4027 blk_cleanup_queue(dd->queue); 4028 blk_mq_free_tag_set(&dd->tags); 4029 block_queue_alloc_init_error: 4030 mtip_hw_debugfs_exit(dd); 4031 disk_index_error: 4032 spin_lock(&rssd_index_lock); 4033 ida_remove(&rssd_index_ida, index); 4034 spin_unlock(&rssd_index_lock); 4035 4036 ida_get_error: 4037 put_disk(dd->disk); 4038 4039 alloc_disk_error: 4040 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ 4041 4042 protocol_init_error: 4043 return rv; 4044 } 4045 4046 /* 4047 * Block layer deinitialization function. 4048 * 4049 * Called by the PCI layer as each P320 device is removed. 4050 * 4051 * @dd Pointer to the driver data structure. 4052 * 4053 * return value 4054 * 0 4055 */ 4056 static int mtip_block_remove(struct driver_data *dd) 4057 { 4058 struct kobject *kobj; 4059 4060 if (!dd->sr) { 4061 mtip_hw_debugfs_exit(dd); 4062 4063 if (dd->mtip_svc_handler) { 4064 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); 4065 wake_up_interruptible(&dd->port->svc_wait); 4066 kthread_stop(dd->mtip_svc_handler); 4067 } 4068 4069 /* Clean up the sysfs attributes, if created */ 4070 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) { 4071 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 4072 if (kobj) { 4073 mtip_hw_sysfs_exit(dd, kobj); 4074 kobject_put(kobj); 4075 } 4076 } 4077 4078 mtip_standby_drive(dd); 4079 4080 /* 4081 * Delete our gendisk structure. This also removes the device 4082 * from /dev 4083 */ 4084 if (dd->bdev) { 4085 bdput(dd->bdev); 4086 dd->bdev = NULL; 4087 } 4088 if (dd->disk) { 4089 if (dd->disk->queue) { 4090 del_gendisk(dd->disk); 4091 blk_cleanup_queue(dd->queue); 4092 blk_mq_free_tag_set(&dd->tags); 4093 dd->queue = NULL; 4094 } else 4095 put_disk(dd->disk); 4096 } 4097 dd->disk = NULL; 4098 4099 spin_lock(&rssd_index_lock); 4100 ida_remove(&rssd_index_ida, dd->index); 4101 spin_unlock(&rssd_index_lock); 4102 } else { 4103 dev_info(&dd->pdev->dev, "device %s surprise removal\n", 4104 dd->disk->disk_name); 4105 } 4106 4107 /* De-initialize the protocol layer. */ 4108 mtip_hw_exit(dd); 4109 4110 return 0; 4111 } 4112 4113 /* 4114 * Function called by the PCI layer when just before the 4115 * machine shuts down. 4116 * 4117 * If a protocol layer shutdown function is present it will be called 4118 * by this function. 4119 * 4120 * @dd Pointer to the driver data structure. 4121 * 4122 * return value 4123 * 0 4124 */ 4125 static int mtip_block_shutdown(struct driver_data *dd) 4126 { 4127 mtip_hw_shutdown(dd); 4128 4129 /* Delete our gendisk structure, and cleanup the blk queue. */ 4130 if (dd->disk) { 4131 dev_info(&dd->pdev->dev, 4132 "Shutting down %s ...\n", dd->disk->disk_name); 4133 4134 if (dd->disk->queue) { 4135 del_gendisk(dd->disk); 4136 blk_cleanup_queue(dd->queue); 4137 blk_mq_free_tag_set(&dd->tags); 4138 } else 4139 put_disk(dd->disk); 4140 dd->disk = NULL; 4141 dd->queue = NULL; 4142 } 4143 4144 spin_lock(&rssd_index_lock); 4145 ida_remove(&rssd_index_ida, dd->index); 4146 spin_unlock(&rssd_index_lock); 4147 return 0; 4148 } 4149 4150 static int mtip_block_suspend(struct driver_data *dd) 4151 { 4152 dev_info(&dd->pdev->dev, 4153 "Suspending %s ...\n", dd->disk->disk_name); 4154 mtip_hw_suspend(dd); 4155 return 0; 4156 } 4157 4158 static int mtip_block_resume(struct driver_data *dd) 4159 { 4160 dev_info(&dd->pdev->dev, "Resuming %s ...\n", 4161 dd->disk->disk_name); 4162 mtip_hw_resume(dd); 4163 return 0; 4164 } 4165 4166 static void drop_cpu(int cpu) 4167 { 4168 cpu_use[cpu]--; 4169 } 4170 4171 static int get_least_used_cpu_on_node(int node) 4172 { 4173 int cpu, least_used_cpu, least_cnt; 4174 const struct cpumask *node_mask; 4175 4176 node_mask = cpumask_of_node(node); 4177 least_used_cpu = cpumask_first(node_mask); 4178 least_cnt = cpu_use[least_used_cpu]; 4179 cpu = least_used_cpu; 4180 4181 for_each_cpu(cpu, node_mask) { 4182 if (cpu_use[cpu] < least_cnt) { 4183 least_used_cpu = cpu; 4184 least_cnt = cpu_use[cpu]; 4185 } 4186 } 4187 cpu_use[least_used_cpu]++; 4188 return least_used_cpu; 4189 } 4190 4191 /* Helper for selecting a node in round robin mode */ 4192 static inline int mtip_get_next_rr_node(void) 4193 { 4194 static int next_node = -1; 4195 4196 if (next_node == -1) { 4197 next_node = first_online_node; 4198 return next_node; 4199 } 4200 4201 next_node = next_online_node(next_node); 4202 if (next_node == MAX_NUMNODES) 4203 next_node = first_online_node; 4204 return next_node; 4205 } 4206 4207 static DEFINE_HANDLER(0); 4208 static DEFINE_HANDLER(1); 4209 static DEFINE_HANDLER(2); 4210 static DEFINE_HANDLER(3); 4211 static DEFINE_HANDLER(4); 4212 static DEFINE_HANDLER(5); 4213 static DEFINE_HANDLER(6); 4214 static DEFINE_HANDLER(7); 4215 4216 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev) 4217 { 4218 int pos; 4219 unsigned short pcie_dev_ctrl; 4220 4221 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); 4222 if (pos) { 4223 pci_read_config_word(pdev, 4224 pos + PCI_EXP_DEVCTL, 4225 &pcie_dev_ctrl); 4226 if (pcie_dev_ctrl & (1 << 11) || 4227 pcie_dev_ctrl & (1 << 4)) { 4228 dev_info(&dd->pdev->dev, 4229 "Disabling ERO/No-Snoop on bridge device %04x:%04x\n", 4230 pdev->vendor, pdev->device); 4231 pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN | 4232 PCI_EXP_DEVCTL_RELAX_EN); 4233 pci_write_config_word(pdev, 4234 pos + PCI_EXP_DEVCTL, 4235 pcie_dev_ctrl); 4236 } 4237 } 4238 } 4239 4240 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev) 4241 { 4242 /* 4243 * This workaround is specific to AMD/ATI chipset with a PCI upstream 4244 * device with device id 0x5aXX 4245 */ 4246 if (pdev->bus && pdev->bus->self) { 4247 if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI && 4248 ((pdev->bus->self->device & 0xff00) == 0x5a00)) { 4249 mtip_disable_link_opts(dd, pdev->bus->self); 4250 } else { 4251 /* Check further up the topology */ 4252 struct pci_dev *parent_dev = pdev->bus->self; 4253 if (parent_dev->bus && 4254 parent_dev->bus->parent && 4255 parent_dev->bus->parent->self && 4256 parent_dev->bus->parent->self->vendor == 4257 PCI_VENDOR_ID_ATI && 4258 (parent_dev->bus->parent->self->device & 4259 0xff00) == 0x5a00) { 4260 mtip_disable_link_opts(dd, 4261 parent_dev->bus->parent->self); 4262 } 4263 } 4264 } 4265 } 4266 4267 /* 4268 * Called for each supported PCI device detected. 4269 * 4270 * This function allocates the private data structure, enables the 4271 * PCI device and then calls the block layer initialization function. 4272 * 4273 * return value 4274 * 0 on success else an error code. 4275 */ 4276 static int mtip_pci_probe(struct pci_dev *pdev, 4277 const struct pci_device_id *ent) 4278 { 4279 int rv = 0; 4280 struct driver_data *dd = NULL; 4281 char cpu_list[256]; 4282 const struct cpumask *node_mask; 4283 int cpu, i = 0, j = 0; 4284 int my_node = NUMA_NO_NODE; 4285 unsigned long flags; 4286 4287 /* Allocate memory for this devices private data. */ 4288 my_node = pcibus_to_node(pdev->bus); 4289 if (my_node != NUMA_NO_NODE) { 4290 if (!node_online(my_node)) 4291 my_node = mtip_get_next_rr_node(); 4292 } else { 4293 dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n"); 4294 my_node = mtip_get_next_rr_node(); 4295 } 4296 dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n", 4297 my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev), 4298 cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id()); 4299 4300 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); 4301 if (dd == NULL) { 4302 dev_err(&pdev->dev, 4303 "Unable to allocate memory for driver data\n"); 4304 return -ENOMEM; 4305 } 4306 4307 /* Attach the private data to this PCI device. */ 4308 pci_set_drvdata(pdev, dd); 4309 4310 rv = pcim_enable_device(pdev); 4311 if (rv < 0) { 4312 dev_err(&pdev->dev, "Unable to enable device\n"); 4313 goto iomap_err; 4314 } 4315 4316 /* Map BAR5 to memory. */ 4317 rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME); 4318 if (rv < 0) { 4319 dev_err(&pdev->dev, "Unable to map regions\n"); 4320 goto iomap_err; 4321 } 4322 4323 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 4324 rv = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4325 4326 if (rv) { 4327 rv = pci_set_consistent_dma_mask(pdev, 4328 DMA_BIT_MASK(32)); 4329 if (rv) { 4330 dev_warn(&pdev->dev, 4331 "64-bit DMA enable failed\n"); 4332 goto setmask_err; 4333 } 4334 } 4335 } 4336 4337 /* Copy the info we may need later into the private data structure. */ 4338 dd->major = mtip_major; 4339 dd->instance = instance; 4340 dd->pdev = pdev; 4341 dd->numa_node = my_node; 4342 4343 INIT_LIST_HEAD(&dd->online_list); 4344 INIT_LIST_HEAD(&dd->remove_list); 4345 4346 memset(dd->workq_name, 0, 32); 4347 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); 4348 4349 dd->isr_workq = create_workqueue(dd->workq_name); 4350 if (!dd->isr_workq) { 4351 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); 4352 rv = -ENOMEM; 4353 goto block_initialize_err; 4354 } 4355 4356 memset(cpu_list, 0, sizeof(cpu_list)); 4357 4358 node_mask = cpumask_of_node(dd->numa_node); 4359 if (!cpumask_empty(node_mask)) { 4360 for_each_cpu(cpu, node_mask) 4361 { 4362 snprintf(&cpu_list[j], 256 - j, "%d ", cpu); 4363 j = strlen(cpu_list); 4364 } 4365 4366 dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n", 4367 dd->numa_node, 4368 topology_physical_package_id(cpumask_first(node_mask)), 4369 nr_cpus_node(dd->numa_node), 4370 cpu_list); 4371 } else 4372 dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n"); 4373 4374 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); 4375 dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n", 4376 cpu_to_node(dd->isr_binding), dd->isr_binding); 4377 4378 /* first worker context always runs in ISR */ 4379 dd->work[0].cpu_binding = dd->isr_binding; 4380 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4381 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4382 dd->work[3].cpu_binding = dd->work[0].cpu_binding; 4383 dd->work[4].cpu_binding = dd->work[1].cpu_binding; 4384 dd->work[5].cpu_binding = dd->work[2].cpu_binding; 4385 dd->work[6].cpu_binding = dd->work[2].cpu_binding; 4386 dd->work[7].cpu_binding = dd->work[1].cpu_binding; 4387 4388 /* Log the bindings */ 4389 for_each_present_cpu(cpu) { 4390 memset(cpu_list, 0, sizeof(cpu_list)); 4391 for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) { 4392 if (dd->work[i].cpu_binding == cpu) { 4393 snprintf(&cpu_list[j], 256 - j, "%d ", i); 4394 j = strlen(cpu_list); 4395 } 4396 } 4397 if (j) 4398 dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list); 4399 } 4400 4401 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); 4402 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); 4403 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); 4404 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); 4405 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); 4406 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); 4407 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); 4408 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); 4409 4410 pci_set_master(pdev); 4411 rv = pci_enable_msi(pdev); 4412 if (rv) { 4413 dev_warn(&pdev->dev, 4414 "Unable to enable MSI interrupt.\n"); 4415 goto msi_initialize_err; 4416 } 4417 4418 mtip_fix_ero_nosnoop(dd, pdev); 4419 4420 /* Initialize the block layer. */ 4421 rv = mtip_block_initialize(dd); 4422 if (rv < 0) { 4423 dev_err(&pdev->dev, 4424 "Unable to initialize block layer\n"); 4425 goto block_initialize_err; 4426 } 4427 4428 /* 4429 * Increment the instance count so that each device has a unique 4430 * instance number. 4431 */ 4432 instance++; 4433 if (rv != MTIP_FTL_REBUILD_MAGIC) 4434 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 4435 else 4436 rv = 0; /* device in rebuild state, return 0 from probe */ 4437 4438 /* Add to online list even if in ftl rebuild */ 4439 spin_lock_irqsave(&dev_lock, flags); 4440 list_add(&dd->online_list, &online_list); 4441 spin_unlock_irqrestore(&dev_lock, flags); 4442 4443 goto done; 4444 4445 block_initialize_err: 4446 pci_disable_msi(pdev); 4447 4448 msi_initialize_err: 4449 if (dd->isr_workq) { 4450 flush_workqueue(dd->isr_workq); 4451 destroy_workqueue(dd->isr_workq); 4452 drop_cpu(dd->work[0].cpu_binding); 4453 drop_cpu(dd->work[1].cpu_binding); 4454 drop_cpu(dd->work[2].cpu_binding); 4455 } 4456 setmask_err: 4457 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4458 4459 iomap_err: 4460 kfree(dd); 4461 pci_set_drvdata(pdev, NULL); 4462 return rv; 4463 done: 4464 return rv; 4465 } 4466 4467 /* 4468 * Called for each probed device when the device is removed or the 4469 * driver is unloaded. 4470 * 4471 * return value 4472 * None 4473 */ 4474 static void mtip_pci_remove(struct pci_dev *pdev) 4475 { 4476 struct driver_data *dd = pci_get_drvdata(pdev); 4477 unsigned long flags, to; 4478 4479 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); 4480 4481 spin_lock_irqsave(&dev_lock, flags); 4482 list_del_init(&dd->online_list); 4483 list_add(&dd->remove_list, &removing_list); 4484 spin_unlock_irqrestore(&dev_lock, flags); 4485 4486 mtip_check_surprise_removal(pdev); 4487 synchronize_irq(dd->pdev->irq); 4488 4489 /* Spin until workers are done */ 4490 to = jiffies + msecs_to_jiffies(4000); 4491 do { 4492 msleep(20); 4493 } while (atomic_read(&dd->irq_workers_active) != 0 && 4494 time_before(jiffies, to)); 4495 4496 if (atomic_read(&dd->irq_workers_active) != 0) { 4497 dev_warn(&dd->pdev->dev, 4498 "Completion workers still active!\n"); 4499 } 4500 4501 /* Clean up the block layer. */ 4502 mtip_block_remove(dd); 4503 4504 if (dd->isr_workq) { 4505 flush_workqueue(dd->isr_workq); 4506 destroy_workqueue(dd->isr_workq); 4507 drop_cpu(dd->work[0].cpu_binding); 4508 drop_cpu(dd->work[1].cpu_binding); 4509 drop_cpu(dd->work[2].cpu_binding); 4510 } 4511 4512 pci_disable_msi(pdev); 4513 4514 spin_lock_irqsave(&dev_lock, flags); 4515 list_del_init(&dd->remove_list); 4516 spin_unlock_irqrestore(&dev_lock, flags); 4517 4518 if (!dd->sr) 4519 kfree(dd); 4520 else 4521 set_bit(MTIP_DDF_REMOVE_DONE_BIT, &dd->dd_flag); 4522 4523 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4524 pci_set_drvdata(pdev, NULL); 4525 } 4526 4527 /* 4528 * Called for each probed device when the device is suspended. 4529 * 4530 * return value 4531 * 0 Success 4532 * <0 Error 4533 */ 4534 static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg) 4535 { 4536 int rv = 0; 4537 struct driver_data *dd = pci_get_drvdata(pdev); 4538 4539 if (!dd) { 4540 dev_err(&pdev->dev, 4541 "Driver private datastructure is NULL\n"); 4542 return -EFAULT; 4543 } 4544 4545 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4546 4547 /* Disable ports & interrupts then send standby immediate */ 4548 rv = mtip_block_suspend(dd); 4549 if (rv < 0) { 4550 dev_err(&pdev->dev, 4551 "Failed to suspend controller\n"); 4552 return rv; 4553 } 4554 4555 /* 4556 * Save the pci config space to pdev structure & 4557 * disable the device 4558 */ 4559 pci_save_state(pdev); 4560 pci_disable_device(pdev); 4561 4562 /* Move to Low power state*/ 4563 pci_set_power_state(pdev, PCI_D3hot); 4564 4565 return rv; 4566 } 4567 4568 /* 4569 * Called for each probed device when the device is resumed. 4570 * 4571 * return value 4572 * 0 Success 4573 * <0 Error 4574 */ 4575 static int mtip_pci_resume(struct pci_dev *pdev) 4576 { 4577 int rv = 0; 4578 struct driver_data *dd; 4579 4580 dd = pci_get_drvdata(pdev); 4581 if (!dd) { 4582 dev_err(&pdev->dev, 4583 "Driver private datastructure is NULL\n"); 4584 return -EFAULT; 4585 } 4586 4587 /* Move the device to active State */ 4588 pci_set_power_state(pdev, PCI_D0); 4589 4590 /* Restore PCI configuration space */ 4591 pci_restore_state(pdev); 4592 4593 /* Enable the PCI device*/ 4594 rv = pcim_enable_device(pdev); 4595 if (rv < 0) { 4596 dev_err(&pdev->dev, 4597 "Failed to enable card during resume\n"); 4598 goto err; 4599 } 4600 pci_set_master(pdev); 4601 4602 /* 4603 * Calls hbaReset, initPort, & startPort function 4604 * then enables interrupts 4605 */ 4606 rv = mtip_block_resume(dd); 4607 if (rv < 0) 4608 dev_err(&pdev->dev, "Unable to resume\n"); 4609 4610 err: 4611 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4612 4613 return rv; 4614 } 4615 4616 /* 4617 * Shutdown routine 4618 * 4619 * return value 4620 * None 4621 */ 4622 static void mtip_pci_shutdown(struct pci_dev *pdev) 4623 { 4624 struct driver_data *dd = pci_get_drvdata(pdev); 4625 if (dd) 4626 mtip_block_shutdown(dd); 4627 } 4628 4629 /* Table of device ids supported by this driver. */ 4630 static DEFINE_PCI_DEVICE_TABLE(mtip_pci_tbl) = { 4631 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) }, 4632 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) }, 4633 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) }, 4634 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) }, 4635 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) }, 4636 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) }, 4637 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) }, 4638 { 0 } 4639 }; 4640 4641 /* Structure that describes the PCI driver functions. */ 4642 static struct pci_driver mtip_pci_driver = { 4643 .name = MTIP_DRV_NAME, 4644 .id_table = mtip_pci_tbl, 4645 .probe = mtip_pci_probe, 4646 .remove = mtip_pci_remove, 4647 .suspend = mtip_pci_suspend, 4648 .resume = mtip_pci_resume, 4649 .shutdown = mtip_pci_shutdown, 4650 }; 4651 4652 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl); 4653 4654 /* 4655 * Module initialization function. 4656 * 4657 * Called once when the module is loaded. This function allocates a major 4658 * block device number to the Cyclone devices and registers the PCI layer 4659 * of the driver. 4660 * 4661 * Return value 4662 * 0 on success else error code. 4663 */ 4664 static int __init mtip_init(void) 4665 { 4666 int error; 4667 4668 pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n"); 4669 4670 spin_lock_init(&dev_lock); 4671 4672 INIT_LIST_HEAD(&online_list); 4673 INIT_LIST_HEAD(&removing_list); 4674 4675 /* Allocate a major block device number to use with this driver. */ 4676 error = register_blkdev(0, MTIP_DRV_NAME); 4677 if (error <= 0) { 4678 pr_err("Unable to register block device (%d)\n", 4679 error); 4680 return -EBUSY; 4681 } 4682 mtip_major = error; 4683 4684 dfs_parent = debugfs_create_dir("rssd", NULL); 4685 if (IS_ERR_OR_NULL(dfs_parent)) { 4686 pr_warn("Error creating debugfs parent\n"); 4687 dfs_parent = NULL; 4688 } 4689 if (dfs_parent) { 4690 dfs_device_status = debugfs_create_file("device_status", 4691 S_IRUGO, dfs_parent, NULL, 4692 &mtip_device_status_fops); 4693 if (IS_ERR_OR_NULL(dfs_device_status)) { 4694 pr_err("Error creating device_status node\n"); 4695 dfs_device_status = NULL; 4696 } 4697 } 4698 4699 /* Register our PCI operations. */ 4700 error = pci_register_driver(&mtip_pci_driver); 4701 if (error) { 4702 debugfs_remove(dfs_parent); 4703 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4704 } 4705 4706 return error; 4707 } 4708 4709 /* 4710 * Module de-initialization function. 4711 * 4712 * Called once when the module is unloaded. This function deallocates 4713 * the major block device number allocated by mtip_init() and 4714 * unregisters the PCI layer of the driver. 4715 * 4716 * Return value 4717 * none 4718 */ 4719 static void __exit mtip_exit(void) 4720 { 4721 /* Release the allocated major block device number. */ 4722 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4723 4724 /* Unregister the PCI driver. */ 4725 pci_unregister_driver(&mtip_pci_driver); 4726 4727 debugfs_remove_recursive(dfs_parent); 4728 } 4729 4730 MODULE_AUTHOR("Micron Technology, Inc"); 4731 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver"); 4732 MODULE_LICENSE("GPL"); 4733 MODULE_VERSION(MTIP_DRV_VERSION); 4734 4735 module_init(mtip_init); 4736 module_exit(mtip_exit); 4737