1 /* 2 * Driver for the Micron P320 SSD 3 * Copyright (C) 2011 Micron Technology, Inc. 4 * 5 * Portions of this code were derived from works subjected to the 6 * following copyright: 7 * Copyright (C) 2009 Integrated Device Technology, Inc. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published by 11 * the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 */ 20 21 #include <linux/pci.h> 22 #include <linux/interrupt.h> 23 #include <linux/ata.h> 24 #include <linux/delay.h> 25 #include <linux/hdreg.h> 26 #include <linux/uaccess.h> 27 #include <linux/random.h> 28 #include <linux/smp.h> 29 #include <linux/compat.h> 30 #include <linux/fs.h> 31 #include <linux/module.h> 32 #include <linux/genhd.h> 33 #include <linux/blkdev.h> 34 #include <linux/blk-mq.h> 35 #include <linux/bio.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/idr.h> 38 #include <linux/kthread.h> 39 #include <../drivers/ata/ahci.h> 40 #include <linux/export.h> 41 #include <linux/debugfs.h> 42 #include <linux/prefetch.h> 43 #include "mtip32xx.h" 44 45 #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) 46 47 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */ 48 #define AHCI_RX_FIS_SZ 0x100 49 #define AHCI_RX_FIS_OFFSET 0x0 50 #define AHCI_IDFY_SZ ATA_SECT_SIZE 51 #define AHCI_IDFY_OFFSET 0x400 52 #define AHCI_SECTBUF_SZ ATA_SECT_SIZE 53 #define AHCI_SECTBUF_OFFSET 0x800 54 #define AHCI_SMARTBUF_SZ ATA_SECT_SIZE 55 #define AHCI_SMARTBUF_OFFSET 0xC00 56 /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */ 57 #define BLOCK_DMA_ALLOC_SZ 4096 58 59 /* DMA region containing command table (should be 8192 bytes) */ 60 #define AHCI_CMD_SLOT_SZ sizeof(struct mtip_cmd_hdr) 61 #define AHCI_CMD_TBL_SZ (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ) 62 #define AHCI_CMD_TBL_OFFSET 0x0 63 64 /* DMA region per command (contains header and SGL) */ 65 #define AHCI_CMD_TBL_HDR_SZ 0x80 66 #define AHCI_CMD_TBL_HDR_OFFSET 0x0 67 #define AHCI_CMD_TBL_SGL_SZ (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg)) 68 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ 69 #define CMD_DMA_ALLOC_SZ (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ) 70 71 72 #define HOST_CAP_NZDMA (1 << 19) 73 #define HOST_HSORG 0xFC 74 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24) 75 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) 76 #define HSORG_HWREV 0xFF00 77 #define HSORG_STYLE 0x8 78 #define HSORG_SLOTGROUPS 0x7 79 80 #define PORT_COMMAND_ISSUE 0x38 81 #define PORT_SDBV 0x7C 82 83 #define PORT_OFFSET 0x100 84 #define PORT_MEM_SIZE 0x80 85 86 #define PORT_IRQ_ERR \ 87 (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \ 88 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \ 89 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \ 90 PORT_IRQ_OVERFLOW) 91 #define PORT_IRQ_LEGACY \ 92 (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS) 93 #define PORT_IRQ_HANDLED \ 94 (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \ 95 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \ 96 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY) 97 #define DEF_PORT_IRQ \ 98 (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS) 99 100 /* product numbers */ 101 #define MTIP_PRODUCT_UNKNOWN 0x00 102 #define MTIP_PRODUCT_ASICFPGA 0x11 103 104 /* Device instance number, incremented each time a device is probed. */ 105 static int instance; 106 107 static struct list_head online_list; 108 static struct list_head removing_list; 109 static spinlock_t dev_lock; 110 111 /* 112 * Global variable used to hold the major block device number 113 * allocated in mtip_init(). 114 */ 115 static int mtip_major; 116 static struct dentry *dfs_parent; 117 static struct dentry *dfs_device_status; 118 119 static u32 cpu_use[NR_CPUS]; 120 121 static DEFINE_SPINLOCK(rssd_index_lock); 122 static DEFINE_IDA(rssd_index_ida); 123 124 static int mtip_block_initialize(struct driver_data *dd); 125 126 #ifdef CONFIG_COMPAT 127 struct mtip_compat_ide_task_request_s { 128 __u8 io_ports[8]; 129 __u8 hob_ports[8]; 130 ide_reg_valid_t out_flags; 131 ide_reg_valid_t in_flags; 132 int data_phase; 133 int req_cmd; 134 compat_ulong_t out_size; 135 compat_ulong_t in_size; 136 }; 137 #endif 138 139 /* 140 * This function check_for_surprise_removal is called 141 * while card is removed from the system and it will 142 * read the vendor id from the configration space 143 * 144 * @pdev Pointer to the pci_dev structure. 145 * 146 * return value 147 * true if device removed, else false 148 */ 149 static bool mtip_check_surprise_removal(struct pci_dev *pdev) 150 { 151 u16 vendor_id = 0; 152 struct driver_data *dd = pci_get_drvdata(pdev); 153 154 if (dd->sr) 155 return true; 156 157 /* Read the vendorID from the configuration space */ 158 pci_read_config_word(pdev, 0x00, &vendor_id); 159 if (vendor_id == 0xFFFF) { 160 dd->sr = true; 161 if (dd->queue) 162 blk_queue_flag_set(QUEUE_FLAG_DEAD, dd->queue); 163 else 164 dev_warn(&dd->pdev->dev, 165 "%s: dd->queue is NULL\n", __func__); 166 return true; /* device removed */ 167 } 168 169 return false; /* device present */ 170 } 171 172 /* we have to use runtime tag to setup command header */ 173 static void mtip_init_cmd_header(struct request *rq) 174 { 175 struct driver_data *dd = rq->q->queuedata; 176 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 177 178 /* Point the command headers at the command tables. */ 179 cmd->command_header = dd->port->command_list + 180 (sizeof(struct mtip_cmd_hdr) * rq->tag); 181 cmd->command_header_dma = dd->port->command_list_dma + 182 (sizeof(struct mtip_cmd_hdr) * rq->tag); 183 184 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) 185 cmd->command_header->ctbau = __force_bit2int cpu_to_le32((cmd->command_dma >> 16) >> 16); 186 187 cmd->command_header->ctba = __force_bit2int cpu_to_le32(cmd->command_dma & 0xFFFFFFFF); 188 } 189 190 static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) 191 { 192 struct request *rq; 193 194 if (mtip_check_surprise_removal(dd->pdev)) 195 return NULL; 196 197 rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED); 198 if (IS_ERR(rq)) 199 return NULL; 200 201 /* Internal cmd isn't submitted via .queue_rq */ 202 mtip_init_cmd_header(rq); 203 204 return blk_mq_rq_to_pdu(rq); 205 } 206 207 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, 208 unsigned int tag) 209 { 210 struct blk_mq_hw_ctx *hctx = dd->queue->queue_hw_ctx[0]; 211 212 return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(hctx->tags, tag)); 213 } 214 215 /* 216 * Reset the HBA (without sleeping) 217 * 218 * @dd Pointer to the driver data structure. 219 * 220 * return value 221 * 0 The reset was successful. 222 * -1 The HBA Reset bit did not clear. 223 */ 224 static int mtip_hba_reset(struct driver_data *dd) 225 { 226 unsigned long timeout; 227 228 /* Set the reset bit */ 229 writel(HOST_RESET, dd->mmio + HOST_CTL); 230 231 /* Flush */ 232 readl(dd->mmio + HOST_CTL); 233 234 /* 235 * Spin for up to 10 seconds waiting for reset acknowledgement. Spec 236 * is 1 sec but in LUN failure conditions, up to 10 secs are required 237 */ 238 timeout = jiffies + msecs_to_jiffies(10000); 239 do { 240 mdelay(10); 241 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) 242 return -1; 243 244 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) 245 && time_before(jiffies, timeout)); 246 247 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) 248 return -1; 249 250 return 0; 251 } 252 253 /* 254 * Issue a command to the hardware. 255 * 256 * Set the appropriate bit in the s_active and Command Issue hardware 257 * registers, causing hardware command processing to begin. 258 * 259 * @port Pointer to the port structure. 260 * @tag The tag of the command to be issued. 261 * 262 * return value 263 * None 264 */ 265 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) 266 { 267 int group = tag >> 5; 268 269 /* guard SACT and CI registers */ 270 spin_lock(&port->cmd_issue_lock[group]); 271 writel((1 << MTIP_TAG_BIT(tag)), 272 port->s_active[MTIP_TAG_INDEX(tag)]); 273 writel((1 << MTIP_TAG_BIT(tag)), 274 port->cmd_issue[MTIP_TAG_INDEX(tag)]); 275 spin_unlock(&port->cmd_issue_lock[group]); 276 } 277 278 /* 279 * Enable/disable the reception of FIS 280 * 281 * @port Pointer to the port data structure 282 * @enable 1 to enable, 0 to disable 283 * 284 * return value 285 * Previous state: 1 enabled, 0 disabled 286 */ 287 static int mtip_enable_fis(struct mtip_port *port, int enable) 288 { 289 u32 tmp; 290 291 /* enable FIS reception */ 292 tmp = readl(port->mmio + PORT_CMD); 293 if (enable) 294 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 295 else 296 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 297 298 /* Flush */ 299 readl(port->mmio + PORT_CMD); 300 301 return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX)); 302 } 303 304 /* 305 * Enable/disable the DMA engine 306 * 307 * @port Pointer to the port data structure 308 * @enable 1 to enable, 0 to disable 309 * 310 * return value 311 * Previous state: 1 enabled, 0 disabled. 312 */ 313 static int mtip_enable_engine(struct mtip_port *port, int enable) 314 { 315 u32 tmp; 316 317 /* enable FIS reception */ 318 tmp = readl(port->mmio + PORT_CMD); 319 if (enable) 320 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); 321 else 322 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); 323 324 readl(port->mmio + PORT_CMD); 325 return (((tmp & PORT_CMD_START) == PORT_CMD_START)); 326 } 327 328 /* 329 * Enables the port DMA engine and FIS reception. 330 * 331 * return value 332 * None 333 */ 334 static inline void mtip_start_port(struct mtip_port *port) 335 { 336 /* Enable FIS reception */ 337 mtip_enable_fis(port, 1); 338 339 /* Enable the DMA engine */ 340 mtip_enable_engine(port, 1); 341 } 342 343 /* 344 * Deinitialize a port by disabling port interrupts, the DMA engine, 345 * and FIS reception. 346 * 347 * @port Pointer to the port structure 348 * 349 * return value 350 * None 351 */ 352 static inline void mtip_deinit_port(struct mtip_port *port) 353 { 354 /* Disable interrupts on this port */ 355 writel(0, port->mmio + PORT_IRQ_MASK); 356 357 /* Disable the DMA engine */ 358 mtip_enable_engine(port, 0); 359 360 /* Disable FIS reception */ 361 mtip_enable_fis(port, 0); 362 } 363 364 /* 365 * Initialize a port. 366 * 367 * This function deinitializes the port by calling mtip_deinit_port() and 368 * then initializes it by setting the command header and RX FIS addresses, 369 * clearing the SError register and any pending port interrupts before 370 * re-enabling the default set of port interrupts. 371 * 372 * @port Pointer to the port structure. 373 * 374 * return value 375 * None 376 */ 377 static void mtip_init_port(struct mtip_port *port) 378 { 379 int i; 380 mtip_deinit_port(port); 381 382 /* Program the command list base and FIS base addresses */ 383 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { 384 writel((port->command_list_dma >> 16) >> 16, 385 port->mmio + PORT_LST_ADDR_HI); 386 writel((port->rxfis_dma >> 16) >> 16, 387 port->mmio + PORT_FIS_ADDR_HI); 388 set_bit(MTIP_PF_HOST_CAP_64, &port->flags); 389 } 390 391 writel(port->command_list_dma & 0xFFFFFFFF, 392 port->mmio + PORT_LST_ADDR); 393 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); 394 395 /* Clear SError */ 396 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); 397 398 /* reset the completed registers.*/ 399 for (i = 0; i < port->dd->slot_groups; i++) 400 writel(0xFFFFFFFF, port->completed[i]); 401 402 /* Clear any pending interrupts for this port */ 403 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); 404 405 /* Clear any pending interrupts on the HBA. */ 406 writel(readl(port->dd->mmio + HOST_IRQ_STAT), 407 port->dd->mmio + HOST_IRQ_STAT); 408 409 /* Enable port interrupts */ 410 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); 411 } 412 413 /* 414 * Restart a port 415 * 416 * @port Pointer to the port data structure. 417 * 418 * return value 419 * None 420 */ 421 static void mtip_restart_port(struct mtip_port *port) 422 { 423 unsigned long timeout; 424 425 /* Disable the DMA engine */ 426 mtip_enable_engine(port, 0); 427 428 /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */ 429 timeout = jiffies + msecs_to_jiffies(500); 430 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) 431 && time_before(jiffies, timeout)) 432 ; 433 434 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 435 return; 436 437 /* 438 * Chip quirk: escalate to hba reset if 439 * PxCMD.CR not clear after 500 ms 440 */ 441 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { 442 dev_warn(&port->dd->pdev->dev, 443 "PxCMD.CR not clear, escalating reset\n"); 444 445 if (mtip_hba_reset(port->dd)) 446 dev_err(&port->dd->pdev->dev, 447 "HBA reset escalation failed.\n"); 448 449 /* 30 ms delay before com reset to quiesce chip */ 450 mdelay(30); 451 } 452 453 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); 454 455 /* Set PxSCTL.DET */ 456 writel(readl(port->mmio + PORT_SCR_CTL) | 457 1, port->mmio + PORT_SCR_CTL); 458 readl(port->mmio + PORT_SCR_CTL); 459 460 /* Wait 1 ms to quiesce chip function */ 461 timeout = jiffies + msecs_to_jiffies(1); 462 while (time_before(jiffies, timeout)) 463 ; 464 465 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 466 return; 467 468 /* Clear PxSCTL.DET */ 469 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, 470 port->mmio + PORT_SCR_CTL); 471 readl(port->mmio + PORT_SCR_CTL); 472 473 /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */ 474 timeout = jiffies + msecs_to_jiffies(500); 475 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 476 && time_before(jiffies, timeout)) 477 ; 478 479 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 480 return; 481 482 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 483 dev_warn(&port->dd->pdev->dev, 484 "COM reset failed\n"); 485 486 mtip_init_port(port); 487 mtip_start_port(port); 488 489 } 490 491 static int mtip_device_reset(struct driver_data *dd) 492 { 493 int rv = 0; 494 495 if (mtip_check_surprise_removal(dd->pdev)) 496 return 0; 497 498 if (mtip_hba_reset(dd) < 0) 499 rv = -EFAULT; 500 501 mdelay(1); 502 mtip_init_port(dd->port); 503 mtip_start_port(dd->port); 504 505 /* Enable interrupts on the HBA. */ 506 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 507 dd->mmio + HOST_CTL); 508 return rv; 509 } 510 511 /* 512 * Helper function for tag logging 513 */ 514 static void print_tags(struct driver_data *dd, 515 char *msg, 516 unsigned long *tagbits, 517 int cnt) 518 { 519 unsigned char tagmap[128]; 520 int group, tagmap_len = 0; 521 522 memset(tagmap, 0, sizeof(tagmap)); 523 for (group = SLOTBITS_IN_LONGS; group > 0; group--) 524 tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ", 525 tagbits[group-1]); 526 dev_warn(&dd->pdev->dev, 527 "%d command(s) %s: tagmap [%s]", cnt, msg, tagmap); 528 } 529 530 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 531 dma_addr_t buffer_dma, unsigned int sectors); 532 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 533 struct smart_attr *attrib); 534 535 static void mtip_complete_command(struct mtip_cmd *cmd, blk_status_t status) 536 { 537 struct request *req = blk_mq_rq_from_pdu(cmd); 538 539 cmd->status = status; 540 blk_mq_complete_request(req); 541 } 542 543 /* 544 * Handle an error. 545 * 546 * @dd Pointer to the DRIVER_DATA structure. 547 * 548 * return value 549 * None 550 */ 551 static void mtip_handle_tfe(struct driver_data *dd) 552 { 553 int group, tag, bit, reissue, rv; 554 struct mtip_port *port; 555 struct mtip_cmd *cmd; 556 u32 completed; 557 struct host_to_dev_fis *fis; 558 unsigned long tagaccum[SLOTBITS_IN_LONGS]; 559 unsigned int cmd_cnt = 0; 560 unsigned char *buf; 561 char *fail_reason = NULL; 562 int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0; 563 564 dev_warn(&dd->pdev->dev, "Taskfile error\n"); 565 566 port = dd->port; 567 568 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { 569 cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); 570 dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n"); 571 mtip_complete_command(cmd, BLK_STS_IOERR); 572 return; 573 } 574 575 /* clear the tag accumulator */ 576 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 577 578 /* Loop through all the groups */ 579 for (group = 0; group < dd->slot_groups; group++) { 580 completed = readl(port->completed[group]); 581 582 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); 583 584 /* clear completed status register in the hardware.*/ 585 writel(completed, port->completed[group]); 586 587 /* Process successfully completed commands */ 588 for (bit = 0; bit < 32 && completed; bit++) { 589 if (!(completed & (1<<bit))) 590 continue; 591 tag = (group << 5) + bit; 592 593 /* Skip the internal command slot */ 594 if (tag == MTIP_TAG_INTERNAL) 595 continue; 596 597 cmd = mtip_cmd_from_tag(dd, tag); 598 mtip_complete_command(cmd, 0); 599 set_bit(tag, tagaccum); 600 cmd_cnt++; 601 } 602 } 603 604 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); 605 606 /* Restart the port */ 607 mdelay(20); 608 mtip_restart_port(port); 609 610 /* Trying to determine the cause of the error */ 611 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 612 dd->port->log_buf, 613 dd->port->log_buf_dma, 1); 614 if (rv) { 615 dev_warn(&dd->pdev->dev, 616 "Error in READ LOG EXT (10h) command\n"); 617 /* non-critical error, don't fail the load */ 618 } else { 619 buf = (unsigned char *)dd->port->log_buf; 620 if (buf[259] & 0x1) { 621 dev_info(&dd->pdev->dev, 622 "Write protect bit is set.\n"); 623 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 624 fail_all_ncq_write = 1; 625 fail_reason = "write protect"; 626 } 627 if (buf[288] == 0xF7) { 628 dev_info(&dd->pdev->dev, 629 "Exceeded Tmax, drive in thermal shutdown.\n"); 630 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 631 fail_all_ncq_cmds = 1; 632 fail_reason = "thermal shutdown"; 633 } 634 if (buf[288] == 0xBF) { 635 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); 636 dev_info(&dd->pdev->dev, 637 "Drive indicates rebuild has failed. Secure erase required.\n"); 638 fail_all_ncq_cmds = 1; 639 fail_reason = "rebuild failed"; 640 } 641 } 642 643 /* clear the tag accumulator */ 644 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 645 646 /* Loop through all the groups */ 647 for (group = 0; group < dd->slot_groups; group++) { 648 for (bit = 0; bit < 32; bit++) { 649 reissue = 1; 650 tag = (group << 5) + bit; 651 cmd = mtip_cmd_from_tag(dd, tag); 652 653 fis = (struct host_to_dev_fis *)cmd->command; 654 655 /* Should re-issue? */ 656 if (tag == MTIP_TAG_INTERNAL || 657 fis->command == ATA_CMD_SET_FEATURES) 658 reissue = 0; 659 else { 660 if (fail_all_ncq_cmds || 661 (fail_all_ncq_write && 662 fis->command == ATA_CMD_FPDMA_WRITE)) { 663 dev_warn(&dd->pdev->dev, 664 " Fail: %s w/tag %d [%s].\n", 665 fis->command == ATA_CMD_FPDMA_WRITE ? 666 "write" : "read", 667 tag, 668 fail_reason != NULL ? 669 fail_reason : "unknown"); 670 mtip_complete_command(cmd, BLK_STS_MEDIUM); 671 continue; 672 } 673 } 674 675 /* 676 * First check if this command has 677 * exceeded its retries. 678 */ 679 if (reissue && (cmd->retries-- > 0)) { 680 681 set_bit(tag, tagaccum); 682 683 /* Re-issue the command. */ 684 mtip_issue_ncq_command(port, tag); 685 686 continue; 687 } 688 689 /* Retire a command that will not be reissued */ 690 dev_warn(&port->dd->pdev->dev, 691 "retiring tag %d\n", tag); 692 693 mtip_complete_command(cmd, BLK_STS_IOERR); 694 } 695 } 696 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); 697 } 698 699 /* 700 * Handle a set device bits interrupt 701 */ 702 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group, 703 u32 completed) 704 { 705 struct driver_data *dd = port->dd; 706 int tag, bit; 707 struct mtip_cmd *command; 708 709 if (!completed) { 710 WARN_ON_ONCE(!completed); 711 return; 712 } 713 /* clear completed status register in the hardware.*/ 714 writel(completed, port->completed[group]); 715 716 /* Process completed commands. */ 717 for (bit = 0; (bit < 32) && completed; bit++) { 718 if (completed & 0x01) { 719 tag = (group << 5) | bit; 720 721 /* skip internal command slot. */ 722 if (unlikely(tag == MTIP_TAG_INTERNAL)) 723 continue; 724 725 command = mtip_cmd_from_tag(dd, tag); 726 mtip_complete_command(command, 0); 727 } 728 completed >>= 1; 729 } 730 731 /* If last, re-enable interrupts */ 732 if (atomic_dec_return(&dd->irq_workers_active) == 0) 733 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); 734 } 735 736 /* 737 * Process legacy pio and d2h interrupts 738 */ 739 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) 740 { 741 struct mtip_port *port = dd->port; 742 struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); 743 744 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) { 745 int group = MTIP_TAG_INDEX(MTIP_TAG_INTERNAL); 746 int status = readl(port->cmd_issue[group]); 747 748 if (!(status & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL)))) 749 mtip_complete_command(cmd, 0); 750 } 751 } 752 753 /* 754 * Demux and handle errors 755 */ 756 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) 757 { 758 if (unlikely(port_stat & PORT_IRQ_CONNECT)) { 759 dev_warn(&dd->pdev->dev, 760 "Clearing PxSERR.DIAG.x\n"); 761 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); 762 } 763 764 if (unlikely(port_stat & PORT_IRQ_PHYRDY)) { 765 dev_warn(&dd->pdev->dev, 766 "Clearing PxSERR.DIAG.n\n"); 767 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); 768 } 769 770 if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) { 771 dev_warn(&dd->pdev->dev, 772 "Port stat errors %x unhandled\n", 773 (port_stat & ~PORT_IRQ_HANDLED)); 774 if (mtip_check_surprise_removal(dd->pdev)) 775 return; 776 } 777 if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) { 778 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); 779 wake_up_interruptible(&dd->port->svc_wait); 780 } 781 } 782 783 static inline irqreturn_t mtip_handle_irq(struct driver_data *data) 784 { 785 struct driver_data *dd = (struct driver_data *) data; 786 struct mtip_port *port = dd->port; 787 u32 hba_stat, port_stat; 788 int rv = IRQ_NONE; 789 int do_irq_enable = 1, i, workers; 790 struct mtip_work *twork; 791 792 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); 793 if (hba_stat) { 794 rv = IRQ_HANDLED; 795 796 /* Acknowledge the interrupt status on the port.*/ 797 port_stat = readl(port->mmio + PORT_IRQ_STAT); 798 if (unlikely(port_stat == 0xFFFFFFFF)) { 799 mtip_check_surprise_removal(dd->pdev); 800 return IRQ_HANDLED; 801 } 802 writel(port_stat, port->mmio + PORT_IRQ_STAT); 803 804 /* Demux port status */ 805 if (likely(port_stat & PORT_IRQ_SDB_FIS)) { 806 do_irq_enable = 0; 807 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); 808 809 /* Start at 1: group zero is always local? */ 810 for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS; 811 i++) { 812 twork = &dd->work[i]; 813 twork->completed = readl(port->completed[i]); 814 if (twork->completed) 815 workers++; 816 } 817 818 atomic_set(&dd->irq_workers_active, workers); 819 if (workers) { 820 for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) { 821 twork = &dd->work[i]; 822 if (twork->completed) 823 queue_work_on( 824 twork->cpu_binding, 825 dd->isr_workq, 826 &twork->work); 827 } 828 829 if (likely(dd->work[0].completed)) 830 mtip_workq_sdbfx(port, 0, 831 dd->work[0].completed); 832 833 } else { 834 /* 835 * Chip quirk: SDB interrupt but nothing 836 * to complete 837 */ 838 do_irq_enable = 1; 839 } 840 } 841 842 if (unlikely(port_stat & PORT_IRQ_ERR)) { 843 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 844 /* don't proceed further */ 845 return IRQ_HANDLED; 846 } 847 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 848 &dd->dd_flag)) 849 return rv; 850 851 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); 852 } 853 854 if (unlikely(port_stat & PORT_IRQ_LEGACY)) 855 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); 856 } 857 858 /* acknowledge interrupt */ 859 if (unlikely(do_irq_enable)) 860 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); 861 862 return rv; 863 } 864 865 /* 866 * HBA interrupt subroutine. 867 * 868 * @irq IRQ number. 869 * @instance Pointer to the driver data structure. 870 * 871 * return value 872 * IRQ_HANDLED A HBA interrupt was pending and handled. 873 * IRQ_NONE This interrupt was not for the HBA. 874 */ 875 static irqreturn_t mtip_irq_handler(int irq, void *instance) 876 { 877 struct driver_data *dd = instance; 878 879 return mtip_handle_irq(dd); 880 } 881 882 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) 883 { 884 writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]); 885 } 886 887 static bool mtip_pause_ncq(struct mtip_port *port, 888 struct host_to_dev_fis *fis) 889 { 890 unsigned long task_file_data; 891 892 task_file_data = readl(port->mmio+PORT_TFDATA); 893 if ((task_file_data & 1)) 894 return false; 895 896 if (fis->command == ATA_CMD_SEC_ERASE_PREP) { 897 port->ic_pause_timer = jiffies; 898 return true; 899 } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) && 900 (fis->features == 0x03)) { 901 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 902 port->ic_pause_timer = jiffies; 903 return true; 904 } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) || 905 ((fis->command == 0xFC) && 906 (fis->features == 0x27 || fis->features == 0x72 || 907 fis->features == 0x62 || fis->features == 0x26))) { 908 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 909 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); 910 /* Com reset after secure erase or lowlevel format */ 911 mtip_restart_port(port); 912 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 913 return false; 914 } 915 916 return false; 917 } 918 919 static bool mtip_commands_active(struct mtip_port *port) 920 { 921 unsigned int active; 922 unsigned int n; 923 924 /* 925 * Ignore s_active bit 0 of array element 0. 926 * This bit will always be set 927 */ 928 active = readl(port->s_active[0]) & 0xFFFFFFFE; 929 for (n = 1; n < port->dd->slot_groups; n++) 930 active |= readl(port->s_active[n]); 931 932 return active != 0; 933 } 934 935 /* 936 * Wait for port to quiesce 937 * 938 * @port Pointer to port data structure 939 * @timeout Max duration to wait (ms) 940 * 941 * return value 942 * 0 Success 943 * -EBUSY Commands still active 944 */ 945 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) 946 { 947 unsigned long to; 948 bool active = true; 949 950 blk_mq_quiesce_queue(port->dd->queue); 951 952 to = jiffies + msecs_to_jiffies(timeout); 953 do { 954 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && 955 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 956 msleep(20); 957 continue; /* svc thd is actively issuing commands */ 958 } 959 960 msleep(100); 961 962 if (mtip_check_surprise_removal(port->dd->pdev)) 963 goto err_fault; 964 965 active = mtip_commands_active(port); 966 if (!active) 967 break; 968 } while (time_before(jiffies, to)); 969 970 blk_mq_unquiesce_queue(port->dd->queue); 971 return active ? -EBUSY : 0; 972 err_fault: 973 blk_mq_unquiesce_queue(port->dd->queue); 974 return -EFAULT; 975 } 976 977 struct mtip_int_cmd { 978 int fis_len; 979 dma_addr_t buffer; 980 int buf_len; 981 u32 opts; 982 }; 983 984 /* 985 * Execute an internal command and wait for the completion. 986 * 987 * @port Pointer to the port data structure. 988 * @fis Pointer to the FIS that describes the command. 989 * @fis_len Length in WORDS of the FIS. 990 * @buffer DMA accessible for command data. 991 * @buf_len Length, in bytes, of the data buffer. 992 * @opts Command header options, excluding the FIS length 993 * and the number of PRD entries. 994 * @timeout Time in ms to wait for the command to complete. 995 * 996 * return value 997 * 0 Command completed successfully. 998 * -EFAULT The buffer address is not correctly aligned. 999 * -EBUSY Internal command or other IO in progress. 1000 * -EAGAIN Time out waiting for command to complete. 1001 */ 1002 static int mtip_exec_internal_command(struct mtip_port *port, 1003 struct host_to_dev_fis *fis, 1004 int fis_len, 1005 dma_addr_t buffer, 1006 int buf_len, 1007 u32 opts, 1008 unsigned long timeout) 1009 { 1010 struct mtip_cmd *int_cmd; 1011 struct driver_data *dd = port->dd; 1012 struct request *rq; 1013 struct mtip_int_cmd icmd = { 1014 .fis_len = fis_len, 1015 .buffer = buffer, 1016 .buf_len = buf_len, 1017 .opts = opts 1018 }; 1019 int rv = 0; 1020 1021 /* Make sure the buffer is 8 byte aligned. This is asic specific. */ 1022 if (buffer & 0x00000007) { 1023 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); 1024 return -EFAULT; 1025 } 1026 1027 int_cmd = mtip_get_int_command(dd); 1028 if (!int_cmd) { 1029 dbg_printk(MTIP_DRV_NAME "Unable to allocate tag for PIO cmd\n"); 1030 return -EFAULT; 1031 } 1032 rq = blk_mq_rq_from_pdu(int_cmd); 1033 rq->special = &icmd; 1034 1035 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1036 1037 if (fis->command == ATA_CMD_SEC_ERASE_PREP) 1038 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 1039 1040 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 1041 1042 if (fis->command != ATA_CMD_STANDBYNOW1) { 1043 /* wait for io to complete if non atomic */ 1044 if (mtip_quiesce_io(port, MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) { 1045 dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n"); 1046 blk_mq_free_request(rq); 1047 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1048 wake_up_interruptible(&port->svc_wait); 1049 return -EBUSY; 1050 } 1051 } 1052 1053 /* Copy the command to the command table */ 1054 memcpy(int_cmd->command, fis, fis_len*4); 1055 1056 rq->timeout = timeout; 1057 1058 /* insert request and run queue */ 1059 blk_execute_rq(rq->q, NULL, rq, true); 1060 1061 if (int_cmd->status) { 1062 dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", 1063 fis->command, int_cmd->status); 1064 rv = -EIO; 1065 1066 if (mtip_check_surprise_removal(dd->pdev) || 1067 test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1068 &dd->dd_flag)) { 1069 dev_err(&dd->pdev->dev, 1070 "Internal command [%02X] wait returned due to SR\n", 1071 fis->command); 1072 rv = -ENXIO; 1073 goto exec_ic_exit; 1074 } 1075 mtip_device_reset(dd); /* recover from timeout issue */ 1076 rv = -EAGAIN; 1077 goto exec_ic_exit; 1078 } 1079 1080 if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)]) 1081 & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))) { 1082 rv = -ENXIO; 1083 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 1084 mtip_device_reset(dd); 1085 rv = -EAGAIN; 1086 } 1087 } 1088 exec_ic_exit: 1089 /* Clear the allocated and active bits for the internal command. */ 1090 blk_mq_free_request(rq); 1091 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1092 if (rv >= 0 && mtip_pause_ncq(port, fis)) { 1093 /* NCQ paused */ 1094 return rv; 1095 } 1096 wake_up_interruptible(&port->svc_wait); 1097 1098 return rv; 1099 } 1100 1101 /* 1102 * Byte-swap ATA ID strings. 1103 * 1104 * ATA identify data contains strings in byte-swapped 16-bit words. 1105 * They must be swapped (on all architectures) to be usable as C strings. 1106 * This function swaps bytes in-place. 1107 * 1108 * @buf The buffer location of the string 1109 * @len The number of bytes to swap 1110 * 1111 * return value 1112 * None 1113 */ 1114 static inline void ata_swap_string(u16 *buf, unsigned int len) 1115 { 1116 int i; 1117 for (i = 0; i < (len/2); i++) 1118 be16_to_cpus(&buf[i]); 1119 } 1120 1121 static void mtip_set_timeout(struct driver_data *dd, 1122 struct host_to_dev_fis *fis, 1123 unsigned int *timeout, u8 erasemode) 1124 { 1125 switch (fis->command) { 1126 case ATA_CMD_DOWNLOAD_MICRO: 1127 *timeout = 120000; /* 2 minutes */ 1128 break; 1129 case ATA_CMD_SEC_ERASE_UNIT: 1130 case 0xFC: 1131 if (erasemode) 1132 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); 1133 else 1134 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); 1135 break; 1136 case ATA_CMD_STANDBYNOW1: 1137 *timeout = 120000; /* 2 minutes */ 1138 break; 1139 case 0xF7: 1140 case 0xFA: 1141 *timeout = 60000; /* 60 seconds */ 1142 break; 1143 case ATA_CMD_SMART: 1144 *timeout = 15000; /* 15 seconds */ 1145 break; 1146 default: 1147 *timeout = MTIP_IOCTL_CMD_TIMEOUT_MS; 1148 break; 1149 } 1150 } 1151 1152 /* 1153 * Request the device identity information. 1154 * 1155 * If a user space buffer is not specified, i.e. is NULL, the 1156 * identify information is still read from the drive and placed 1157 * into the identify data buffer (@e port->identify) in the 1158 * port data structure. 1159 * When the identify buffer contains valid identify information @e 1160 * port->identify_valid is non-zero. 1161 * 1162 * @port Pointer to the port structure. 1163 * @user_buffer A user space buffer where the identify data should be 1164 * copied. 1165 * 1166 * return value 1167 * 0 Command completed successfully. 1168 * -EFAULT An error occurred while coping data to the user buffer. 1169 * -1 Command failed. 1170 */ 1171 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer) 1172 { 1173 int rv = 0; 1174 struct host_to_dev_fis fis; 1175 1176 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 1177 return -EFAULT; 1178 1179 /* Build the FIS. */ 1180 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1181 fis.type = 0x27; 1182 fis.opts = 1 << 7; 1183 fis.command = ATA_CMD_ID_ATA; 1184 1185 /* Set the identify information as invalid. */ 1186 port->identify_valid = 0; 1187 1188 /* Clear the identify information. */ 1189 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); 1190 1191 /* Execute the command. */ 1192 if (mtip_exec_internal_command(port, 1193 &fis, 1194 5, 1195 port->identify_dma, 1196 sizeof(u16) * ATA_ID_WORDS, 1197 0, 1198 MTIP_INT_CMD_TIMEOUT_MS) 1199 < 0) { 1200 rv = -1; 1201 goto out; 1202 } 1203 1204 /* 1205 * Perform any necessary byte-swapping. Yes, the kernel does in fact 1206 * perform field-sensitive swapping on the string fields. 1207 * See the kernel use of ata_id_string() for proof of this. 1208 */ 1209 #ifdef __LITTLE_ENDIAN 1210 ata_swap_string(port->identify + 27, 40); /* model string*/ 1211 ata_swap_string(port->identify + 23, 8); /* firmware string*/ 1212 ata_swap_string(port->identify + 10, 20); /* serial# string*/ 1213 #else 1214 { 1215 int i; 1216 for (i = 0; i < ATA_ID_WORDS; i++) 1217 port->identify[i] = le16_to_cpu(port->identify[i]); 1218 } 1219 #endif 1220 1221 /* Check security locked state */ 1222 if (port->identify[128] & 0x4) 1223 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1224 else 1225 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1226 1227 #ifdef MTIP_TRIM /* Disabling TRIM support temporarily */ 1228 /* Demux ID.DRAT & ID.RZAT to determine trim support */ 1229 if (port->identify[69] & (1 << 14) && port->identify[69] & (1 << 5)) 1230 port->dd->trim_supp = true; 1231 else 1232 #endif 1233 port->dd->trim_supp = false; 1234 1235 /* Set the identify buffer as valid. */ 1236 port->identify_valid = 1; 1237 1238 if (user_buffer) { 1239 if (copy_to_user( 1240 user_buffer, 1241 port->identify, 1242 ATA_ID_WORDS * sizeof(u16))) { 1243 rv = -EFAULT; 1244 goto out; 1245 } 1246 } 1247 1248 out: 1249 return rv; 1250 } 1251 1252 /* 1253 * Issue a standby immediate command to the device. 1254 * 1255 * @port Pointer to the port structure. 1256 * 1257 * return value 1258 * 0 Command was executed successfully. 1259 * -1 An error occurred while executing the command. 1260 */ 1261 static int mtip_standby_immediate(struct mtip_port *port) 1262 { 1263 int rv; 1264 struct host_to_dev_fis fis; 1265 unsigned long start; 1266 unsigned int timeout; 1267 1268 /* Build the FIS. */ 1269 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1270 fis.type = 0x27; 1271 fis.opts = 1 << 7; 1272 fis.command = ATA_CMD_STANDBYNOW1; 1273 1274 mtip_set_timeout(port->dd, &fis, &timeout, 0); 1275 1276 start = jiffies; 1277 rv = mtip_exec_internal_command(port, 1278 &fis, 1279 5, 1280 0, 1281 0, 1282 0, 1283 timeout); 1284 dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n", 1285 jiffies_to_msecs(jiffies - start)); 1286 if (rv) 1287 dev_warn(&port->dd->pdev->dev, 1288 "STANDBY IMMEDIATE command failed.\n"); 1289 1290 return rv; 1291 } 1292 1293 /* 1294 * Issue a READ LOG EXT command to the device. 1295 * 1296 * @port pointer to the port structure. 1297 * @page page number to fetch 1298 * @buffer pointer to buffer 1299 * @buffer_dma dma address corresponding to @buffer 1300 * @sectors page length to fetch, in sectors 1301 * 1302 * return value 1303 * @rv return value from mtip_exec_internal_command() 1304 */ 1305 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 1306 dma_addr_t buffer_dma, unsigned int sectors) 1307 { 1308 struct host_to_dev_fis fis; 1309 1310 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1311 fis.type = 0x27; 1312 fis.opts = 1 << 7; 1313 fis.command = ATA_CMD_READ_LOG_EXT; 1314 fis.sect_count = sectors & 0xFF; 1315 fis.sect_cnt_ex = (sectors >> 8) & 0xFF; 1316 fis.lba_low = page; 1317 fis.lba_mid = 0; 1318 fis.device = ATA_DEVICE_OBS; 1319 1320 memset(buffer, 0, sectors * ATA_SECT_SIZE); 1321 1322 return mtip_exec_internal_command(port, 1323 &fis, 1324 5, 1325 buffer_dma, 1326 sectors * ATA_SECT_SIZE, 1327 0, 1328 MTIP_INT_CMD_TIMEOUT_MS); 1329 } 1330 1331 /* 1332 * Issue a SMART READ DATA command to the device. 1333 * 1334 * @port pointer to the port structure. 1335 * @buffer pointer to buffer 1336 * @buffer_dma dma address corresponding to @buffer 1337 * 1338 * return value 1339 * @rv return value from mtip_exec_internal_command() 1340 */ 1341 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, 1342 dma_addr_t buffer_dma) 1343 { 1344 struct host_to_dev_fis fis; 1345 1346 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1347 fis.type = 0x27; 1348 fis.opts = 1 << 7; 1349 fis.command = ATA_CMD_SMART; 1350 fis.features = 0xD0; 1351 fis.sect_count = 1; 1352 fis.lba_mid = 0x4F; 1353 fis.lba_hi = 0xC2; 1354 fis.device = ATA_DEVICE_OBS; 1355 1356 return mtip_exec_internal_command(port, 1357 &fis, 1358 5, 1359 buffer_dma, 1360 ATA_SECT_SIZE, 1361 0, 1362 15000); 1363 } 1364 1365 /* 1366 * Get the value of a smart attribute 1367 * 1368 * @port pointer to the port structure 1369 * @id attribute number 1370 * @attrib pointer to return attrib information corresponding to @id 1371 * 1372 * return value 1373 * -EINVAL NULL buffer passed or unsupported attribute @id. 1374 * -EPERM Identify data not valid, SMART not supported or not enabled 1375 */ 1376 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 1377 struct smart_attr *attrib) 1378 { 1379 int rv, i; 1380 struct smart_attr *pattr; 1381 1382 if (!attrib) 1383 return -EINVAL; 1384 1385 if (!port->identify_valid) { 1386 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); 1387 return -EPERM; 1388 } 1389 if (!(port->identify[82] & 0x1)) { 1390 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); 1391 return -EPERM; 1392 } 1393 if (!(port->identify[85] & 0x1)) { 1394 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); 1395 return -EPERM; 1396 } 1397 1398 memset(port->smart_buf, 0, ATA_SECT_SIZE); 1399 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); 1400 if (rv) { 1401 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); 1402 return rv; 1403 } 1404 1405 pattr = (struct smart_attr *)(port->smart_buf + 2); 1406 for (i = 0; i < 29; i++, pattr++) 1407 if (pattr->attr_id == id) { 1408 memcpy(attrib, pattr, sizeof(struct smart_attr)); 1409 break; 1410 } 1411 1412 if (i == 29) { 1413 dev_warn(&port->dd->pdev->dev, 1414 "Query for invalid SMART attribute ID\n"); 1415 rv = -EINVAL; 1416 } 1417 1418 return rv; 1419 } 1420 1421 /* 1422 * Trim unused sectors 1423 * 1424 * @dd pointer to driver_data structure 1425 * @lba starting lba 1426 * @len # of 512b sectors to trim 1427 * 1428 * return value 1429 * -ENOMEM Out of dma memory 1430 * -EINVAL Invalid parameters passed in, trim not supported 1431 * -EIO Error submitting trim request to hw 1432 */ 1433 static int mtip_send_trim(struct driver_data *dd, unsigned int lba, 1434 unsigned int len) 1435 { 1436 int i, rv = 0; 1437 u64 tlba, tlen, sect_left; 1438 struct mtip_trim_entry *buf; 1439 dma_addr_t dma_addr; 1440 struct host_to_dev_fis fis; 1441 1442 if (!len || dd->trim_supp == false) 1443 return -EINVAL; 1444 1445 /* Trim request too big */ 1446 WARN_ON(len > (MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES)); 1447 1448 /* Trim request not aligned on 4k boundary */ 1449 WARN_ON(len % 8 != 0); 1450 1451 /* Warn if vu_trim structure is too big */ 1452 WARN_ON(sizeof(struct mtip_trim) > ATA_SECT_SIZE); 1453 1454 /* Allocate a DMA buffer for the trim structure */ 1455 buf = dmam_alloc_coherent(&dd->pdev->dev, ATA_SECT_SIZE, &dma_addr, 1456 GFP_KERNEL); 1457 if (!buf) 1458 return -ENOMEM; 1459 memset(buf, 0, ATA_SECT_SIZE); 1460 1461 for (i = 0, sect_left = len, tlba = lba; 1462 i < MTIP_MAX_TRIM_ENTRIES && sect_left; 1463 i++) { 1464 tlen = (sect_left >= MTIP_MAX_TRIM_ENTRY_LEN ? 1465 MTIP_MAX_TRIM_ENTRY_LEN : 1466 sect_left); 1467 buf[i].lba = __force_bit2int cpu_to_le32(tlba); 1468 buf[i].range = __force_bit2int cpu_to_le16(tlen); 1469 tlba += tlen; 1470 sect_left -= tlen; 1471 } 1472 WARN_ON(sect_left != 0); 1473 1474 /* Build the fis */ 1475 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1476 fis.type = 0x27; 1477 fis.opts = 1 << 7; 1478 fis.command = 0xfb; 1479 fis.features = 0x60; 1480 fis.sect_count = 1; 1481 fis.device = ATA_DEVICE_OBS; 1482 1483 if (mtip_exec_internal_command(dd->port, 1484 &fis, 1485 5, 1486 dma_addr, 1487 ATA_SECT_SIZE, 1488 0, 1489 MTIP_TRIM_TIMEOUT_MS) < 0) 1490 rv = -EIO; 1491 1492 dmam_free_coherent(&dd->pdev->dev, ATA_SECT_SIZE, buf, dma_addr); 1493 return rv; 1494 } 1495 1496 /* 1497 * Get the drive capacity. 1498 * 1499 * @dd Pointer to the device data structure. 1500 * @sectors Pointer to the variable that will receive the sector count. 1501 * 1502 * return value 1503 * 1 Capacity was returned successfully. 1504 * 0 The identify information is invalid. 1505 */ 1506 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) 1507 { 1508 struct mtip_port *port = dd->port; 1509 u64 total, raw0, raw1, raw2, raw3; 1510 raw0 = port->identify[100]; 1511 raw1 = port->identify[101]; 1512 raw2 = port->identify[102]; 1513 raw3 = port->identify[103]; 1514 total = raw0 | raw1<<16 | raw2<<32 | raw3<<48; 1515 *sectors = total; 1516 return (bool) !!port->identify_valid; 1517 } 1518 1519 /* 1520 * Display the identify command data. 1521 * 1522 * @port Pointer to the port data structure. 1523 * 1524 * return value 1525 * None 1526 */ 1527 static void mtip_dump_identify(struct mtip_port *port) 1528 { 1529 sector_t sectors; 1530 unsigned short revid; 1531 char cbuf[42]; 1532 1533 if (!port->identify_valid) 1534 return; 1535 1536 strlcpy(cbuf, (char *)(port->identify+10), 21); 1537 dev_info(&port->dd->pdev->dev, 1538 "Serial No.: %s\n", cbuf); 1539 1540 strlcpy(cbuf, (char *)(port->identify+23), 9); 1541 dev_info(&port->dd->pdev->dev, 1542 "Firmware Ver.: %s\n", cbuf); 1543 1544 strlcpy(cbuf, (char *)(port->identify+27), 41); 1545 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); 1546 1547 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", 1548 port->identify[128], 1549 port->identify[128] & 0x4 ? "(LOCKED)" : ""); 1550 1551 if (mtip_hw_get_capacity(port->dd, §ors)) 1552 dev_info(&port->dd->pdev->dev, 1553 "Capacity: %llu sectors (%llu MB)\n", 1554 (u64)sectors, 1555 ((u64)sectors) * ATA_SECT_SIZE >> 20); 1556 1557 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); 1558 switch (revid & 0xFF) { 1559 case 0x1: 1560 strlcpy(cbuf, "A0", 3); 1561 break; 1562 case 0x3: 1563 strlcpy(cbuf, "A2", 3); 1564 break; 1565 default: 1566 strlcpy(cbuf, "?", 2); 1567 break; 1568 } 1569 dev_info(&port->dd->pdev->dev, 1570 "Card Type: %s\n", cbuf); 1571 } 1572 1573 /* 1574 * Map the commands scatter list into the command table. 1575 * 1576 * @command Pointer to the command. 1577 * @nents Number of scatter list entries. 1578 * 1579 * return value 1580 * None 1581 */ 1582 static inline void fill_command_sg(struct driver_data *dd, 1583 struct mtip_cmd *command, 1584 int nents) 1585 { 1586 int n; 1587 unsigned int dma_len; 1588 struct mtip_cmd_sg *command_sg; 1589 struct scatterlist *sg = command->sg; 1590 1591 command_sg = command->command + AHCI_CMD_TBL_HDR_SZ; 1592 1593 for (n = 0; n < nents; n++) { 1594 dma_len = sg_dma_len(sg); 1595 if (dma_len > 0x400000) 1596 dev_err(&dd->pdev->dev, 1597 "DMA segment length truncated\n"); 1598 command_sg->info = __force_bit2int 1599 cpu_to_le32((dma_len-1) & 0x3FFFFF); 1600 command_sg->dba = __force_bit2int 1601 cpu_to_le32(sg_dma_address(sg)); 1602 command_sg->dba_upper = __force_bit2int 1603 cpu_to_le32((sg_dma_address(sg) >> 16) >> 16); 1604 command_sg++; 1605 sg++; 1606 } 1607 } 1608 1609 /* 1610 * @brief Execute a drive command. 1611 * 1612 * return value 0 The command completed successfully. 1613 * return value -1 An error occurred while executing the command. 1614 */ 1615 static int exec_drive_task(struct mtip_port *port, u8 *command) 1616 { 1617 struct host_to_dev_fis fis; 1618 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); 1619 unsigned int to; 1620 1621 /* Build the FIS. */ 1622 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1623 fis.type = 0x27; 1624 fis.opts = 1 << 7; 1625 fis.command = command[0]; 1626 fis.features = command[1]; 1627 fis.sect_count = command[2]; 1628 fis.sector = command[3]; 1629 fis.cyl_low = command[4]; 1630 fis.cyl_hi = command[5]; 1631 fis.device = command[6] & ~0x10; /* Clear the dev bit*/ 1632 1633 mtip_set_timeout(port->dd, &fis, &to, 0); 1634 1635 dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n", 1636 __func__, 1637 command[0], 1638 command[1], 1639 command[2], 1640 command[3], 1641 command[4], 1642 command[5], 1643 command[6]); 1644 1645 /* Execute the command. */ 1646 if (mtip_exec_internal_command(port, 1647 &fis, 1648 5, 1649 0, 1650 0, 1651 0, 1652 to) < 0) { 1653 return -1; 1654 } 1655 1656 command[0] = reply->command; /* Status*/ 1657 command[1] = reply->features; /* Error*/ 1658 command[4] = reply->cyl_low; 1659 command[5] = reply->cyl_hi; 1660 1661 dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n", 1662 __func__, 1663 command[0], 1664 command[1], 1665 command[4], 1666 command[5]); 1667 1668 return 0; 1669 } 1670 1671 /* 1672 * @brief Execute a drive command. 1673 * 1674 * @param port Pointer to the port data structure. 1675 * @param command Pointer to the user specified command parameters. 1676 * @param user_buffer Pointer to the user space buffer where read sector 1677 * data should be copied. 1678 * 1679 * return value 0 The command completed successfully. 1680 * return value -EFAULT An error occurred while copying the completion 1681 * data to the user space buffer. 1682 * return value -1 An error occurred while executing the command. 1683 */ 1684 static int exec_drive_command(struct mtip_port *port, u8 *command, 1685 void __user *user_buffer) 1686 { 1687 struct host_to_dev_fis fis; 1688 struct host_to_dev_fis *reply; 1689 u8 *buf = NULL; 1690 dma_addr_t dma_addr = 0; 1691 int rv = 0, xfer_sz = command[3]; 1692 unsigned int to; 1693 1694 if (xfer_sz) { 1695 if (!user_buffer) 1696 return -EFAULT; 1697 1698 buf = dmam_alloc_coherent(&port->dd->pdev->dev, 1699 ATA_SECT_SIZE * xfer_sz, 1700 &dma_addr, 1701 GFP_KERNEL); 1702 if (!buf) { 1703 dev_err(&port->dd->pdev->dev, 1704 "Memory allocation failed (%d bytes)\n", 1705 ATA_SECT_SIZE * xfer_sz); 1706 return -ENOMEM; 1707 } 1708 memset(buf, 0, ATA_SECT_SIZE * xfer_sz); 1709 } 1710 1711 /* Build the FIS. */ 1712 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1713 fis.type = 0x27; 1714 fis.opts = 1 << 7; 1715 fis.command = command[0]; 1716 fis.features = command[2]; 1717 fis.sect_count = command[3]; 1718 if (fis.command == ATA_CMD_SMART) { 1719 fis.sector = command[1]; 1720 fis.cyl_low = 0x4F; 1721 fis.cyl_hi = 0xC2; 1722 } 1723 1724 mtip_set_timeout(port->dd, &fis, &to, 0); 1725 1726 if (xfer_sz) 1727 reply = (port->rxfis + RX_FIS_PIO_SETUP); 1728 else 1729 reply = (port->rxfis + RX_FIS_D2H_REG); 1730 1731 dbg_printk(MTIP_DRV_NAME 1732 " %s: User Command: cmd %x, sect %x, " 1733 "feat %x, sectcnt %x\n", 1734 __func__, 1735 command[0], 1736 command[1], 1737 command[2], 1738 command[3]); 1739 1740 /* Execute the command. */ 1741 if (mtip_exec_internal_command(port, 1742 &fis, 1743 5, 1744 (xfer_sz ? dma_addr : 0), 1745 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0), 1746 0, 1747 to) 1748 < 0) { 1749 rv = -EFAULT; 1750 goto exit_drive_command; 1751 } 1752 1753 /* Collect the completion status. */ 1754 command[0] = reply->command; /* Status*/ 1755 command[1] = reply->features; /* Error*/ 1756 command[2] = reply->sect_count; 1757 1758 dbg_printk(MTIP_DRV_NAME 1759 " %s: Completion Status: stat %x, " 1760 "err %x, nsect %x\n", 1761 __func__, 1762 command[0], 1763 command[1], 1764 command[2]); 1765 1766 if (xfer_sz) { 1767 if (copy_to_user(user_buffer, 1768 buf, 1769 ATA_SECT_SIZE * command[3])) { 1770 rv = -EFAULT; 1771 goto exit_drive_command; 1772 } 1773 } 1774 exit_drive_command: 1775 if (buf) 1776 dmam_free_coherent(&port->dd->pdev->dev, 1777 ATA_SECT_SIZE * xfer_sz, buf, dma_addr); 1778 return rv; 1779 } 1780 1781 /* 1782 * Indicates whether a command has a single sector payload. 1783 * 1784 * @command passed to the device to perform the certain event. 1785 * @features passed to the device to perform the certain event. 1786 * 1787 * return value 1788 * 1 command is one that always has a single sector payload, 1789 * regardless of the value in the Sector Count field. 1790 * 0 otherwise 1791 * 1792 */ 1793 static unsigned int implicit_sector(unsigned char command, 1794 unsigned char features) 1795 { 1796 unsigned int rv = 0; 1797 1798 /* list of commands that have an implicit sector count of 1 */ 1799 switch (command) { 1800 case ATA_CMD_SEC_SET_PASS: 1801 case ATA_CMD_SEC_UNLOCK: 1802 case ATA_CMD_SEC_ERASE_PREP: 1803 case ATA_CMD_SEC_ERASE_UNIT: 1804 case ATA_CMD_SEC_FREEZE_LOCK: 1805 case ATA_CMD_SEC_DISABLE_PASS: 1806 case ATA_CMD_PMP_READ: 1807 case ATA_CMD_PMP_WRITE: 1808 rv = 1; 1809 break; 1810 case ATA_CMD_SET_MAX: 1811 if (features == ATA_SET_MAX_UNLOCK) 1812 rv = 1; 1813 break; 1814 case ATA_CMD_SMART: 1815 if ((features == ATA_SMART_READ_VALUES) || 1816 (features == ATA_SMART_READ_THRESHOLDS)) 1817 rv = 1; 1818 break; 1819 case ATA_CMD_CONF_OVERLAY: 1820 if ((features == ATA_DCO_IDENTIFY) || 1821 (features == ATA_DCO_SET)) 1822 rv = 1; 1823 break; 1824 } 1825 return rv; 1826 } 1827 1828 /* 1829 * Executes a taskfile 1830 * See ide_taskfile_ioctl() for derivation 1831 */ 1832 static int exec_drive_taskfile(struct driver_data *dd, 1833 void __user *buf, 1834 ide_task_request_t *req_task, 1835 int outtotal) 1836 { 1837 struct host_to_dev_fis fis; 1838 struct host_to_dev_fis *reply; 1839 u8 *outbuf = NULL; 1840 u8 *inbuf = NULL; 1841 dma_addr_t outbuf_dma = 0; 1842 dma_addr_t inbuf_dma = 0; 1843 dma_addr_t dma_buffer = 0; 1844 int err = 0; 1845 unsigned int taskin = 0; 1846 unsigned int taskout = 0; 1847 u8 nsect = 0; 1848 unsigned int timeout; 1849 unsigned int force_single_sector; 1850 unsigned int transfer_size; 1851 unsigned long task_file_data; 1852 int intotal = outtotal + req_task->out_size; 1853 int erasemode = 0; 1854 1855 taskout = req_task->out_size; 1856 taskin = req_task->in_size; 1857 /* 130560 = 512 * 0xFF*/ 1858 if (taskin > 130560 || taskout > 130560) 1859 return -EINVAL; 1860 1861 if (taskout) { 1862 outbuf = memdup_user(buf + outtotal, taskout); 1863 if (IS_ERR(outbuf)) 1864 return PTR_ERR(outbuf); 1865 1866 outbuf_dma = pci_map_single(dd->pdev, 1867 outbuf, 1868 taskout, 1869 DMA_TO_DEVICE); 1870 if (pci_dma_mapping_error(dd->pdev, outbuf_dma)) { 1871 err = -ENOMEM; 1872 goto abort; 1873 } 1874 dma_buffer = outbuf_dma; 1875 } 1876 1877 if (taskin) { 1878 inbuf = memdup_user(buf + intotal, taskin); 1879 if (IS_ERR(inbuf)) { 1880 err = PTR_ERR(inbuf); 1881 inbuf = NULL; 1882 goto abort; 1883 } 1884 inbuf_dma = pci_map_single(dd->pdev, 1885 inbuf, 1886 taskin, DMA_FROM_DEVICE); 1887 if (pci_dma_mapping_error(dd->pdev, inbuf_dma)) { 1888 err = -ENOMEM; 1889 goto abort; 1890 } 1891 dma_buffer = inbuf_dma; 1892 } 1893 1894 /* only supports PIO and non-data commands from this ioctl. */ 1895 switch (req_task->data_phase) { 1896 case TASKFILE_OUT: 1897 nsect = taskout / ATA_SECT_SIZE; 1898 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 1899 break; 1900 case TASKFILE_IN: 1901 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 1902 break; 1903 case TASKFILE_NO_DATA: 1904 reply = (dd->port->rxfis + RX_FIS_D2H_REG); 1905 break; 1906 default: 1907 err = -EINVAL; 1908 goto abort; 1909 } 1910 1911 /* Build the FIS. */ 1912 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1913 1914 fis.type = 0x27; 1915 fis.opts = 1 << 7; 1916 fis.command = req_task->io_ports[7]; 1917 fis.features = req_task->io_ports[1]; 1918 fis.sect_count = req_task->io_ports[2]; 1919 fis.lba_low = req_task->io_ports[3]; 1920 fis.lba_mid = req_task->io_ports[4]; 1921 fis.lba_hi = req_task->io_ports[5]; 1922 /* Clear the dev bit*/ 1923 fis.device = req_task->io_ports[6] & ~0x10; 1924 1925 if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) { 1926 req_task->in_flags.all = 1927 IDE_TASKFILE_STD_IN_FLAGS | 1928 (IDE_HOB_STD_IN_FLAGS << 8); 1929 fis.lba_low_ex = req_task->hob_ports[3]; 1930 fis.lba_mid_ex = req_task->hob_ports[4]; 1931 fis.lba_hi_ex = req_task->hob_ports[5]; 1932 fis.features_ex = req_task->hob_ports[1]; 1933 fis.sect_cnt_ex = req_task->hob_ports[2]; 1934 1935 } else { 1936 req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS; 1937 } 1938 1939 force_single_sector = implicit_sector(fis.command, fis.features); 1940 1941 if ((taskin || taskout) && (!fis.sect_count)) { 1942 if (nsect) 1943 fis.sect_count = nsect; 1944 else { 1945 if (!force_single_sector) { 1946 dev_warn(&dd->pdev->dev, 1947 "data movement but " 1948 "sect_count is 0\n"); 1949 err = -EINVAL; 1950 goto abort; 1951 } 1952 } 1953 } 1954 1955 dbg_printk(MTIP_DRV_NAME 1956 " %s: cmd %x, feat %x, nsect %x," 1957 " sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x," 1958 " head/dev %x\n", 1959 __func__, 1960 fis.command, 1961 fis.features, 1962 fis.sect_count, 1963 fis.lba_low, 1964 fis.lba_mid, 1965 fis.lba_hi, 1966 fis.device); 1967 1968 /* check for erase mode support during secure erase.*/ 1969 if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf && 1970 (outbuf[0] & MTIP_SEC_ERASE_MODE)) { 1971 erasemode = 1; 1972 } 1973 1974 mtip_set_timeout(dd, &fis, &timeout, erasemode); 1975 1976 /* Determine the correct transfer size.*/ 1977 if (force_single_sector) 1978 transfer_size = ATA_SECT_SIZE; 1979 else 1980 transfer_size = ATA_SECT_SIZE * fis.sect_count; 1981 1982 /* Execute the command.*/ 1983 if (mtip_exec_internal_command(dd->port, 1984 &fis, 1985 5, 1986 dma_buffer, 1987 transfer_size, 1988 0, 1989 timeout) < 0) { 1990 err = -EIO; 1991 goto abort; 1992 } 1993 1994 task_file_data = readl(dd->port->mmio+PORT_TFDATA); 1995 1996 if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) { 1997 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; 1998 req_task->io_ports[7] = reply->control; 1999 } else { 2000 reply = dd->port->rxfis + RX_FIS_D2H_REG; 2001 req_task->io_ports[7] = reply->command; 2002 } 2003 2004 /* reclaim the DMA buffers.*/ 2005 if (inbuf_dma) 2006 pci_unmap_single(dd->pdev, inbuf_dma, 2007 taskin, DMA_FROM_DEVICE); 2008 if (outbuf_dma) 2009 pci_unmap_single(dd->pdev, outbuf_dma, 2010 taskout, DMA_TO_DEVICE); 2011 inbuf_dma = 0; 2012 outbuf_dma = 0; 2013 2014 /* return the ATA registers to the caller.*/ 2015 req_task->io_ports[1] = reply->features; 2016 req_task->io_ports[2] = reply->sect_count; 2017 req_task->io_ports[3] = reply->lba_low; 2018 req_task->io_ports[4] = reply->lba_mid; 2019 req_task->io_ports[5] = reply->lba_hi; 2020 req_task->io_ports[6] = reply->device; 2021 2022 if (req_task->out_flags.all & 1) { 2023 2024 req_task->hob_ports[3] = reply->lba_low_ex; 2025 req_task->hob_ports[4] = reply->lba_mid_ex; 2026 req_task->hob_ports[5] = reply->lba_hi_ex; 2027 req_task->hob_ports[1] = reply->features_ex; 2028 req_task->hob_ports[2] = reply->sect_cnt_ex; 2029 } 2030 dbg_printk(MTIP_DRV_NAME 2031 " %s: Completion: stat %x," 2032 "err %x, sect_cnt %x, lbalo %x," 2033 "lbamid %x, lbahi %x, dev %x\n", 2034 __func__, 2035 req_task->io_ports[7], 2036 req_task->io_ports[1], 2037 req_task->io_ports[2], 2038 req_task->io_ports[3], 2039 req_task->io_ports[4], 2040 req_task->io_ports[5], 2041 req_task->io_ports[6]); 2042 2043 if (taskout) { 2044 if (copy_to_user(buf + outtotal, outbuf, taskout)) { 2045 err = -EFAULT; 2046 goto abort; 2047 } 2048 } 2049 if (taskin) { 2050 if (copy_to_user(buf + intotal, inbuf, taskin)) { 2051 err = -EFAULT; 2052 goto abort; 2053 } 2054 } 2055 abort: 2056 if (inbuf_dma) 2057 pci_unmap_single(dd->pdev, inbuf_dma, 2058 taskin, DMA_FROM_DEVICE); 2059 if (outbuf_dma) 2060 pci_unmap_single(dd->pdev, outbuf_dma, 2061 taskout, DMA_TO_DEVICE); 2062 kfree(outbuf); 2063 kfree(inbuf); 2064 2065 return err; 2066 } 2067 2068 /* 2069 * Handle IOCTL calls from the Block Layer. 2070 * 2071 * This function is called by the Block Layer when it receives an IOCTL 2072 * command that it does not understand. If the IOCTL command is not supported 2073 * this function returns -ENOTTY. 2074 * 2075 * @dd Pointer to the driver data structure. 2076 * @cmd IOCTL command passed from the Block Layer. 2077 * @arg IOCTL argument passed from the Block Layer. 2078 * 2079 * return value 2080 * 0 The IOCTL completed successfully. 2081 * -ENOTTY The specified command is not supported. 2082 * -EFAULT An error occurred copying data to a user space buffer. 2083 * -EIO An error occurred while executing the command. 2084 */ 2085 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, 2086 unsigned long arg) 2087 { 2088 switch (cmd) { 2089 case HDIO_GET_IDENTITY: 2090 { 2091 if (copy_to_user((void __user *)arg, dd->port->identify, 2092 sizeof(u16) * ATA_ID_WORDS)) 2093 return -EFAULT; 2094 break; 2095 } 2096 case HDIO_DRIVE_CMD: 2097 { 2098 u8 drive_command[4]; 2099 2100 /* Copy the user command info to our buffer. */ 2101 if (copy_from_user(drive_command, 2102 (void __user *) arg, 2103 sizeof(drive_command))) 2104 return -EFAULT; 2105 2106 /* Execute the drive command. */ 2107 if (exec_drive_command(dd->port, 2108 drive_command, 2109 (void __user *) (arg+4))) 2110 return -EIO; 2111 2112 /* Copy the status back to the users buffer. */ 2113 if (copy_to_user((void __user *) arg, 2114 drive_command, 2115 sizeof(drive_command))) 2116 return -EFAULT; 2117 2118 break; 2119 } 2120 case HDIO_DRIVE_TASK: 2121 { 2122 u8 drive_command[7]; 2123 2124 /* Copy the user command info to our buffer. */ 2125 if (copy_from_user(drive_command, 2126 (void __user *) arg, 2127 sizeof(drive_command))) 2128 return -EFAULT; 2129 2130 /* Execute the drive command. */ 2131 if (exec_drive_task(dd->port, drive_command)) 2132 return -EIO; 2133 2134 /* Copy the status back to the users buffer. */ 2135 if (copy_to_user((void __user *) arg, 2136 drive_command, 2137 sizeof(drive_command))) 2138 return -EFAULT; 2139 2140 break; 2141 } 2142 case HDIO_DRIVE_TASKFILE: { 2143 ide_task_request_t req_task; 2144 int ret, outtotal; 2145 2146 if (copy_from_user(&req_task, (void __user *) arg, 2147 sizeof(req_task))) 2148 return -EFAULT; 2149 2150 outtotal = sizeof(req_task); 2151 2152 ret = exec_drive_taskfile(dd, (void __user *) arg, 2153 &req_task, outtotal); 2154 2155 if (copy_to_user((void __user *) arg, &req_task, 2156 sizeof(req_task))) 2157 return -EFAULT; 2158 2159 return ret; 2160 } 2161 2162 default: 2163 return -EINVAL; 2164 } 2165 return 0; 2166 } 2167 2168 /* 2169 * Submit an IO to the hw 2170 * 2171 * This function is called by the block layer to issue an io 2172 * to the device. Upon completion, the callback function will 2173 * be called with the data parameter passed as the callback data. 2174 * 2175 * @dd Pointer to the driver data structure. 2176 * @start First sector to read. 2177 * @nsect Number of sectors to read. 2178 * @nents Number of entries in scatter list for the read command. 2179 * @tag The tag of this read command. 2180 * @callback Pointer to the function that should be called 2181 * when the read completes. 2182 * @data Callback data passed to the callback function 2183 * when the read completes. 2184 * @dir Direction (read or write) 2185 * 2186 * return value 2187 * None 2188 */ 2189 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, 2190 struct mtip_cmd *command, int nents, 2191 struct blk_mq_hw_ctx *hctx) 2192 { 2193 struct host_to_dev_fis *fis; 2194 struct mtip_port *port = dd->port; 2195 int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 2196 u64 start = blk_rq_pos(rq); 2197 unsigned int nsect = blk_rq_sectors(rq); 2198 2199 /* Map the scatter list for DMA access */ 2200 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); 2201 2202 prefetch(&port->flags); 2203 2204 command->scatter_ents = nents; 2205 2206 /* 2207 * The number of retries for this command before it is 2208 * reported as a failure to the upper layers. 2209 */ 2210 command->retries = MTIP_MAX_RETRIES; 2211 2212 /* Fill out fis */ 2213 fis = command->command; 2214 fis->type = 0x27; 2215 fis->opts = 1 << 7; 2216 if (dma_dir == DMA_FROM_DEVICE) 2217 fis->command = ATA_CMD_FPDMA_READ; 2218 else 2219 fis->command = ATA_CMD_FPDMA_WRITE; 2220 fis->lba_low = start & 0xFF; 2221 fis->lba_mid = (start >> 8) & 0xFF; 2222 fis->lba_hi = (start >> 16) & 0xFF; 2223 fis->lba_low_ex = (start >> 24) & 0xFF; 2224 fis->lba_mid_ex = (start >> 32) & 0xFF; 2225 fis->lba_hi_ex = (start >> 40) & 0xFF; 2226 fis->device = 1 << 6; 2227 fis->features = nsect & 0xFF; 2228 fis->features_ex = (nsect >> 8) & 0xFF; 2229 fis->sect_count = ((rq->tag << 3) | (rq->tag >> 5)); 2230 fis->sect_cnt_ex = 0; 2231 fis->control = 0; 2232 fis->res2 = 0; 2233 fis->res3 = 0; 2234 fill_command_sg(dd, command, nents); 2235 2236 if (unlikely(command->unaligned)) 2237 fis->device |= 1 << 7; 2238 2239 /* Populate the command header */ 2240 command->command_header->opts = 2241 __force_bit2int cpu_to_le32( 2242 (nents << 16) | 5 | AHCI_CMD_PREFETCH); 2243 command->command_header->byte_count = 0; 2244 2245 command->direction = dma_dir; 2246 2247 /* 2248 * To prevent this command from being issued 2249 * if an internal command is in progress or error handling is active. 2250 */ 2251 if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) { 2252 set_bit(rq->tag, port->cmds_to_issue); 2253 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2254 return; 2255 } 2256 2257 /* Issue the command to the hardware */ 2258 mtip_issue_ncq_command(port, rq->tag); 2259 } 2260 2261 /* 2262 * Sysfs status dump. 2263 * 2264 * @dev Pointer to the device structure, passed by the kernrel. 2265 * @attr Pointer to the device_attribute structure passed by the kernel. 2266 * @buf Pointer to the char buffer that will receive the stats info. 2267 * 2268 * return value 2269 * The size, in bytes, of the data copied into buf. 2270 */ 2271 static ssize_t mtip_hw_show_status(struct device *dev, 2272 struct device_attribute *attr, 2273 char *buf) 2274 { 2275 struct driver_data *dd = dev_to_disk(dev)->private_data; 2276 int size = 0; 2277 2278 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) 2279 size += sprintf(buf, "%s", "thermal_shutdown\n"); 2280 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) 2281 size += sprintf(buf, "%s", "write_protect\n"); 2282 else 2283 size += sprintf(buf, "%s", "online\n"); 2284 2285 return size; 2286 } 2287 2288 static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL); 2289 2290 /* debugsfs entries */ 2291 2292 static ssize_t show_device_status(struct device_driver *drv, char *buf) 2293 { 2294 int size = 0; 2295 struct driver_data *dd, *tmp; 2296 unsigned long flags; 2297 char id_buf[42]; 2298 u16 status = 0; 2299 2300 spin_lock_irqsave(&dev_lock, flags); 2301 size += sprintf(&buf[size], "Devices Present:\n"); 2302 list_for_each_entry_safe(dd, tmp, &online_list, online_list) { 2303 if (dd->pdev) { 2304 if (dd->port && 2305 dd->port->identify && 2306 dd->port->identify_valid) { 2307 strlcpy(id_buf, 2308 (char *) (dd->port->identify + 10), 21); 2309 status = *(dd->port->identify + 141); 2310 } else { 2311 memset(id_buf, 0, 42); 2312 status = 0; 2313 } 2314 2315 if (dd->port && 2316 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { 2317 size += sprintf(&buf[size], 2318 " device %s %s (ftl rebuild %d %%)\n", 2319 dev_name(&dd->pdev->dev), 2320 id_buf, 2321 status); 2322 } else { 2323 size += sprintf(&buf[size], 2324 " device %s %s\n", 2325 dev_name(&dd->pdev->dev), 2326 id_buf); 2327 } 2328 } 2329 } 2330 2331 size += sprintf(&buf[size], "Devices Being Removed:\n"); 2332 list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) { 2333 if (dd->pdev) { 2334 if (dd->port && 2335 dd->port->identify && 2336 dd->port->identify_valid) { 2337 strlcpy(id_buf, 2338 (char *) (dd->port->identify+10), 21); 2339 status = *(dd->port->identify + 141); 2340 } else { 2341 memset(id_buf, 0, 42); 2342 status = 0; 2343 } 2344 2345 if (dd->port && 2346 test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) { 2347 size += sprintf(&buf[size], 2348 " device %s %s (ftl rebuild %d %%)\n", 2349 dev_name(&dd->pdev->dev), 2350 id_buf, 2351 status); 2352 } else { 2353 size += sprintf(&buf[size], 2354 " device %s %s\n", 2355 dev_name(&dd->pdev->dev), 2356 id_buf); 2357 } 2358 } 2359 } 2360 spin_unlock_irqrestore(&dev_lock, flags); 2361 2362 return size; 2363 } 2364 2365 static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf, 2366 size_t len, loff_t *offset) 2367 { 2368 struct driver_data *dd = (struct driver_data *)f->private_data; 2369 int size = *offset; 2370 char *buf; 2371 int rv = 0; 2372 2373 if (!len || *offset) 2374 return 0; 2375 2376 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2377 if (!buf) { 2378 dev_err(&dd->pdev->dev, 2379 "Memory allocation: status buffer\n"); 2380 return -ENOMEM; 2381 } 2382 2383 size += show_device_status(NULL, buf); 2384 2385 *offset = size <= len ? size : len; 2386 size = copy_to_user(ubuf, buf, *offset); 2387 if (size) 2388 rv = -EFAULT; 2389 2390 kfree(buf); 2391 return rv ? rv : *offset; 2392 } 2393 2394 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf, 2395 size_t len, loff_t *offset) 2396 { 2397 struct driver_data *dd = (struct driver_data *)f->private_data; 2398 char *buf; 2399 u32 group_allocated; 2400 int size = *offset; 2401 int n, rv = 0; 2402 2403 if (!len || size) 2404 return 0; 2405 2406 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2407 if (!buf) { 2408 dev_err(&dd->pdev->dev, 2409 "Memory allocation: register buffer\n"); 2410 return -ENOMEM; 2411 } 2412 2413 size += sprintf(&buf[size], "H/ S ACTive : [ 0x"); 2414 2415 for (n = dd->slot_groups-1; n >= 0; n--) 2416 size += sprintf(&buf[size], "%08X ", 2417 readl(dd->port->s_active[n])); 2418 2419 size += sprintf(&buf[size], "]\n"); 2420 size += sprintf(&buf[size], "H/ Command Issue : [ 0x"); 2421 2422 for (n = dd->slot_groups-1; n >= 0; n--) 2423 size += sprintf(&buf[size], "%08X ", 2424 readl(dd->port->cmd_issue[n])); 2425 2426 size += sprintf(&buf[size], "]\n"); 2427 size += sprintf(&buf[size], "H/ Completed : [ 0x"); 2428 2429 for (n = dd->slot_groups-1; n >= 0; n--) 2430 size += sprintf(&buf[size], "%08X ", 2431 readl(dd->port->completed[n])); 2432 2433 size += sprintf(&buf[size], "]\n"); 2434 size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n", 2435 readl(dd->port->mmio + PORT_IRQ_STAT)); 2436 size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n", 2437 readl(dd->mmio + HOST_IRQ_STAT)); 2438 size += sprintf(&buf[size], "\n"); 2439 2440 size += sprintf(&buf[size], "L/ Commands in Q : [ 0x"); 2441 2442 for (n = dd->slot_groups-1; n >= 0; n--) { 2443 if (sizeof(long) > sizeof(u32)) 2444 group_allocated = 2445 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); 2446 else 2447 group_allocated = dd->port->cmds_to_issue[n]; 2448 size += sprintf(&buf[size], "%08X ", group_allocated); 2449 } 2450 size += sprintf(&buf[size], "]\n"); 2451 2452 *offset = size <= len ? size : len; 2453 size = copy_to_user(ubuf, buf, *offset); 2454 if (size) 2455 rv = -EFAULT; 2456 2457 kfree(buf); 2458 return rv ? rv : *offset; 2459 } 2460 2461 static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf, 2462 size_t len, loff_t *offset) 2463 { 2464 struct driver_data *dd = (struct driver_data *)f->private_data; 2465 char *buf; 2466 int size = *offset; 2467 int rv = 0; 2468 2469 if (!len || size) 2470 return 0; 2471 2472 buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL); 2473 if (!buf) { 2474 dev_err(&dd->pdev->dev, 2475 "Memory allocation: flag buffer\n"); 2476 return -ENOMEM; 2477 } 2478 2479 size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n", 2480 dd->port->flags); 2481 size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n", 2482 dd->dd_flag); 2483 2484 *offset = size <= len ? size : len; 2485 size = copy_to_user(ubuf, buf, *offset); 2486 if (size) 2487 rv = -EFAULT; 2488 2489 kfree(buf); 2490 return rv ? rv : *offset; 2491 } 2492 2493 static const struct file_operations mtip_device_status_fops = { 2494 .owner = THIS_MODULE, 2495 .open = simple_open, 2496 .read = mtip_hw_read_device_status, 2497 .llseek = no_llseek, 2498 }; 2499 2500 static const struct file_operations mtip_regs_fops = { 2501 .owner = THIS_MODULE, 2502 .open = simple_open, 2503 .read = mtip_hw_read_registers, 2504 .llseek = no_llseek, 2505 }; 2506 2507 static const struct file_operations mtip_flags_fops = { 2508 .owner = THIS_MODULE, 2509 .open = simple_open, 2510 .read = mtip_hw_read_flags, 2511 .llseek = no_llseek, 2512 }; 2513 2514 /* 2515 * Create the sysfs related attributes. 2516 * 2517 * @dd Pointer to the driver data structure. 2518 * @kobj Pointer to the kobj for the block device. 2519 * 2520 * return value 2521 * 0 Operation completed successfully. 2522 * -EINVAL Invalid parameter. 2523 */ 2524 static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj) 2525 { 2526 if (!kobj || !dd) 2527 return -EINVAL; 2528 2529 if (sysfs_create_file(kobj, &dev_attr_status.attr)) 2530 dev_warn(&dd->pdev->dev, 2531 "Error creating 'status' sysfs entry\n"); 2532 return 0; 2533 } 2534 2535 /* 2536 * Remove the sysfs related attributes. 2537 * 2538 * @dd Pointer to the driver data structure. 2539 * @kobj Pointer to the kobj for the block device. 2540 * 2541 * return value 2542 * 0 Operation completed successfully. 2543 * -EINVAL Invalid parameter. 2544 */ 2545 static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj) 2546 { 2547 if (!kobj || !dd) 2548 return -EINVAL; 2549 2550 sysfs_remove_file(kobj, &dev_attr_status.attr); 2551 2552 return 0; 2553 } 2554 2555 static int mtip_hw_debugfs_init(struct driver_data *dd) 2556 { 2557 if (!dfs_parent) 2558 return -1; 2559 2560 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); 2561 if (IS_ERR_OR_NULL(dd->dfs_node)) { 2562 dev_warn(&dd->pdev->dev, 2563 "Error creating node %s under debugfs\n", 2564 dd->disk->disk_name); 2565 dd->dfs_node = NULL; 2566 return -1; 2567 } 2568 2569 debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops); 2570 debugfs_create_file("registers", 0444, dd->dfs_node, dd, 2571 &mtip_regs_fops); 2572 2573 return 0; 2574 } 2575 2576 static void mtip_hw_debugfs_exit(struct driver_data *dd) 2577 { 2578 if (dd->dfs_node) 2579 debugfs_remove_recursive(dd->dfs_node); 2580 } 2581 2582 /* 2583 * Perform any init/resume time hardware setup 2584 * 2585 * @dd Pointer to the driver data structure. 2586 * 2587 * return value 2588 * None 2589 */ 2590 static inline void hba_setup(struct driver_data *dd) 2591 { 2592 u32 hwdata; 2593 hwdata = readl(dd->mmio + HOST_HSORG); 2594 2595 /* interrupt bug workaround: use only 1 IS bit.*/ 2596 writel(hwdata | 2597 HSORG_DISABLE_SLOTGRP_INTR | 2598 HSORG_DISABLE_SLOTGRP_PXIS, 2599 dd->mmio + HOST_HSORG); 2600 } 2601 2602 static int mtip_device_unaligned_constrained(struct driver_data *dd) 2603 { 2604 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); 2605 } 2606 2607 /* 2608 * Detect the details of the product, and store anything needed 2609 * into the driver data structure. This includes product type and 2610 * version and number of slot groups. 2611 * 2612 * @dd Pointer to the driver data structure. 2613 * 2614 * return value 2615 * None 2616 */ 2617 static void mtip_detect_product(struct driver_data *dd) 2618 { 2619 u32 hwdata; 2620 unsigned int rev, slotgroups; 2621 2622 /* 2623 * HBA base + 0xFC [15:0] - vendor-specific hardware interface 2624 * info register: 2625 * [15:8] hardware/software interface rev# 2626 * [ 3] asic-style interface 2627 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style). 2628 */ 2629 hwdata = readl(dd->mmio + HOST_HSORG); 2630 2631 dd->product_type = MTIP_PRODUCT_UNKNOWN; 2632 dd->slot_groups = 1; 2633 2634 if (hwdata & 0x8) { 2635 dd->product_type = MTIP_PRODUCT_ASICFPGA; 2636 rev = (hwdata & HSORG_HWREV) >> 8; 2637 slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1; 2638 dev_info(&dd->pdev->dev, 2639 "ASIC-FPGA design, HS rev 0x%x, " 2640 "%i slot groups [%i slots]\n", 2641 rev, 2642 slotgroups, 2643 slotgroups * 32); 2644 2645 if (slotgroups > MTIP_MAX_SLOT_GROUPS) { 2646 dev_warn(&dd->pdev->dev, 2647 "Warning: driver only supports " 2648 "%i slot groups.\n", MTIP_MAX_SLOT_GROUPS); 2649 slotgroups = MTIP_MAX_SLOT_GROUPS; 2650 } 2651 dd->slot_groups = slotgroups; 2652 return; 2653 } 2654 2655 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); 2656 } 2657 2658 /* 2659 * Blocking wait for FTL rebuild to complete 2660 * 2661 * @dd Pointer to the DRIVER_DATA structure. 2662 * 2663 * return value 2664 * 0 FTL rebuild completed successfully 2665 * -EFAULT FTL rebuild error/timeout/interruption 2666 */ 2667 static int mtip_ftl_rebuild_poll(struct driver_data *dd) 2668 { 2669 unsigned long timeout, cnt = 0, start; 2670 2671 dev_warn(&dd->pdev->dev, 2672 "FTL rebuild in progress. Polling for completion.\n"); 2673 2674 start = jiffies; 2675 timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS); 2676 2677 do { 2678 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2679 &dd->dd_flag))) 2680 return -EFAULT; 2681 if (mtip_check_surprise_removal(dd->pdev)) 2682 return -EFAULT; 2683 2684 if (mtip_get_identify(dd->port, NULL) < 0) 2685 return -EFAULT; 2686 2687 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 2688 MTIP_FTL_REBUILD_MAGIC) { 2689 ssleep(1); 2690 /* Print message every 3 minutes */ 2691 if (cnt++ >= 180) { 2692 dev_warn(&dd->pdev->dev, 2693 "FTL rebuild in progress (%d secs).\n", 2694 jiffies_to_msecs(jiffies - start) / 1000); 2695 cnt = 0; 2696 } 2697 } else { 2698 dev_warn(&dd->pdev->dev, 2699 "FTL rebuild complete (%d secs).\n", 2700 jiffies_to_msecs(jiffies - start) / 1000); 2701 mtip_block_initialize(dd); 2702 return 0; 2703 } 2704 } while (time_before(jiffies, timeout)); 2705 2706 /* Check for timeout */ 2707 dev_err(&dd->pdev->dev, 2708 "Timed out waiting for FTL rebuild to complete (%d secs).\n", 2709 jiffies_to_msecs(jiffies - start) / 1000); 2710 return -EFAULT; 2711 } 2712 2713 static void mtip_softirq_done_fn(struct request *rq) 2714 { 2715 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 2716 struct driver_data *dd = rq->q->queuedata; 2717 2718 /* Unmap the DMA scatter list entries */ 2719 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, 2720 cmd->direction); 2721 2722 if (unlikely(cmd->unaligned)) 2723 up(&dd->port->cmd_slot_unal); 2724 2725 blk_mq_end_request(rq, cmd->status); 2726 } 2727 2728 static void mtip_abort_cmd(struct request *req, void *data, bool reserved) 2729 { 2730 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req); 2731 struct driver_data *dd = data; 2732 2733 dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag); 2734 2735 clear_bit(req->tag, dd->port->cmds_to_issue); 2736 cmd->status = BLK_STS_IOERR; 2737 mtip_softirq_done_fn(req); 2738 } 2739 2740 static void mtip_queue_cmd(struct request *req, void *data, bool reserved) 2741 { 2742 struct driver_data *dd = data; 2743 2744 set_bit(req->tag, dd->port->cmds_to_issue); 2745 blk_abort_request(req); 2746 } 2747 2748 /* 2749 * service thread to issue queued commands 2750 * 2751 * @data Pointer to the driver data structure. 2752 * 2753 * return value 2754 * 0 2755 */ 2756 2757 static int mtip_service_thread(void *data) 2758 { 2759 struct driver_data *dd = (struct driver_data *)data; 2760 unsigned long slot, slot_start, slot_wrap, to; 2761 unsigned int num_cmd_slots = dd->slot_groups * 32; 2762 struct mtip_port *port = dd->port; 2763 2764 while (1) { 2765 if (kthread_should_stop() || 2766 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 2767 goto st_out; 2768 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 2769 2770 /* 2771 * the condition is to check neither an internal command is 2772 * is in progress nor error handling is active 2773 */ 2774 wait_event_interruptible(port->svc_wait, (port->flags) && 2775 (port->flags & MTIP_PF_SVC_THD_WORK)); 2776 2777 if (kthread_should_stop() || 2778 test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 2779 goto st_out; 2780 2781 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2782 &dd->dd_flag))) 2783 goto st_out; 2784 2785 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 2786 2787 restart_eh: 2788 /* Demux bits: start with error handling */ 2789 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) { 2790 mtip_handle_tfe(dd); 2791 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 2792 } 2793 2794 if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) 2795 goto restart_eh; 2796 2797 if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) { 2798 to = jiffies + msecs_to_jiffies(5000); 2799 2800 do { 2801 mdelay(100); 2802 } while (atomic_read(&dd->irq_workers_active) != 0 && 2803 time_before(jiffies, to)); 2804 2805 if (atomic_read(&dd->irq_workers_active) != 0) 2806 dev_warn(&dd->pdev->dev, 2807 "Completion workers still active!"); 2808 2809 blk_mq_quiesce_queue(dd->queue); 2810 2811 spin_lock(dd->queue->queue_lock); 2812 blk_mq_tagset_busy_iter(&dd->tags, 2813 mtip_queue_cmd, dd); 2814 spin_unlock(dd->queue->queue_lock); 2815 2816 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags); 2817 2818 if (mtip_device_reset(dd)) 2819 blk_mq_tagset_busy_iter(&dd->tags, 2820 mtip_abort_cmd, dd); 2821 2822 clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags); 2823 2824 blk_mq_unquiesce_queue(dd->queue); 2825 } 2826 2827 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 2828 slot = 1; 2829 /* used to restrict the loop to one iteration */ 2830 slot_start = num_cmd_slots; 2831 slot_wrap = 0; 2832 while (1) { 2833 slot = find_next_bit(port->cmds_to_issue, 2834 num_cmd_slots, slot); 2835 if (slot_wrap == 1) { 2836 if ((slot_start >= slot) || 2837 (slot >= num_cmd_slots)) 2838 break; 2839 } 2840 if (unlikely(slot_start == num_cmd_slots)) 2841 slot_start = slot; 2842 2843 if (unlikely(slot == num_cmd_slots)) { 2844 slot = 1; 2845 slot_wrap = 1; 2846 continue; 2847 } 2848 2849 /* Issue the command to the hardware */ 2850 mtip_issue_ncq_command(port, slot); 2851 2852 clear_bit(slot, port->cmds_to_issue); 2853 } 2854 2855 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2856 } 2857 2858 if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { 2859 if (mtip_ftl_rebuild_poll(dd) == 0) 2860 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); 2861 } 2862 } 2863 2864 st_out: 2865 return 0; 2866 } 2867 2868 /* 2869 * DMA region teardown 2870 * 2871 * @dd Pointer to driver_data structure 2872 * 2873 * return value 2874 * None 2875 */ 2876 static void mtip_dma_free(struct driver_data *dd) 2877 { 2878 struct mtip_port *port = dd->port; 2879 2880 if (port->block1) 2881 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 2882 port->block1, port->block1_dma); 2883 2884 if (port->command_list) { 2885 dmam_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, 2886 port->command_list, port->command_list_dma); 2887 } 2888 } 2889 2890 /* 2891 * DMA region setup 2892 * 2893 * @dd Pointer to driver_data structure 2894 * 2895 * return value 2896 * -ENOMEM Not enough free DMA region space to initialize driver 2897 */ 2898 static int mtip_dma_alloc(struct driver_data *dd) 2899 { 2900 struct mtip_port *port = dd->port; 2901 2902 /* Allocate dma memory for RX Fis, Identify, and Sector Bufffer */ 2903 port->block1 = 2904 dmam_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 2905 &port->block1_dma, GFP_KERNEL); 2906 if (!port->block1) 2907 return -ENOMEM; 2908 memset(port->block1, 0, BLOCK_DMA_ALLOC_SZ); 2909 2910 /* Allocate dma memory for command list */ 2911 port->command_list = 2912 dmam_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, 2913 &port->command_list_dma, GFP_KERNEL); 2914 if (!port->command_list) { 2915 dmam_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, 2916 port->block1, port->block1_dma); 2917 port->block1 = NULL; 2918 port->block1_dma = 0; 2919 return -ENOMEM; 2920 } 2921 memset(port->command_list, 0, AHCI_CMD_TBL_SZ); 2922 2923 /* Setup all pointers into first DMA region */ 2924 port->rxfis = port->block1 + AHCI_RX_FIS_OFFSET; 2925 port->rxfis_dma = port->block1_dma + AHCI_RX_FIS_OFFSET; 2926 port->identify = port->block1 + AHCI_IDFY_OFFSET; 2927 port->identify_dma = port->block1_dma + AHCI_IDFY_OFFSET; 2928 port->log_buf = port->block1 + AHCI_SECTBUF_OFFSET; 2929 port->log_buf_dma = port->block1_dma + AHCI_SECTBUF_OFFSET; 2930 port->smart_buf = port->block1 + AHCI_SMARTBUF_OFFSET; 2931 port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET; 2932 2933 return 0; 2934 } 2935 2936 static int mtip_hw_get_identify(struct driver_data *dd) 2937 { 2938 struct smart_attr attr242; 2939 unsigned char *buf; 2940 int rv; 2941 2942 if (mtip_get_identify(dd->port, NULL) < 0) 2943 return -EFAULT; 2944 2945 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 2946 MTIP_FTL_REBUILD_MAGIC) { 2947 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); 2948 return MTIP_FTL_REBUILD_MAGIC; 2949 } 2950 mtip_dump_identify(dd->port); 2951 2952 /* check write protect, over temp and rebuild statuses */ 2953 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 2954 dd->port->log_buf, 2955 dd->port->log_buf_dma, 1); 2956 if (rv) { 2957 dev_warn(&dd->pdev->dev, 2958 "Error in READ LOG EXT (10h) command\n"); 2959 /* non-critical error, don't fail the load */ 2960 } else { 2961 buf = (unsigned char *)dd->port->log_buf; 2962 if (buf[259] & 0x1) { 2963 dev_info(&dd->pdev->dev, 2964 "Write protect bit is set.\n"); 2965 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 2966 } 2967 if (buf[288] == 0xF7) { 2968 dev_info(&dd->pdev->dev, 2969 "Exceeded Tmax, drive in thermal shutdown.\n"); 2970 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 2971 } 2972 if (buf[288] == 0xBF) { 2973 dev_info(&dd->pdev->dev, 2974 "Drive indicates rebuild has failed.\n"); 2975 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); 2976 } 2977 } 2978 2979 /* get write protect progess */ 2980 memset(&attr242, 0, sizeof(struct smart_attr)); 2981 if (mtip_get_smart_attr(dd->port, 242, &attr242)) 2982 dev_warn(&dd->pdev->dev, 2983 "Unable to check write protect progress\n"); 2984 else 2985 dev_info(&dd->pdev->dev, 2986 "Write protect progress: %u%% (%u blocks)\n", 2987 attr242.cur, le32_to_cpu(attr242.data)); 2988 2989 return rv; 2990 } 2991 2992 /* 2993 * Called once for each card. 2994 * 2995 * @dd Pointer to the driver data structure. 2996 * 2997 * return value 2998 * 0 on success, else an error code. 2999 */ 3000 static int mtip_hw_init(struct driver_data *dd) 3001 { 3002 int i; 3003 int rv; 3004 unsigned long timeout, timetaken; 3005 3006 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; 3007 3008 mtip_detect_product(dd); 3009 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { 3010 rv = -EIO; 3011 goto out1; 3012 } 3013 3014 hba_setup(dd); 3015 3016 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, 3017 dd->numa_node); 3018 if (!dd->port) { 3019 dev_err(&dd->pdev->dev, 3020 "Memory allocation: port structure\n"); 3021 return -ENOMEM; 3022 } 3023 3024 /* Continue workqueue setup */ 3025 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3026 dd->work[i].port = dd->port; 3027 3028 /* Enable unaligned IO constraints for some devices */ 3029 if (mtip_device_unaligned_constrained(dd)) 3030 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; 3031 else 3032 dd->unal_qdepth = 0; 3033 3034 sema_init(&dd->port->cmd_slot_unal, dd->unal_qdepth); 3035 3036 /* Spinlock to prevent concurrent issue */ 3037 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3038 spin_lock_init(&dd->port->cmd_issue_lock[i]); 3039 3040 /* Set the port mmio base address. */ 3041 dd->port->mmio = dd->mmio + PORT_OFFSET; 3042 dd->port->dd = dd; 3043 3044 /* DMA allocations */ 3045 rv = mtip_dma_alloc(dd); 3046 if (rv < 0) 3047 goto out1; 3048 3049 /* Setup the pointers to the extended s_active and CI registers. */ 3050 for (i = 0; i < dd->slot_groups; i++) { 3051 dd->port->s_active[i] = 3052 dd->port->mmio + i*0x80 + PORT_SCR_ACT; 3053 dd->port->cmd_issue[i] = 3054 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; 3055 dd->port->completed[i] = 3056 dd->port->mmio + i*0x80 + PORT_SDBV; 3057 } 3058 3059 timetaken = jiffies; 3060 timeout = jiffies + msecs_to_jiffies(30000); 3061 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && 3062 time_before(jiffies, timeout)) { 3063 mdelay(100); 3064 } 3065 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 3066 timetaken = jiffies - timetaken; 3067 dev_warn(&dd->pdev->dev, 3068 "Surprise removal detected at %u ms\n", 3069 jiffies_to_msecs(timetaken)); 3070 rv = -ENODEV; 3071 goto out2 ; 3072 } 3073 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { 3074 timetaken = jiffies - timetaken; 3075 dev_warn(&dd->pdev->dev, 3076 "Removal detected at %u ms\n", 3077 jiffies_to_msecs(timetaken)); 3078 rv = -EFAULT; 3079 goto out2; 3080 } 3081 3082 /* Conditionally reset the HBA. */ 3083 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { 3084 if (mtip_hba_reset(dd) < 0) { 3085 dev_err(&dd->pdev->dev, 3086 "Card did not reset within timeout\n"); 3087 rv = -EIO; 3088 goto out2; 3089 } 3090 } else { 3091 /* Clear any pending interrupts on the HBA */ 3092 writel(readl(dd->mmio + HOST_IRQ_STAT), 3093 dd->mmio + HOST_IRQ_STAT); 3094 } 3095 3096 mtip_init_port(dd->port); 3097 mtip_start_port(dd->port); 3098 3099 /* Setup the ISR and enable interrupts. */ 3100 rv = devm_request_irq(&dd->pdev->dev, 3101 dd->pdev->irq, 3102 mtip_irq_handler, 3103 IRQF_SHARED, 3104 dev_driver_string(&dd->pdev->dev), 3105 dd); 3106 3107 if (rv) { 3108 dev_err(&dd->pdev->dev, 3109 "Unable to allocate IRQ %d\n", dd->pdev->irq); 3110 goto out2; 3111 } 3112 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); 3113 3114 /* Enable interrupts on the HBA. */ 3115 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3116 dd->mmio + HOST_CTL); 3117 3118 init_waitqueue_head(&dd->port->svc_wait); 3119 3120 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 3121 rv = -EFAULT; 3122 goto out3; 3123 } 3124 3125 return rv; 3126 3127 out3: 3128 /* Disable interrupts on the HBA. */ 3129 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3130 dd->mmio + HOST_CTL); 3131 3132 /* Release the IRQ. */ 3133 irq_set_affinity_hint(dd->pdev->irq, NULL); 3134 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3135 3136 out2: 3137 mtip_deinit_port(dd->port); 3138 mtip_dma_free(dd); 3139 3140 out1: 3141 /* Free the memory allocated for the for structure. */ 3142 kfree(dd->port); 3143 3144 return rv; 3145 } 3146 3147 static int mtip_standby_drive(struct driver_data *dd) 3148 { 3149 int rv = 0; 3150 3151 if (dd->sr || !dd->port) 3152 return -ENODEV; 3153 /* 3154 * Send standby immediate (E0h) to the drive so that it 3155 * saves its state. 3156 */ 3157 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && 3158 !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) && 3159 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) { 3160 rv = mtip_standby_immediate(dd->port); 3161 if (rv) 3162 dev_warn(&dd->pdev->dev, 3163 "STANDBY IMMEDIATE failed\n"); 3164 } 3165 return rv; 3166 } 3167 3168 /* 3169 * Called to deinitialize an interface. 3170 * 3171 * @dd Pointer to the driver data structure. 3172 * 3173 * return value 3174 * 0 3175 */ 3176 static int mtip_hw_exit(struct driver_data *dd) 3177 { 3178 if (!dd->sr) { 3179 /* de-initialize the port. */ 3180 mtip_deinit_port(dd->port); 3181 3182 /* Disable interrupts on the HBA. */ 3183 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3184 dd->mmio + HOST_CTL); 3185 } 3186 3187 /* Release the IRQ. */ 3188 irq_set_affinity_hint(dd->pdev->irq, NULL); 3189 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3190 msleep(1000); 3191 3192 /* Free dma regions */ 3193 mtip_dma_free(dd); 3194 3195 /* Free the memory allocated for the for structure. */ 3196 kfree(dd->port); 3197 dd->port = NULL; 3198 3199 return 0; 3200 } 3201 3202 /* 3203 * Issue a Standby Immediate command to the device. 3204 * 3205 * This function is called by the Block Layer just before the 3206 * system powers off during a shutdown. 3207 * 3208 * @dd Pointer to the driver data structure. 3209 * 3210 * return value 3211 * 0 3212 */ 3213 static int mtip_hw_shutdown(struct driver_data *dd) 3214 { 3215 /* 3216 * Send standby immediate (E0h) to the drive so that it 3217 * saves its state. 3218 */ 3219 mtip_standby_drive(dd); 3220 3221 return 0; 3222 } 3223 3224 /* 3225 * Suspend function 3226 * 3227 * This function is called by the Block Layer just before the 3228 * system hibernates. 3229 * 3230 * @dd Pointer to the driver data structure. 3231 * 3232 * return value 3233 * 0 Suspend was successful 3234 * -EFAULT Suspend was not successful 3235 */ 3236 static int mtip_hw_suspend(struct driver_data *dd) 3237 { 3238 /* 3239 * Send standby immediate (E0h) to the drive 3240 * so that it saves its state. 3241 */ 3242 if (mtip_standby_drive(dd) != 0) { 3243 dev_err(&dd->pdev->dev, 3244 "Failed standby-immediate command\n"); 3245 return -EFAULT; 3246 } 3247 3248 /* Disable interrupts on the HBA.*/ 3249 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3250 dd->mmio + HOST_CTL); 3251 mtip_deinit_port(dd->port); 3252 3253 return 0; 3254 } 3255 3256 /* 3257 * Resume function 3258 * 3259 * This function is called by the Block Layer as the 3260 * system resumes. 3261 * 3262 * @dd Pointer to the driver data structure. 3263 * 3264 * return value 3265 * 0 Resume was successful 3266 * -EFAULT Resume was not successful 3267 */ 3268 static int mtip_hw_resume(struct driver_data *dd) 3269 { 3270 /* Perform any needed hardware setup steps */ 3271 hba_setup(dd); 3272 3273 /* Reset the HBA */ 3274 if (mtip_hba_reset(dd) != 0) { 3275 dev_err(&dd->pdev->dev, 3276 "Unable to reset the HBA\n"); 3277 return -EFAULT; 3278 } 3279 3280 /* 3281 * Enable the port, DMA engine, and FIS reception specific 3282 * h/w in controller. 3283 */ 3284 mtip_init_port(dd->port); 3285 mtip_start_port(dd->port); 3286 3287 /* Enable interrupts on the HBA.*/ 3288 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3289 dd->mmio + HOST_CTL); 3290 3291 return 0; 3292 } 3293 3294 /* 3295 * Helper function for reusing disk name 3296 * upon hot insertion. 3297 */ 3298 static int rssd_disk_name_format(char *prefix, 3299 int index, 3300 char *buf, 3301 int buflen) 3302 { 3303 const int base = 'z' - 'a' + 1; 3304 char *begin = buf + strlen(prefix); 3305 char *end = buf + buflen; 3306 char *p; 3307 int unit; 3308 3309 p = end - 1; 3310 *p = '\0'; 3311 unit = base; 3312 do { 3313 if (p == begin) 3314 return -EINVAL; 3315 *--p = 'a' + (index % unit); 3316 index = (index / unit) - 1; 3317 } while (index >= 0); 3318 3319 memmove(begin, p, end - p); 3320 memcpy(buf, prefix, strlen(prefix)); 3321 3322 return 0; 3323 } 3324 3325 /* 3326 * Block layer IOCTL handler. 3327 * 3328 * @dev Pointer to the block_device structure. 3329 * @mode ignored 3330 * @cmd IOCTL command passed from the user application. 3331 * @arg Argument passed from the user application. 3332 * 3333 * return value 3334 * 0 IOCTL completed successfully. 3335 * -ENOTTY IOCTL not supported or invalid driver data 3336 * structure pointer. 3337 */ 3338 static int mtip_block_ioctl(struct block_device *dev, 3339 fmode_t mode, 3340 unsigned cmd, 3341 unsigned long arg) 3342 { 3343 struct driver_data *dd = dev->bd_disk->private_data; 3344 3345 if (!capable(CAP_SYS_ADMIN)) 3346 return -EACCES; 3347 3348 if (!dd) 3349 return -ENOTTY; 3350 3351 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3352 return -ENOTTY; 3353 3354 switch (cmd) { 3355 case BLKFLSBUF: 3356 return -ENOTTY; 3357 default: 3358 return mtip_hw_ioctl(dd, cmd, arg); 3359 } 3360 } 3361 3362 #ifdef CONFIG_COMPAT 3363 /* 3364 * Block layer compat IOCTL handler. 3365 * 3366 * @dev Pointer to the block_device structure. 3367 * @mode ignored 3368 * @cmd IOCTL command passed from the user application. 3369 * @arg Argument passed from the user application. 3370 * 3371 * return value 3372 * 0 IOCTL completed successfully. 3373 * -ENOTTY IOCTL not supported or invalid driver data 3374 * structure pointer. 3375 */ 3376 static int mtip_block_compat_ioctl(struct block_device *dev, 3377 fmode_t mode, 3378 unsigned cmd, 3379 unsigned long arg) 3380 { 3381 struct driver_data *dd = dev->bd_disk->private_data; 3382 3383 if (!capable(CAP_SYS_ADMIN)) 3384 return -EACCES; 3385 3386 if (!dd) 3387 return -ENOTTY; 3388 3389 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3390 return -ENOTTY; 3391 3392 switch (cmd) { 3393 case BLKFLSBUF: 3394 return -ENOTTY; 3395 case HDIO_DRIVE_TASKFILE: { 3396 struct mtip_compat_ide_task_request_s __user *compat_req_task; 3397 ide_task_request_t req_task; 3398 int compat_tasksize, outtotal, ret; 3399 3400 compat_tasksize = 3401 sizeof(struct mtip_compat_ide_task_request_s); 3402 3403 compat_req_task = 3404 (struct mtip_compat_ide_task_request_s __user *) arg; 3405 3406 if (copy_from_user(&req_task, (void __user *) arg, 3407 compat_tasksize - (2 * sizeof(compat_long_t)))) 3408 return -EFAULT; 3409 3410 if (get_user(req_task.out_size, &compat_req_task->out_size)) 3411 return -EFAULT; 3412 3413 if (get_user(req_task.in_size, &compat_req_task->in_size)) 3414 return -EFAULT; 3415 3416 outtotal = sizeof(struct mtip_compat_ide_task_request_s); 3417 3418 ret = exec_drive_taskfile(dd, (void __user *) arg, 3419 &req_task, outtotal); 3420 3421 if (copy_to_user((void __user *) arg, &req_task, 3422 compat_tasksize - 3423 (2 * sizeof(compat_long_t)))) 3424 return -EFAULT; 3425 3426 if (put_user(req_task.out_size, &compat_req_task->out_size)) 3427 return -EFAULT; 3428 3429 if (put_user(req_task.in_size, &compat_req_task->in_size)) 3430 return -EFAULT; 3431 3432 return ret; 3433 } 3434 default: 3435 return mtip_hw_ioctl(dd, cmd, arg); 3436 } 3437 } 3438 #endif 3439 3440 /* 3441 * Obtain the geometry of the device. 3442 * 3443 * You may think that this function is obsolete, but some applications, 3444 * fdisk for example still used CHS values. This function describes the 3445 * device as having 224 heads and 56 sectors per cylinder. These values are 3446 * chosen so that each cylinder is aligned on a 4KB boundary. Since a 3447 * partition is described in terms of a start and end cylinder this means 3448 * that each partition is also 4KB aligned. Non-aligned partitions adversely 3449 * affects performance. 3450 * 3451 * @dev Pointer to the block_device strucutre. 3452 * @geo Pointer to a hd_geometry structure. 3453 * 3454 * return value 3455 * 0 Operation completed successfully. 3456 * -ENOTTY An error occurred while reading the drive capacity. 3457 */ 3458 static int mtip_block_getgeo(struct block_device *dev, 3459 struct hd_geometry *geo) 3460 { 3461 struct driver_data *dd = dev->bd_disk->private_data; 3462 sector_t capacity; 3463 3464 if (!dd) 3465 return -ENOTTY; 3466 3467 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3468 dev_warn(&dd->pdev->dev, 3469 "Could not get drive capacity.\n"); 3470 return -ENOTTY; 3471 } 3472 3473 geo->heads = 224; 3474 geo->sectors = 56; 3475 sector_div(capacity, (geo->heads * geo->sectors)); 3476 geo->cylinders = capacity; 3477 return 0; 3478 } 3479 3480 static int mtip_block_open(struct block_device *dev, fmode_t mode) 3481 { 3482 struct driver_data *dd; 3483 3484 if (dev && dev->bd_disk) { 3485 dd = (struct driver_data *) dev->bd_disk->private_data; 3486 3487 if (dd) { 3488 if (test_bit(MTIP_DDF_REMOVAL_BIT, 3489 &dd->dd_flag)) { 3490 return -ENODEV; 3491 } 3492 return 0; 3493 } 3494 } 3495 return -ENODEV; 3496 } 3497 3498 static void mtip_block_release(struct gendisk *disk, fmode_t mode) 3499 { 3500 } 3501 3502 /* 3503 * Block device operation function. 3504 * 3505 * This structure contains pointers to the functions required by the block 3506 * layer. 3507 */ 3508 static const struct block_device_operations mtip_block_ops = { 3509 .open = mtip_block_open, 3510 .release = mtip_block_release, 3511 .ioctl = mtip_block_ioctl, 3512 #ifdef CONFIG_COMPAT 3513 .compat_ioctl = mtip_block_compat_ioctl, 3514 #endif 3515 .getgeo = mtip_block_getgeo, 3516 .owner = THIS_MODULE 3517 }; 3518 3519 static inline bool is_se_active(struct driver_data *dd) 3520 { 3521 if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) { 3522 if (dd->port->ic_pause_timer) { 3523 unsigned long to = dd->port->ic_pause_timer + 3524 msecs_to_jiffies(1000); 3525 if (time_after(jiffies, to)) { 3526 clear_bit(MTIP_PF_SE_ACTIVE_BIT, 3527 &dd->port->flags); 3528 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); 3529 dd->port->ic_pause_timer = 0; 3530 wake_up_interruptible(&dd->port->svc_wait); 3531 return false; 3532 } 3533 } 3534 return true; 3535 } 3536 return false; 3537 } 3538 3539 /* 3540 * Block layer make request function. 3541 * 3542 * This function is called by the kernel to process a BIO for 3543 * the P320 device. 3544 * 3545 * @queue Pointer to the request queue. Unused other than to obtain 3546 * the driver data structure. 3547 * @rq Pointer to the request. 3548 * 3549 */ 3550 static int mtip_submit_request(struct blk_mq_hw_ctx *hctx, struct request *rq) 3551 { 3552 struct driver_data *dd = hctx->queue->queuedata; 3553 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3554 unsigned int nents; 3555 3556 if (is_se_active(dd)) 3557 return -ENODATA; 3558 3559 if (unlikely(dd->dd_flag & MTIP_DDF_STOP_IO)) { 3560 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 3561 &dd->dd_flag))) { 3562 return -ENXIO; 3563 } 3564 if (unlikely(test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))) { 3565 return -ENODATA; 3566 } 3567 if (unlikely(test_bit(MTIP_DDF_WRITE_PROTECT_BIT, 3568 &dd->dd_flag) && 3569 rq_data_dir(rq))) { 3570 return -ENODATA; 3571 } 3572 if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag) || 3573 test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))) 3574 return -ENODATA; 3575 } 3576 3577 if (req_op(rq) == REQ_OP_DISCARD) { 3578 int err; 3579 3580 err = mtip_send_trim(dd, blk_rq_pos(rq), blk_rq_sectors(rq)); 3581 blk_mq_end_request(rq, err ? BLK_STS_IOERR : BLK_STS_OK); 3582 return 0; 3583 } 3584 3585 /* Create the scatter list for this request. */ 3586 nents = blk_rq_map_sg(hctx->queue, rq, cmd->sg); 3587 3588 /* Issue the read/write. */ 3589 mtip_hw_submit_io(dd, rq, cmd, nents, hctx); 3590 return 0; 3591 } 3592 3593 static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx, 3594 struct request *rq) 3595 { 3596 struct driver_data *dd = hctx->queue->queuedata; 3597 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3598 3599 if (rq_data_dir(rq) == READ || !dd->unal_qdepth) 3600 return false; 3601 3602 /* 3603 * If unaligned depth must be limited on this controller, mark it 3604 * as unaligned if the IO isn't on a 4k boundary (start of length). 3605 */ 3606 if (blk_rq_sectors(rq) <= 64) { 3607 if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7)) 3608 cmd->unaligned = 1; 3609 } 3610 3611 if (cmd->unaligned && down_trylock(&dd->port->cmd_slot_unal)) 3612 return true; 3613 3614 return false; 3615 } 3616 3617 static blk_status_t mtip_issue_reserved_cmd(struct blk_mq_hw_ctx *hctx, 3618 struct request *rq) 3619 { 3620 struct driver_data *dd = hctx->queue->queuedata; 3621 struct mtip_int_cmd *icmd = rq->special; 3622 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3623 struct mtip_cmd_sg *command_sg; 3624 3625 if (mtip_commands_active(dd->port)) 3626 return BLK_STS_RESOURCE; 3627 3628 /* Populate the SG list */ 3629 cmd->command_header->opts = 3630 __force_bit2int cpu_to_le32(icmd->opts | icmd->fis_len); 3631 if (icmd->buf_len) { 3632 command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ; 3633 3634 command_sg->info = 3635 __force_bit2int cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF); 3636 command_sg->dba = 3637 __force_bit2int cpu_to_le32(icmd->buffer & 0xFFFFFFFF); 3638 command_sg->dba_upper = 3639 __force_bit2int cpu_to_le32((icmd->buffer >> 16) >> 16); 3640 3641 cmd->command_header->opts |= 3642 __force_bit2int cpu_to_le32((1 << 16)); 3643 } 3644 3645 /* Populate the command header */ 3646 cmd->command_header->byte_count = 0; 3647 3648 blk_mq_start_request(rq); 3649 mtip_issue_non_ncq_command(dd->port, rq->tag); 3650 return 0; 3651 } 3652 3653 static blk_status_t mtip_queue_rq(struct blk_mq_hw_ctx *hctx, 3654 const struct blk_mq_queue_data *bd) 3655 { 3656 struct request *rq = bd->rq; 3657 int ret; 3658 3659 mtip_init_cmd_header(rq); 3660 3661 if (blk_rq_is_passthrough(rq)) 3662 return mtip_issue_reserved_cmd(hctx, rq); 3663 3664 if (unlikely(mtip_check_unal_depth(hctx, rq))) 3665 return BLK_STS_RESOURCE; 3666 3667 blk_mq_start_request(rq); 3668 3669 ret = mtip_submit_request(hctx, rq); 3670 if (likely(!ret)) 3671 return BLK_STS_OK; 3672 return BLK_STS_IOERR; 3673 } 3674 3675 static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq, 3676 unsigned int hctx_idx) 3677 { 3678 struct driver_data *dd = set->driver_data; 3679 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3680 3681 if (!cmd->command) 3682 return; 3683 3684 dmam_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, 3685 cmd->command, cmd->command_dma); 3686 } 3687 3688 static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq, 3689 unsigned int hctx_idx, unsigned int numa_node) 3690 { 3691 struct driver_data *dd = set->driver_data; 3692 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3693 3694 cmd->command = dmam_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, 3695 &cmd->command_dma, GFP_KERNEL); 3696 if (!cmd->command) 3697 return -ENOMEM; 3698 3699 memset(cmd->command, 0, CMD_DMA_ALLOC_SZ); 3700 3701 sg_init_table(cmd->sg, MTIP_MAX_SG); 3702 return 0; 3703 } 3704 3705 static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req, 3706 bool reserved) 3707 { 3708 struct driver_data *dd = req->q->queuedata; 3709 3710 if (reserved) { 3711 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req); 3712 3713 cmd->status = BLK_STS_TIMEOUT; 3714 blk_mq_complete_request(req); 3715 return BLK_EH_DONE; 3716 } 3717 3718 if (test_bit(req->tag, dd->port->cmds_to_issue)) 3719 goto exit_handler; 3720 3721 if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags)) 3722 goto exit_handler; 3723 3724 wake_up_interruptible(&dd->port->svc_wait); 3725 exit_handler: 3726 return BLK_EH_RESET_TIMER; 3727 } 3728 3729 static const struct blk_mq_ops mtip_mq_ops = { 3730 .queue_rq = mtip_queue_rq, 3731 .init_request = mtip_init_cmd, 3732 .exit_request = mtip_free_cmd, 3733 .complete = mtip_softirq_done_fn, 3734 .timeout = mtip_cmd_timeout, 3735 }; 3736 3737 /* 3738 * Block layer initialization function. 3739 * 3740 * This function is called once by the PCI layer for each P320 3741 * device that is connected to the system. 3742 * 3743 * @dd Pointer to the driver data structure. 3744 * 3745 * return value 3746 * 0 on success else an error code. 3747 */ 3748 static int mtip_block_initialize(struct driver_data *dd) 3749 { 3750 int rv = 0, wait_for_rebuild = 0; 3751 sector_t capacity; 3752 unsigned int index = 0; 3753 struct kobject *kobj; 3754 3755 if (dd->disk) 3756 goto skip_create_disk; /* hw init done, before rebuild */ 3757 3758 if (mtip_hw_init(dd)) { 3759 rv = -EINVAL; 3760 goto protocol_init_error; 3761 } 3762 3763 dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node); 3764 if (dd->disk == NULL) { 3765 dev_err(&dd->pdev->dev, 3766 "Unable to allocate gendisk structure\n"); 3767 rv = -EINVAL; 3768 goto alloc_disk_error; 3769 } 3770 3771 /* Generate the disk name, implemented same as in sd.c */ 3772 do { 3773 if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL)) { 3774 rv = -ENOMEM; 3775 goto ida_get_error; 3776 } 3777 3778 spin_lock(&rssd_index_lock); 3779 rv = ida_get_new(&rssd_index_ida, &index); 3780 spin_unlock(&rssd_index_lock); 3781 } while (rv == -EAGAIN); 3782 3783 if (rv) 3784 goto ida_get_error; 3785 3786 rv = rssd_disk_name_format("rssd", 3787 index, 3788 dd->disk->disk_name, 3789 DISK_NAME_LEN); 3790 if (rv) 3791 goto disk_index_error; 3792 3793 dd->disk->major = dd->major; 3794 dd->disk->first_minor = index * MTIP_MAX_MINORS; 3795 dd->disk->minors = MTIP_MAX_MINORS; 3796 dd->disk->fops = &mtip_block_ops; 3797 dd->disk->private_data = dd; 3798 dd->index = index; 3799 3800 mtip_hw_debugfs_init(dd); 3801 3802 memset(&dd->tags, 0, sizeof(dd->tags)); 3803 dd->tags.ops = &mtip_mq_ops; 3804 dd->tags.nr_hw_queues = 1; 3805 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; 3806 dd->tags.reserved_tags = 1; 3807 dd->tags.cmd_size = sizeof(struct mtip_cmd); 3808 dd->tags.numa_node = dd->numa_node; 3809 dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; 3810 dd->tags.driver_data = dd; 3811 dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS; 3812 3813 rv = blk_mq_alloc_tag_set(&dd->tags); 3814 if (rv) { 3815 dev_err(&dd->pdev->dev, 3816 "Unable to allocate request queue\n"); 3817 goto block_queue_alloc_tag_error; 3818 } 3819 3820 /* Allocate the request queue. */ 3821 dd->queue = blk_mq_init_queue(&dd->tags); 3822 if (IS_ERR(dd->queue)) { 3823 dev_err(&dd->pdev->dev, 3824 "Unable to allocate request queue\n"); 3825 rv = -ENOMEM; 3826 goto block_queue_alloc_init_error; 3827 } 3828 3829 dd->disk->queue = dd->queue; 3830 dd->queue->queuedata = dd; 3831 3832 skip_create_disk: 3833 /* Initialize the protocol layer. */ 3834 wait_for_rebuild = mtip_hw_get_identify(dd); 3835 if (wait_for_rebuild < 0) { 3836 dev_err(&dd->pdev->dev, 3837 "Protocol layer initialization failed\n"); 3838 rv = -EINVAL; 3839 goto init_hw_cmds_error; 3840 } 3841 3842 /* 3843 * if rebuild pending, start the service thread, and delay the block 3844 * queue creation and device_add_disk() 3845 */ 3846 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3847 goto start_service_thread; 3848 3849 /* Set device limits. */ 3850 blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue); 3851 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue); 3852 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); 3853 blk_queue_physical_block_size(dd->queue, 4096); 3854 blk_queue_max_hw_sectors(dd->queue, 0xffff); 3855 blk_queue_max_segment_size(dd->queue, 0x400000); 3856 blk_queue_io_min(dd->queue, 4096); 3857 3858 /* Signal trim support */ 3859 if (dd->trim_supp == true) { 3860 blk_queue_flag_set(QUEUE_FLAG_DISCARD, dd->queue); 3861 dd->queue->limits.discard_granularity = 4096; 3862 blk_queue_max_discard_sectors(dd->queue, 3863 MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES); 3864 } 3865 3866 /* Set the capacity of the device in 512 byte sectors. */ 3867 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3868 dev_warn(&dd->pdev->dev, 3869 "Could not read drive capacity\n"); 3870 rv = -EIO; 3871 goto read_capacity_error; 3872 } 3873 set_capacity(dd->disk, capacity); 3874 3875 /* Enable the block device and add it to /dev */ 3876 device_add_disk(&dd->pdev->dev, dd->disk); 3877 3878 dd->bdev = bdget_disk(dd->disk, 0); 3879 /* 3880 * Now that the disk is active, initialize any sysfs attributes 3881 * managed by the protocol layer. 3882 */ 3883 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 3884 if (kobj) { 3885 mtip_hw_sysfs_init(dd, kobj); 3886 kobject_put(kobj); 3887 } 3888 3889 if (dd->mtip_svc_handler) { 3890 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 3891 return rv; /* service thread created for handling rebuild */ 3892 } 3893 3894 start_service_thread: 3895 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, 3896 dd, dd->numa_node, 3897 "mtip_svc_thd_%02d", index); 3898 3899 if (IS_ERR(dd->mtip_svc_handler)) { 3900 dev_err(&dd->pdev->dev, "service thread failed to start\n"); 3901 dd->mtip_svc_handler = NULL; 3902 rv = -EFAULT; 3903 goto kthread_run_error; 3904 } 3905 wake_up_process(dd->mtip_svc_handler); 3906 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3907 rv = wait_for_rebuild; 3908 3909 return rv; 3910 3911 kthread_run_error: 3912 bdput(dd->bdev); 3913 dd->bdev = NULL; 3914 3915 /* Delete our gendisk. This also removes the device from /dev */ 3916 del_gendisk(dd->disk); 3917 3918 read_capacity_error: 3919 init_hw_cmds_error: 3920 blk_cleanup_queue(dd->queue); 3921 block_queue_alloc_init_error: 3922 blk_mq_free_tag_set(&dd->tags); 3923 block_queue_alloc_tag_error: 3924 mtip_hw_debugfs_exit(dd); 3925 disk_index_error: 3926 spin_lock(&rssd_index_lock); 3927 ida_remove(&rssd_index_ida, index); 3928 spin_unlock(&rssd_index_lock); 3929 3930 ida_get_error: 3931 put_disk(dd->disk); 3932 3933 alloc_disk_error: 3934 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ 3935 3936 protocol_init_error: 3937 return rv; 3938 } 3939 3940 static void mtip_no_dev_cleanup(struct request *rq, void *data, bool reserv) 3941 { 3942 struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); 3943 3944 cmd->status = BLK_STS_IOERR; 3945 blk_mq_complete_request(rq); 3946 } 3947 3948 /* 3949 * Block layer deinitialization function. 3950 * 3951 * Called by the PCI layer as each P320 device is removed. 3952 * 3953 * @dd Pointer to the driver data structure. 3954 * 3955 * return value 3956 * 0 3957 */ 3958 static int mtip_block_remove(struct driver_data *dd) 3959 { 3960 struct kobject *kobj; 3961 3962 mtip_hw_debugfs_exit(dd); 3963 3964 if (dd->mtip_svc_handler) { 3965 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); 3966 wake_up_interruptible(&dd->port->svc_wait); 3967 kthread_stop(dd->mtip_svc_handler); 3968 } 3969 3970 /* Clean up the sysfs attributes, if created */ 3971 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) { 3972 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 3973 if (kobj) { 3974 mtip_hw_sysfs_exit(dd, kobj); 3975 kobject_put(kobj); 3976 } 3977 } 3978 3979 if (!dd->sr) { 3980 /* 3981 * Explicitly wait here for IOs to quiesce, 3982 * as mtip_standby_drive usually won't wait for IOs. 3983 */ 3984 if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS)) 3985 mtip_standby_drive(dd); 3986 } 3987 else 3988 dev_info(&dd->pdev->dev, "device %s surprise removal\n", 3989 dd->disk->disk_name); 3990 3991 blk_freeze_queue_start(dd->queue); 3992 blk_mq_quiesce_queue(dd->queue); 3993 blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd); 3994 blk_mq_unquiesce_queue(dd->queue); 3995 3996 /* 3997 * Delete our gendisk structure. This also removes the device 3998 * from /dev 3999 */ 4000 if (dd->bdev) { 4001 bdput(dd->bdev); 4002 dd->bdev = NULL; 4003 } 4004 if (dd->disk) { 4005 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) 4006 del_gendisk(dd->disk); 4007 if (dd->disk->queue) { 4008 blk_cleanup_queue(dd->queue); 4009 blk_mq_free_tag_set(&dd->tags); 4010 dd->queue = NULL; 4011 } 4012 put_disk(dd->disk); 4013 } 4014 dd->disk = NULL; 4015 4016 spin_lock(&rssd_index_lock); 4017 ida_remove(&rssd_index_ida, dd->index); 4018 spin_unlock(&rssd_index_lock); 4019 4020 /* De-initialize the protocol layer. */ 4021 mtip_hw_exit(dd); 4022 4023 return 0; 4024 } 4025 4026 /* 4027 * Function called by the PCI layer when just before the 4028 * machine shuts down. 4029 * 4030 * If a protocol layer shutdown function is present it will be called 4031 * by this function. 4032 * 4033 * @dd Pointer to the driver data structure. 4034 * 4035 * return value 4036 * 0 4037 */ 4038 static int mtip_block_shutdown(struct driver_data *dd) 4039 { 4040 mtip_hw_shutdown(dd); 4041 4042 /* Delete our gendisk structure, and cleanup the blk queue. */ 4043 if (dd->disk) { 4044 dev_info(&dd->pdev->dev, 4045 "Shutting down %s ...\n", dd->disk->disk_name); 4046 4047 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) 4048 del_gendisk(dd->disk); 4049 if (dd->disk->queue) { 4050 blk_cleanup_queue(dd->queue); 4051 blk_mq_free_tag_set(&dd->tags); 4052 } 4053 put_disk(dd->disk); 4054 dd->disk = NULL; 4055 dd->queue = NULL; 4056 } 4057 4058 spin_lock(&rssd_index_lock); 4059 ida_remove(&rssd_index_ida, dd->index); 4060 spin_unlock(&rssd_index_lock); 4061 return 0; 4062 } 4063 4064 static int mtip_block_suspend(struct driver_data *dd) 4065 { 4066 dev_info(&dd->pdev->dev, 4067 "Suspending %s ...\n", dd->disk->disk_name); 4068 mtip_hw_suspend(dd); 4069 return 0; 4070 } 4071 4072 static int mtip_block_resume(struct driver_data *dd) 4073 { 4074 dev_info(&dd->pdev->dev, "Resuming %s ...\n", 4075 dd->disk->disk_name); 4076 mtip_hw_resume(dd); 4077 return 0; 4078 } 4079 4080 static void drop_cpu(int cpu) 4081 { 4082 cpu_use[cpu]--; 4083 } 4084 4085 static int get_least_used_cpu_on_node(int node) 4086 { 4087 int cpu, least_used_cpu, least_cnt; 4088 const struct cpumask *node_mask; 4089 4090 node_mask = cpumask_of_node(node); 4091 least_used_cpu = cpumask_first(node_mask); 4092 least_cnt = cpu_use[least_used_cpu]; 4093 cpu = least_used_cpu; 4094 4095 for_each_cpu(cpu, node_mask) { 4096 if (cpu_use[cpu] < least_cnt) { 4097 least_used_cpu = cpu; 4098 least_cnt = cpu_use[cpu]; 4099 } 4100 } 4101 cpu_use[least_used_cpu]++; 4102 return least_used_cpu; 4103 } 4104 4105 /* Helper for selecting a node in round robin mode */ 4106 static inline int mtip_get_next_rr_node(void) 4107 { 4108 static int next_node = -1; 4109 4110 if (next_node == -1) { 4111 next_node = first_online_node; 4112 return next_node; 4113 } 4114 4115 next_node = next_online_node(next_node); 4116 if (next_node == MAX_NUMNODES) 4117 next_node = first_online_node; 4118 return next_node; 4119 } 4120 4121 static DEFINE_HANDLER(0); 4122 static DEFINE_HANDLER(1); 4123 static DEFINE_HANDLER(2); 4124 static DEFINE_HANDLER(3); 4125 static DEFINE_HANDLER(4); 4126 static DEFINE_HANDLER(5); 4127 static DEFINE_HANDLER(6); 4128 static DEFINE_HANDLER(7); 4129 4130 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev) 4131 { 4132 int pos; 4133 unsigned short pcie_dev_ctrl; 4134 4135 pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); 4136 if (pos) { 4137 pci_read_config_word(pdev, 4138 pos + PCI_EXP_DEVCTL, 4139 &pcie_dev_ctrl); 4140 if (pcie_dev_ctrl & (1 << 11) || 4141 pcie_dev_ctrl & (1 << 4)) { 4142 dev_info(&dd->pdev->dev, 4143 "Disabling ERO/No-Snoop on bridge device %04x:%04x\n", 4144 pdev->vendor, pdev->device); 4145 pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN | 4146 PCI_EXP_DEVCTL_RELAX_EN); 4147 pci_write_config_word(pdev, 4148 pos + PCI_EXP_DEVCTL, 4149 pcie_dev_ctrl); 4150 } 4151 } 4152 } 4153 4154 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev) 4155 { 4156 /* 4157 * This workaround is specific to AMD/ATI chipset with a PCI upstream 4158 * device with device id 0x5aXX 4159 */ 4160 if (pdev->bus && pdev->bus->self) { 4161 if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI && 4162 ((pdev->bus->self->device & 0xff00) == 0x5a00)) { 4163 mtip_disable_link_opts(dd, pdev->bus->self); 4164 } else { 4165 /* Check further up the topology */ 4166 struct pci_dev *parent_dev = pdev->bus->self; 4167 if (parent_dev->bus && 4168 parent_dev->bus->parent && 4169 parent_dev->bus->parent->self && 4170 parent_dev->bus->parent->self->vendor == 4171 PCI_VENDOR_ID_ATI && 4172 (parent_dev->bus->parent->self->device & 4173 0xff00) == 0x5a00) { 4174 mtip_disable_link_opts(dd, 4175 parent_dev->bus->parent->self); 4176 } 4177 } 4178 } 4179 } 4180 4181 /* 4182 * Called for each supported PCI device detected. 4183 * 4184 * This function allocates the private data structure, enables the 4185 * PCI device and then calls the block layer initialization function. 4186 * 4187 * return value 4188 * 0 on success else an error code. 4189 */ 4190 static int mtip_pci_probe(struct pci_dev *pdev, 4191 const struct pci_device_id *ent) 4192 { 4193 int rv = 0; 4194 struct driver_data *dd = NULL; 4195 char cpu_list[256]; 4196 const struct cpumask *node_mask; 4197 int cpu, i = 0, j = 0; 4198 int my_node = NUMA_NO_NODE; 4199 unsigned long flags; 4200 4201 /* Allocate memory for this devices private data. */ 4202 my_node = pcibus_to_node(pdev->bus); 4203 if (my_node != NUMA_NO_NODE) { 4204 if (!node_online(my_node)) 4205 my_node = mtip_get_next_rr_node(); 4206 } else { 4207 dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n"); 4208 my_node = mtip_get_next_rr_node(); 4209 } 4210 dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n", 4211 my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev), 4212 cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id()); 4213 4214 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); 4215 if (dd == NULL) { 4216 dev_err(&pdev->dev, 4217 "Unable to allocate memory for driver data\n"); 4218 return -ENOMEM; 4219 } 4220 4221 /* Attach the private data to this PCI device. */ 4222 pci_set_drvdata(pdev, dd); 4223 4224 rv = pcim_enable_device(pdev); 4225 if (rv < 0) { 4226 dev_err(&pdev->dev, "Unable to enable device\n"); 4227 goto iomap_err; 4228 } 4229 4230 /* Map BAR5 to memory. */ 4231 rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME); 4232 if (rv < 0) { 4233 dev_err(&pdev->dev, "Unable to map regions\n"); 4234 goto iomap_err; 4235 } 4236 4237 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 4238 rv = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4239 4240 if (rv) { 4241 rv = pci_set_consistent_dma_mask(pdev, 4242 DMA_BIT_MASK(32)); 4243 if (rv) { 4244 dev_warn(&pdev->dev, 4245 "64-bit DMA enable failed\n"); 4246 goto setmask_err; 4247 } 4248 } 4249 } 4250 4251 /* Copy the info we may need later into the private data structure. */ 4252 dd->major = mtip_major; 4253 dd->instance = instance; 4254 dd->pdev = pdev; 4255 dd->numa_node = my_node; 4256 4257 INIT_LIST_HEAD(&dd->online_list); 4258 INIT_LIST_HEAD(&dd->remove_list); 4259 4260 memset(dd->workq_name, 0, 32); 4261 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); 4262 4263 dd->isr_workq = create_workqueue(dd->workq_name); 4264 if (!dd->isr_workq) { 4265 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); 4266 rv = -ENOMEM; 4267 goto setmask_err; 4268 } 4269 4270 memset(cpu_list, 0, sizeof(cpu_list)); 4271 4272 node_mask = cpumask_of_node(dd->numa_node); 4273 if (!cpumask_empty(node_mask)) { 4274 for_each_cpu(cpu, node_mask) 4275 { 4276 snprintf(&cpu_list[j], 256 - j, "%d ", cpu); 4277 j = strlen(cpu_list); 4278 } 4279 4280 dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n", 4281 dd->numa_node, 4282 topology_physical_package_id(cpumask_first(node_mask)), 4283 nr_cpus_node(dd->numa_node), 4284 cpu_list); 4285 } else 4286 dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n"); 4287 4288 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); 4289 dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n", 4290 cpu_to_node(dd->isr_binding), dd->isr_binding); 4291 4292 /* first worker context always runs in ISR */ 4293 dd->work[0].cpu_binding = dd->isr_binding; 4294 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4295 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4296 dd->work[3].cpu_binding = dd->work[0].cpu_binding; 4297 dd->work[4].cpu_binding = dd->work[1].cpu_binding; 4298 dd->work[5].cpu_binding = dd->work[2].cpu_binding; 4299 dd->work[6].cpu_binding = dd->work[2].cpu_binding; 4300 dd->work[7].cpu_binding = dd->work[1].cpu_binding; 4301 4302 /* Log the bindings */ 4303 for_each_present_cpu(cpu) { 4304 memset(cpu_list, 0, sizeof(cpu_list)); 4305 for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) { 4306 if (dd->work[i].cpu_binding == cpu) { 4307 snprintf(&cpu_list[j], 256 - j, "%d ", i); 4308 j = strlen(cpu_list); 4309 } 4310 } 4311 if (j) 4312 dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list); 4313 } 4314 4315 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); 4316 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); 4317 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); 4318 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); 4319 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); 4320 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); 4321 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); 4322 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); 4323 4324 pci_set_master(pdev); 4325 rv = pci_enable_msi(pdev); 4326 if (rv) { 4327 dev_warn(&pdev->dev, 4328 "Unable to enable MSI interrupt.\n"); 4329 goto msi_initialize_err; 4330 } 4331 4332 mtip_fix_ero_nosnoop(dd, pdev); 4333 4334 /* Initialize the block layer. */ 4335 rv = mtip_block_initialize(dd); 4336 if (rv < 0) { 4337 dev_err(&pdev->dev, 4338 "Unable to initialize block layer\n"); 4339 goto block_initialize_err; 4340 } 4341 4342 /* 4343 * Increment the instance count so that each device has a unique 4344 * instance number. 4345 */ 4346 instance++; 4347 if (rv != MTIP_FTL_REBUILD_MAGIC) 4348 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 4349 else 4350 rv = 0; /* device in rebuild state, return 0 from probe */ 4351 4352 /* Add to online list even if in ftl rebuild */ 4353 spin_lock_irqsave(&dev_lock, flags); 4354 list_add(&dd->online_list, &online_list); 4355 spin_unlock_irqrestore(&dev_lock, flags); 4356 4357 goto done; 4358 4359 block_initialize_err: 4360 pci_disable_msi(pdev); 4361 4362 msi_initialize_err: 4363 if (dd->isr_workq) { 4364 flush_workqueue(dd->isr_workq); 4365 destroy_workqueue(dd->isr_workq); 4366 drop_cpu(dd->work[0].cpu_binding); 4367 drop_cpu(dd->work[1].cpu_binding); 4368 drop_cpu(dd->work[2].cpu_binding); 4369 } 4370 setmask_err: 4371 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4372 4373 iomap_err: 4374 kfree(dd); 4375 pci_set_drvdata(pdev, NULL); 4376 return rv; 4377 done: 4378 return rv; 4379 } 4380 4381 /* 4382 * Called for each probed device when the device is removed or the 4383 * driver is unloaded. 4384 * 4385 * return value 4386 * None 4387 */ 4388 static void mtip_pci_remove(struct pci_dev *pdev) 4389 { 4390 struct driver_data *dd = pci_get_drvdata(pdev); 4391 unsigned long flags, to; 4392 4393 set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag); 4394 4395 spin_lock_irqsave(&dev_lock, flags); 4396 list_del_init(&dd->online_list); 4397 list_add(&dd->remove_list, &removing_list); 4398 spin_unlock_irqrestore(&dev_lock, flags); 4399 4400 mtip_check_surprise_removal(pdev); 4401 synchronize_irq(dd->pdev->irq); 4402 4403 /* Spin until workers are done */ 4404 to = jiffies + msecs_to_jiffies(4000); 4405 do { 4406 msleep(20); 4407 } while (atomic_read(&dd->irq_workers_active) != 0 && 4408 time_before(jiffies, to)); 4409 4410 if (!dd->sr) 4411 fsync_bdev(dd->bdev); 4412 4413 if (atomic_read(&dd->irq_workers_active) != 0) { 4414 dev_warn(&dd->pdev->dev, 4415 "Completion workers still active!\n"); 4416 } 4417 4418 blk_set_queue_dying(dd->queue); 4419 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); 4420 4421 /* Clean up the block layer. */ 4422 mtip_block_remove(dd); 4423 4424 if (dd->isr_workq) { 4425 flush_workqueue(dd->isr_workq); 4426 destroy_workqueue(dd->isr_workq); 4427 drop_cpu(dd->work[0].cpu_binding); 4428 drop_cpu(dd->work[1].cpu_binding); 4429 drop_cpu(dd->work[2].cpu_binding); 4430 } 4431 4432 pci_disable_msi(pdev); 4433 4434 spin_lock_irqsave(&dev_lock, flags); 4435 list_del_init(&dd->remove_list); 4436 spin_unlock_irqrestore(&dev_lock, flags); 4437 4438 kfree(dd); 4439 4440 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4441 pci_set_drvdata(pdev, NULL); 4442 } 4443 4444 /* 4445 * Called for each probed device when the device is suspended. 4446 * 4447 * return value 4448 * 0 Success 4449 * <0 Error 4450 */ 4451 static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg) 4452 { 4453 int rv = 0; 4454 struct driver_data *dd = pci_get_drvdata(pdev); 4455 4456 if (!dd) { 4457 dev_err(&pdev->dev, 4458 "Driver private datastructure is NULL\n"); 4459 return -EFAULT; 4460 } 4461 4462 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4463 4464 /* Disable ports & interrupts then send standby immediate */ 4465 rv = mtip_block_suspend(dd); 4466 if (rv < 0) { 4467 dev_err(&pdev->dev, 4468 "Failed to suspend controller\n"); 4469 return rv; 4470 } 4471 4472 /* 4473 * Save the pci config space to pdev structure & 4474 * disable the device 4475 */ 4476 pci_save_state(pdev); 4477 pci_disable_device(pdev); 4478 4479 /* Move to Low power state*/ 4480 pci_set_power_state(pdev, PCI_D3hot); 4481 4482 return rv; 4483 } 4484 4485 /* 4486 * Called for each probed device when the device is resumed. 4487 * 4488 * return value 4489 * 0 Success 4490 * <0 Error 4491 */ 4492 static int mtip_pci_resume(struct pci_dev *pdev) 4493 { 4494 int rv = 0; 4495 struct driver_data *dd; 4496 4497 dd = pci_get_drvdata(pdev); 4498 if (!dd) { 4499 dev_err(&pdev->dev, 4500 "Driver private datastructure is NULL\n"); 4501 return -EFAULT; 4502 } 4503 4504 /* Move the device to active State */ 4505 pci_set_power_state(pdev, PCI_D0); 4506 4507 /* Restore PCI configuration space */ 4508 pci_restore_state(pdev); 4509 4510 /* Enable the PCI device*/ 4511 rv = pcim_enable_device(pdev); 4512 if (rv < 0) { 4513 dev_err(&pdev->dev, 4514 "Failed to enable card during resume\n"); 4515 goto err; 4516 } 4517 pci_set_master(pdev); 4518 4519 /* 4520 * Calls hbaReset, initPort, & startPort function 4521 * then enables interrupts 4522 */ 4523 rv = mtip_block_resume(dd); 4524 if (rv < 0) 4525 dev_err(&pdev->dev, "Unable to resume\n"); 4526 4527 err: 4528 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4529 4530 return rv; 4531 } 4532 4533 /* 4534 * Shutdown routine 4535 * 4536 * return value 4537 * None 4538 */ 4539 static void mtip_pci_shutdown(struct pci_dev *pdev) 4540 { 4541 struct driver_data *dd = pci_get_drvdata(pdev); 4542 if (dd) 4543 mtip_block_shutdown(dd); 4544 } 4545 4546 /* Table of device ids supported by this driver. */ 4547 static const struct pci_device_id mtip_pci_tbl[] = { 4548 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) }, 4549 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) }, 4550 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) }, 4551 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) }, 4552 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) }, 4553 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) }, 4554 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) }, 4555 { 0 } 4556 }; 4557 4558 /* Structure that describes the PCI driver functions. */ 4559 static struct pci_driver mtip_pci_driver = { 4560 .name = MTIP_DRV_NAME, 4561 .id_table = mtip_pci_tbl, 4562 .probe = mtip_pci_probe, 4563 .remove = mtip_pci_remove, 4564 .suspend = mtip_pci_suspend, 4565 .resume = mtip_pci_resume, 4566 .shutdown = mtip_pci_shutdown, 4567 }; 4568 4569 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl); 4570 4571 /* 4572 * Module initialization function. 4573 * 4574 * Called once when the module is loaded. This function allocates a major 4575 * block device number to the Cyclone devices and registers the PCI layer 4576 * of the driver. 4577 * 4578 * Return value 4579 * 0 on success else error code. 4580 */ 4581 static int __init mtip_init(void) 4582 { 4583 int error; 4584 4585 pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n"); 4586 4587 spin_lock_init(&dev_lock); 4588 4589 INIT_LIST_HEAD(&online_list); 4590 INIT_LIST_HEAD(&removing_list); 4591 4592 /* Allocate a major block device number to use with this driver. */ 4593 error = register_blkdev(0, MTIP_DRV_NAME); 4594 if (error <= 0) { 4595 pr_err("Unable to register block device (%d)\n", 4596 error); 4597 return -EBUSY; 4598 } 4599 mtip_major = error; 4600 4601 dfs_parent = debugfs_create_dir("rssd", NULL); 4602 if (IS_ERR_OR_NULL(dfs_parent)) { 4603 pr_warn("Error creating debugfs parent\n"); 4604 dfs_parent = NULL; 4605 } 4606 if (dfs_parent) { 4607 dfs_device_status = debugfs_create_file("device_status", 4608 0444, dfs_parent, NULL, 4609 &mtip_device_status_fops); 4610 if (IS_ERR_OR_NULL(dfs_device_status)) { 4611 pr_err("Error creating device_status node\n"); 4612 dfs_device_status = NULL; 4613 } 4614 } 4615 4616 /* Register our PCI operations. */ 4617 error = pci_register_driver(&mtip_pci_driver); 4618 if (error) { 4619 debugfs_remove(dfs_parent); 4620 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4621 } 4622 4623 return error; 4624 } 4625 4626 /* 4627 * Module de-initialization function. 4628 * 4629 * Called once when the module is unloaded. This function deallocates 4630 * the major block device number allocated by mtip_init() and 4631 * unregisters the PCI layer of the driver. 4632 * 4633 * Return value 4634 * none 4635 */ 4636 static void __exit mtip_exit(void) 4637 { 4638 /* Release the allocated major block device number. */ 4639 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4640 4641 /* Unregister the PCI driver. */ 4642 pci_unregister_driver(&mtip_pci_driver); 4643 4644 debugfs_remove_recursive(dfs_parent); 4645 } 4646 4647 MODULE_AUTHOR("Micron Technology, Inc"); 4648 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver"); 4649 MODULE_LICENSE("GPL"); 4650 MODULE_VERSION(MTIP_DRV_VERSION); 4651 4652 module_init(mtip_init); 4653 module_exit(mtip_exit); 4654