1 /* 2 * Driver for the Micron P320 SSD 3 * Copyright (C) 2011 Micron Technology, Inc. 4 * 5 * Portions of this code were derived from works subjected to the 6 * following copyright: 7 * Copyright (C) 2009 Integrated Device Technology, Inc. 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License as published by 11 * the Free Software Foundation; either version 2 of the License, or 12 * (at your option) any later version. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 */ 20 21 #include <linux/pci.h> 22 #include <linux/interrupt.h> 23 #include <linux/ata.h> 24 #include <linux/delay.h> 25 #include <linux/hdreg.h> 26 #include <linux/uaccess.h> 27 #include <linux/random.h> 28 #include <linux/smp.h> 29 #include <linux/compat.h> 30 #include <linux/fs.h> 31 #include <linux/module.h> 32 #include <linux/genhd.h> 33 #include <linux/blkdev.h> 34 #include <linux/bio.h> 35 #include <linux/dma-mapping.h> 36 #include <linux/idr.h> 37 #include <linux/kthread.h> 38 #include <../drivers/ata/ahci.h> 39 #include <linux/export.h> 40 #include <linux/debugfs.h> 41 #include "mtip32xx.h" 42 43 #define HW_CMD_SLOT_SZ (MTIP_MAX_COMMAND_SLOTS * 32) 44 #define HW_CMD_TBL_SZ (AHCI_CMD_TBL_HDR_SZ + (MTIP_MAX_SG * 16)) 45 #define HW_CMD_TBL_AR_SZ (HW_CMD_TBL_SZ * MTIP_MAX_COMMAND_SLOTS) 46 #define HW_PORT_PRIV_DMA_SZ \ 47 (HW_CMD_SLOT_SZ + HW_CMD_TBL_AR_SZ + AHCI_RX_FIS_SZ) 48 49 #define HOST_CAP_NZDMA (1 << 19) 50 #define HOST_HSORG 0xFC 51 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24) 52 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16) 53 #define HSORG_HWREV 0xFF00 54 #define HSORG_STYLE 0x8 55 #define HSORG_SLOTGROUPS 0x7 56 57 #define PORT_COMMAND_ISSUE 0x38 58 #define PORT_SDBV 0x7C 59 60 #define PORT_OFFSET 0x100 61 #define PORT_MEM_SIZE 0x80 62 63 #define PORT_IRQ_ERR \ 64 (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \ 65 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \ 66 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \ 67 PORT_IRQ_OVERFLOW) 68 #define PORT_IRQ_LEGACY \ 69 (PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS) 70 #define PORT_IRQ_HANDLED \ 71 (PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \ 72 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \ 73 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY) 74 #define DEF_PORT_IRQ \ 75 (PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS) 76 77 /* product numbers */ 78 #define MTIP_PRODUCT_UNKNOWN 0x00 79 #define MTIP_PRODUCT_ASICFPGA 0x11 80 81 /* Device instance number, incremented each time a device is probed. */ 82 static int instance; 83 84 /* 85 * Global variable used to hold the major block device number 86 * allocated in mtip_init(). 87 */ 88 static int mtip_major; 89 static struct dentry *dfs_parent; 90 91 static u32 cpu_use[NR_CPUS]; 92 93 static DEFINE_SPINLOCK(rssd_index_lock); 94 static DEFINE_IDA(rssd_index_ida); 95 96 static int mtip_block_initialize(struct driver_data *dd); 97 98 #ifdef CONFIG_COMPAT 99 struct mtip_compat_ide_task_request_s { 100 __u8 io_ports[8]; 101 __u8 hob_ports[8]; 102 ide_reg_valid_t out_flags; 103 ide_reg_valid_t in_flags; 104 int data_phase; 105 int req_cmd; 106 compat_ulong_t out_size; 107 compat_ulong_t in_size; 108 }; 109 #endif 110 111 /* 112 * This function check_for_surprise_removal is called 113 * while card is removed from the system and it will 114 * read the vendor id from the configration space 115 * 116 * @pdev Pointer to the pci_dev structure. 117 * 118 * return value 119 * true if device removed, else false 120 */ 121 static bool mtip_check_surprise_removal(struct pci_dev *pdev) 122 { 123 u16 vendor_id = 0; 124 125 /* Read the vendorID from the configuration space */ 126 pci_read_config_word(pdev, 0x00, &vendor_id); 127 if (vendor_id == 0xFFFF) 128 return true; /* device removed */ 129 130 return false; /* device present */ 131 } 132 133 /* 134 * This function is called for clean the pending command in the 135 * command slot during the surprise removal of device and return 136 * error to the upper layer. 137 * 138 * @dd Pointer to the DRIVER_DATA structure. 139 * 140 * return value 141 * None 142 */ 143 static void mtip_command_cleanup(struct driver_data *dd) 144 { 145 int group = 0, commandslot = 0, commandindex = 0; 146 struct mtip_cmd *command; 147 struct mtip_port *port = dd->port; 148 static int in_progress; 149 150 if (in_progress) 151 return; 152 153 in_progress = 1; 154 155 for (group = 0; group < 4; group++) { 156 for (commandslot = 0; commandslot < 32; commandslot++) { 157 if (!(port->allocated[group] & (1 << commandslot))) 158 continue; 159 160 commandindex = group << 5 | commandslot; 161 command = &port->commands[commandindex]; 162 163 if (atomic_read(&command->active) 164 && (command->async_callback)) { 165 command->async_callback(command->async_data, 166 -ENODEV); 167 command->async_callback = NULL; 168 command->async_data = NULL; 169 } 170 171 dma_unmap_sg(&port->dd->pdev->dev, 172 command->sg, 173 command->scatter_ents, 174 command->direction); 175 } 176 } 177 178 up(&port->cmd_slot); 179 180 set_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag); 181 in_progress = 0; 182 } 183 184 /* 185 * Obtain an empty command slot. 186 * 187 * This function needs to be reentrant since it could be called 188 * at the same time on multiple CPUs. The allocation of the 189 * command slot must be atomic. 190 * 191 * @port Pointer to the port data structure. 192 * 193 * return value 194 * >= 0 Index of command slot obtained. 195 * -1 No command slots available. 196 */ 197 static int get_slot(struct mtip_port *port) 198 { 199 int slot, i; 200 unsigned int num_command_slots = port->dd->slot_groups * 32; 201 202 /* 203 * Try 10 times, because there is a small race here. 204 * that's ok, because it's still cheaper than a lock. 205 * 206 * Race: Since this section is not protected by lock, same bit 207 * could be chosen by different process contexts running in 208 * different processor. So instead of costly lock, we are going 209 * with loop. 210 */ 211 for (i = 0; i < 10; i++) { 212 slot = find_next_zero_bit(port->allocated, 213 num_command_slots, 1); 214 if ((slot < num_command_slots) && 215 (!test_and_set_bit(slot, port->allocated))) 216 return slot; 217 } 218 dev_warn(&port->dd->pdev->dev, "Failed to get a tag.\n"); 219 220 if (mtip_check_surprise_removal(port->dd->pdev)) { 221 /* Device not present, clean outstanding commands */ 222 mtip_command_cleanup(port->dd); 223 } 224 return -1; 225 } 226 227 /* 228 * Release a command slot. 229 * 230 * @port Pointer to the port data structure. 231 * @tag Tag of command to release 232 * 233 * return value 234 * None 235 */ 236 static inline void release_slot(struct mtip_port *port, int tag) 237 { 238 smp_mb__before_clear_bit(); 239 clear_bit(tag, port->allocated); 240 smp_mb__after_clear_bit(); 241 } 242 243 /* 244 * Reset the HBA (without sleeping) 245 * 246 * Just like hba_reset, except does not call sleep, so can be 247 * run from interrupt/tasklet context. 248 * 249 * @dd Pointer to the driver data structure. 250 * 251 * return value 252 * 0 The reset was successful. 253 * -1 The HBA Reset bit did not clear. 254 */ 255 static int hba_reset_nosleep(struct driver_data *dd) 256 { 257 unsigned long timeout; 258 259 /* Chip quirk: quiesce any chip function */ 260 mdelay(10); 261 262 /* Set the reset bit */ 263 writel(HOST_RESET, dd->mmio + HOST_CTL); 264 265 /* Flush */ 266 readl(dd->mmio + HOST_CTL); 267 268 /* 269 * Wait 10ms then spin for up to 1 second 270 * waiting for reset acknowledgement 271 */ 272 timeout = jiffies + msecs_to_jiffies(1000); 273 mdelay(10); 274 while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) 275 && time_before(jiffies, timeout)) 276 mdelay(1); 277 278 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) 279 return -1; 280 281 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) 282 return -1; 283 284 return 0; 285 } 286 287 /* 288 * Issue a command to the hardware. 289 * 290 * Set the appropriate bit in the s_active and Command Issue hardware 291 * registers, causing hardware command processing to begin. 292 * 293 * @port Pointer to the port structure. 294 * @tag The tag of the command to be issued. 295 * 296 * return value 297 * None 298 */ 299 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag) 300 { 301 int group = tag >> 5; 302 303 atomic_set(&port->commands[tag].active, 1); 304 305 /* guard SACT and CI registers */ 306 spin_lock(&port->cmd_issue_lock[group]); 307 writel((1 << MTIP_TAG_BIT(tag)), 308 port->s_active[MTIP_TAG_INDEX(tag)]); 309 writel((1 << MTIP_TAG_BIT(tag)), 310 port->cmd_issue[MTIP_TAG_INDEX(tag)]); 311 spin_unlock(&port->cmd_issue_lock[group]); 312 313 /* Set the command's timeout value.*/ 314 port->commands[tag].comp_time = jiffies + msecs_to_jiffies( 315 MTIP_NCQ_COMMAND_TIMEOUT_MS); 316 } 317 318 /* 319 * Enable/disable the reception of FIS 320 * 321 * @port Pointer to the port data structure 322 * @enable 1 to enable, 0 to disable 323 * 324 * return value 325 * Previous state: 1 enabled, 0 disabled 326 */ 327 static int mtip_enable_fis(struct mtip_port *port, int enable) 328 { 329 u32 tmp; 330 331 /* enable FIS reception */ 332 tmp = readl(port->mmio + PORT_CMD); 333 if (enable) 334 writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 335 else 336 writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD); 337 338 /* Flush */ 339 readl(port->mmio + PORT_CMD); 340 341 return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX)); 342 } 343 344 /* 345 * Enable/disable the DMA engine 346 * 347 * @port Pointer to the port data structure 348 * @enable 1 to enable, 0 to disable 349 * 350 * return value 351 * Previous state: 1 enabled, 0 disabled. 352 */ 353 static int mtip_enable_engine(struct mtip_port *port, int enable) 354 { 355 u32 tmp; 356 357 /* enable FIS reception */ 358 tmp = readl(port->mmio + PORT_CMD); 359 if (enable) 360 writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD); 361 else 362 writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD); 363 364 readl(port->mmio + PORT_CMD); 365 return (((tmp & PORT_CMD_START) == PORT_CMD_START)); 366 } 367 368 /* 369 * Enables the port DMA engine and FIS reception. 370 * 371 * return value 372 * None 373 */ 374 static inline void mtip_start_port(struct mtip_port *port) 375 { 376 /* Enable FIS reception */ 377 mtip_enable_fis(port, 1); 378 379 /* Enable the DMA engine */ 380 mtip_enable_engine(port, 1); 381 } 382 383 /* 384 * Deinitialize a port by disabling port interrupts, the DMA engine, 385 * and FIS reception. 386 * 387 * @port Pointer to the port structure 388 * 389 * return value 390 * None 391 */ 392 static inline void mtip_deinit_port(struct mtip_port *port) 393 { 394 /* Disable interrupts on this port */ 395 writel(0, port->mmio + PORT_IRQ_MASK); 396 397 /* Disable the DMA engine */ 398 mtip_enable_engine(port, 0); 399 400 /* Disable FIS reception */ 401 mtip_enable_fis(port, 0); 402 } 403 404 /* 405 * Initialize a port. 406 * 407 * This function deinitializes the port by calling mtip_deinit_port() and 408 * then initializes it by setting the command header and RX FIS addresses, 409 * clearing the SError register and any pending port interrupts before 410 * re-enabling the default set of port interrupts. 411 * 412 * @port Pointer to the port structure. 413 * 414 * return value 415 * None 416 */ 417 static void mtip_init_port(struct mtip_port *port) 418 { 419 int i; 420 mtip_deinit_port(port); 421 422 /* Program the command list base and FIS base addresses */ 423 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { 424 writel((port->command_list_dma >> 16) >> 16, 425 port->mmio + PORT_LST_ADDR_HI); 426 writel((port->rxfis_dma >> 16) >> 16, 427 port->mmio + PORT_FIS_ADDR_HI); 428 } 429 430 writel(port->command_list_dma & 0xFFFFFFFF, 431 port->mmio + PORT_LST_ADDR); 432 writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR); 433 434 /* Clear SError */ 435 writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR); 436 437 /* reset the completed registers.*/ 438 for (i = 0; i < port->dd->slot_groups; i++) 439 writel(0xFFFFFFFF, port->completed[i]); 440 441 /* Clear any pending interrupts for this port */ 442 writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT); 443 444 /* Clear any pending interrupts on the HBA. */ 445 writel(readl(port->dd->mmio + HOST_IRQ_STAT), 446 port->dd->mmio + HOST_IRQ_STAT); 447 448 /* Enable port interrupts */ 449 writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK); 450 } 451 452 /* 453 * Restart a port 454 * 455 * @port Pointer to the port data structure. 456 * 457 * return value 458 * None 459 */ 460 static void mtip_restart_port(struct mtip_port *port) 461 { 462 unsigned long timeout; 463 464 /* Disable the DMA engine */ 465 mtip_enable_engine(port, 0); 466 467 /* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */ 468 timeout = jiffies + msecs_to_jiffies(500); 469 while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) 470 && time_before(jiffies, timeout)) 471 ; 472 473 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 474 return; 475 476 /* 477 * Chip quirk: escalate to hba reset if 478 * PxCMD.CR not clear after 500 ms 479 */ 480 if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) { 481 dev_warn(&port->dd->pdev->dev, 482 "PxCMD.CR not clear, escalating reset\n"); 483 484 if (hba_reset_nosleep(port->dd)) 485 dev_err(&port->dd->pdev->dev, 486 "HBA reset escalation failed.\n"); 487 488 /* 30 ms delay before com reset to quiesce chip */ 489 mdelay(30); 490 } 491 492 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); 493 494 /* Set PxSCTL.DET */ 495 writel(readl(port->mmio + PORT_SCR_CTL) | 496 1, port->mmio + PORT_SCR_CTL); 497 readl(port->mmio + PORT_SCR_CTL); 498 499 /* Wait 1 ms to quiesce chip function */ 500 timeout = jiffies + msecs_to_jiffies(1); 501 while (time_before(jiffies, timeout)) 502 ; 503 504 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 505 return; 506 507 /* Clear PxSCTL.DET */ 508 writel(readl(port->mmio + PORT_SCR_CTL) & ~1, 509 port->mmio + PORT_SCR_CTL); 510 readl(port->mmio + PORT_SCR_CTL); 511 512 /* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */ 513 timeout = jiffies + msecs_to_jiffies(500); 514 while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 515 && time_before(jiffies, timeout)) 516 ; 517 518 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 519 return; 520 521 if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0) 522 dev_warn(&port->dd->pdev->dev, 523 "COM reset failed\n"); 524 525 mtip_init_port(port); 526 mtip_start_port(port); 527 528 } 529 530 /* 531 * Helper function for tag logging 532 */ 533 static void print_tags(struct driver_data *dd, 534 char *msg, 535 unsigned long *tagbits, 536 int cnt) 537 { 538 unsigned char tagmap[128]; 539 int group, tagmap_len = 0; 540 541 memset(tagmap, 0, sizeof(tagmap)); 542 for (group = SLOTBITS_IN_LONGS; group > 0; group--) 543 tagmap_len = sprintf(tagmap + tagmap_len, "%016lX ", 544 tagbits[group-1]); 545 dev_warn(&dd->pdev->dev, 546 "%d command(s) %s: tagmap [%s]", cnt, msg, tagmap); 547 } 548 549 /* 550 * Called periodically to see if any read/write commands are 551 * taking too long to complete. 552 * 553 * @data Pointer to the PORT data structure. 554 * 555 * return value 556 * None 557 */ 558 static void mtip_timeout_function(unsigned long int data) 559 { 560 struct mtip_port *port = (struct mtip_port *) data; 561 struct host_to_dev_fis *fis; 562 struct mtip_cmd *command; 563 int tag, cmdto_cnt = 0; 564 unsigned int bit, group; 565 unsigned int num_command_slots; 566 unsigned long to, tagaccum[SLOTBITS_IN_LONGS]; 567 568 if (unlikely(!port)) 569 return; 570 571 if (test_bit(MTIP_DDF_RESUME_BIT, &port->dd->dd_flag)) { 572 mod_timer(&port->cmd_timer, 573 jiffies + msecs_to_jiffies(30000)); 574 return; 575 } 576 /* clear the tag accumulator */ 577 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 578 num_command_slots = port->dd->slot_groups * 32; 579 580 for (tag = 0; tag < num_command_slots; tag++) { 581 /* 582 * Skip internal command slot as it has 583 * its own timeout mechanism 584 */ 585 if (tag == MTIP_TAG_INTERNAL) 586 continue; 587 588 if (atomic_read(&port->commands[tag].active) && 589 (time_after(jiffies, port->commands[tag].comp_time))) { 590 group = tag >> 5; 591 bit = tag & 0x1F; 592 593 command = &port->commands[tag]; 594 fis = (struct host_to_dev_fis *) command->command; 595 596 set_bit(tag, tagaccum); 597 cmdto_cnt++; 598 if (cmdto_cnt == 1) 599 set_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 600 601 /* 602 * Clear the completed bit. This should prevent 603 * any interrupt handlers from trying to retire 604 * the command. 605 */ 606 writel(1 << bit, port->completed[group]); 607 608 /* Call the async completion callback. */ 609 if (likely(command->async_callback)) 610 command->async_callback(command->async_data, 611 -EIO); 612 command->async_callback = NULL; 613 command->comp_func = NULL; 614 615 /* Unmap the DMA scatter list entries */ 616 dma_unmap_sg(&port->dd->pdev->dev, 617 command->sg, 618 command->scatter_ents, 619 command->direction); 620 621 /* 622 * Clear the allocated bit and active tag for the 623 * command. 624 */ 625 atomic_set(&port->commands[tag].active, 0); 626 release_slot(port, tag); 627 628 up(&port->cmd_slot); 629 } 630 } 631 632 if (cmdto_cnt) { 633 print_tags(port->dd, "timed out", tagaccum, cmdto_cnt); 634 if (!test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { 635 mtip_restart_port(port); 636 wake_up_interruptible(&port->svc_wait); 637 } 638 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 639 } 640 641 if (port->ic_pause_timer) { 642 to = port->ic_pause_timer + msecs_to_jiffies(1000); 643 if (time_after(jiffies, to)) { 644 if (!test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) { 645 port->ic_pause_timer = 0; 646 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 647 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 648 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 649 wake_up_interruptible(&port->svc_wait); 650 } 651 652 653 } 654 } 655 656 /* Restart the timer */ 657 mod_timer(&port->cmd_timer, 658 jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD)); 659 } 660 661 /* 662 * IO completion function. 663 * 664 * This completion function is called by the driver ISR when a 665 * command that was issued by the kernel completes. It first calls the 666 * asynchronous completion function which normally calls back into the block 667 * layer passing the asynchronous callback data, then unmaps the 668 * scatter list associated with the completed command, and finally 669 * clears the allocated bit associated with the completed command. 670 * 671 * @port Pointer to the port data structure. 672 * @tag Tag of the command. 673 * @data Pointer to driver_data. 674 * @status Completion status. 675 * 676 * return value 677 * None 678 */ 679 static void mtip_async_complete(struct mtip_port *port, 680 int tag, 681 void *data, 682 int status) 683 { 684 struct mtip_cmd *command; 685 struct driver_data *dd = data; 686 int cb_status = status ? -EIO : 0; 687 688 if (unlikely(!dd) || unlikely(!port)) 689 return; 690 691 command = &port->commands[tag]; 692 693 if (unlikely(status == PORT_IRQ_TF_ERR)) { 694 dev_warn(&port->dd->pdev->dev, 695 "Command tag %d failed due to TFE\n", tag); 696 } 697 698 /* Upper layer callback */ 699 if (likely(command->async_callback)) 700 command->async_callback(command->async_data, cb_status); 701 702 command->async_callback = NULL; 703 command->comp_func = NULL; 704 705 /* Unmap the DMA scatter list entries */ 706 dma_unmap_sg(&dd->pdev->dev, 707 command->sg, 708 command->scatter_ents, 709 command->direction); 710 711 /* Clear the allocated and active bits for the command */ 712 atomic_set(&port->commands[tag].active, 0); 713 release_slot(port, tag); 714 715 up(&port->cmd_slot); 716 } 717 718 /* 719 * Internal command completion callback function. 720 * 721 * This function is normally called by the driver ISR when an internal 722 * command completed. This function signals the command completion by 723 * calling complete(). 724 * 725 * @port Pointer to the port data structure. 726 * @tag Tag of the command that has completed. 727 * @data Pointer to a completion structure. 728 * @status Completion status. 729 * 730 * return value 731 * None 732 */ 733 static void mtip_completion(struct mtip_port *port, 734 int tag, 735 void *data, 736 int status) 737 { 738 struct mtip_cmd *command = &port->commands[tag]; 739 struct completion *waiting = data; 740 if (unlikely(status == PORT_IRQ_TF_ERR)) 741 dev_warn(&port->dd->pdev->dev, 742 "Internal command %d completed with TFE\n", tag); 743 744 command->async_callback = NULL; 745 command->comp_func = NULL; 746 747 complete(waiting); 748 } 749 750 static void mtip_null_completion(struct mtip_port *port, 751 int tag, 752 void *data, 753 int status) 754 { 755 return; 756 } 757 758 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 759 dma_addr_t buffer_dma, unsigned int sectors); 760 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 761 struct smart_attr *attrib); 762 /* 763 * Handle an error. 764 * 765 * @dd Pointer to the DRIVER_DATA structure. 766 * 767 * return value 768 * None 769 */ 770 static void mtip_handle_tfe(struct driver_data *dd) 771 { 772 int group, tag, bit, reissue, rv; 773 struct mtip_port *port; 774 struct mtip_cmd *cmd; 775 u32 completed; 776 struct host_to_dev_fis *fis; 777 unsigned long tagaccum[SLOTBITS_IN_LONGS]; 778 unsigned int cmd_cnt = 0; 779 unsigned char *buf; 780 char *fail_reason = NULL; 781 int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0; 782 783 dev_warn(&dd->pdev->dev, "Taskfile error\n"); 784 785 port = dd->port; 786 787 /* Stop the timer to prevent command timeouts. */ 788 del_timer(&port->cmd_timer); 789 set_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 790 791 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && 792 test_bit(MTIP_TAG_INTERNAL, port->allocated)) { 793 cmd = &port->commands[MTIP_TAG_INTERNAL]; 794 dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n"); 795 796 atomic_inc(&cmd->active); /* active > 1 indicates error */ 797 if (cmd->comp_data && cmd->comp_func) { 798 cmd->comp_func(port, MTIP_TAG_INTERNAL, 799 cmd->comp_data, PORT_IRQ_TF_ERR); 800 } 801 goto handle_tfe_exit; 802 } 803 804 /* clear the tag accumulator */ 805 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 806 807 /* Loop through all the groups */ 808 for (group = 0; group < dd->slot_groups; group++) { 809 completed = readl(port->completed[group]); 810 811 /* clear completed status register in the hardware.*/ 812 writel(completed, port->completed[group]); 813 814 /* Process successfully completed commands */ 815 for (bit = 0; bit < 32 && completed; bit++) { 816 if (!(completed & (1<<bit))) 817 continue; 818 tag = (group << 5) + bit; 819 820 /* Skip the internal command slot */ 821 if (tag == MTIP_TAG_INTERNAL) 822 continue; 823 824 cmd = &port->commands[tag]; 825 if (likely(cmd->comp_func)) { 826 set_bit(tag, tagaccum); 827 cmd_cnt++; 828 atomic_set(&cmd->active, 0); 829 cmd->comp_func(port, 830 tag, 831 cmd->comp_data, 832 0); 833 } else { 834 dev_err(&port->dd->pdev->dev, 835 "Missing completion func for tag %d", 836 tag); 837 if (mtip_check_surprise_removal(dd->pdev)) { 838 mtip_command_cleanup(dd); 839 /* don't proceed further */ 840 return; 841 } 842 } 843 } 844 } 845 846 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); 847 848 /* Restart the port */ 849 mdelay(20); 850 mtip_restart_port(port); 851 852 /* Trying to determine the cause of the error */ 853 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 854 dd->port->log_buf, 855 dd->port->log_buf_dma, 1); 856 if (rv) { 857 dev_warn(&dd->pdev->dev, 858 "Error in READ LOG EXT (10h) command\n"); 859 /* non-critical error, don't fail the load */ 860 } else { 861 buf = (unsigned char *)dd->port->log_buf; 862 if (buf[259] & 0x1) { 863 dev_info(&dd->pdev->dev, 864 "Write protect bit is set.\n"); 865 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 866 fail_all_ncq_write = 1; 867 fail_reason = "write protect"; 868 } 869 if (buf[288] == 0xF7) { 870 dev_info(&dd->pdev->dev, 871 "Exceeded Tmax, drive in thermal shutdown.\n"); 872 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 873 fail_all_ncq_cmds = 1; 874 fail_reason = "thermal shutdown"; 875 } 876 if (buf[288] == 0xBF) { 877 dev_info(&dd->pdev->dev, 878 "Drive indicates rebuild has failed.\n"); 879 fail_all_ncq_cmds = 1; 880 fail_reason = "rebuild failed"; 881 } 882 } 883 884 /* clear the tag accumulator */ 885 memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long)); 886 887 /* Loop through all the groups */ 888 for (group = 0; group < dd->slot_groups; group++) { 889 for (bit = 0; bit < 32; bit++) { 890 reissue = 1; 891 tag = (group << 5) + bit; 892 cmd = &port->commands[tag]; 893 894 /* If the active bit is set re-issue the command */ 895 if (atomic_read(&cmd->active) == 0) 896 continue; 897 898 fis = (struct host_to_dev_fis *)cmd->command; 899 900 /* Should re-issue? */ 901 if (tag == MTIP_TAG_INTERNAL || 902 fis->command == ATA_CMD_SET_FEATURES) 903 reissue = 0; 904 else { 905 if (fail_all_ncq_cmds || 906 (fail_all_ncq_write && 907 fis->command == ATA_CMD_FPDMA_WRITE)) { 908 dev_warn(&dd->pdev->dev, 909 " Fail: %s w/tag %d [%s].\n", 910 fis->command == ATA_CMD_FPDMA_WRITE ? 911 "write" : "read", 912 tag, 913 fail_reason != NULL ? 914 fail_reason : "unknown"); 915 atomic_set(&cmd->active, 0); 916 if (cmd->comp_func) { 917 cmd->comp_func(port, tag, 918 cmd->comp_data, 919 -ENODATA); 920 } 921 continue; 922 } 923 } 924 925 /* 926 * First check if this command has 927 * exceeded its retries. 928 */ 929 if (reissue && (cmd->retries-- > 0)) { 930 931 set_bit(tag, tagaccum); 932 933 /* Re-issue the command. */ 934 mtip_issue_ncq_command(port, tag); 935 936 continue; 937 } 938 939 /* Retire a command that will not be reissued */ 940 dev_warn(&port->dd->pdev->dev, 941 "retiring tag %d\n", tag); 942 atomic_set(&cmd->active, 0); 943 944 if (cmd->comp_func) 945 cmd->comp_func( 946 port, 947 tag, 948 cmd->comp_data, 949 PORT_IRQ_TF_ERR); 950 else 951 dev_warn(&port->dd->pdev->dev, 952 "Bad completion for tag %d\n", 953 tag); 954 } 955 } 956 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); 957 958 handle_tfe_exit: 959 /* clear eh_active */ 960 clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags); 961 wake_up_interruptible(&port->svc_wait); 962 963 mod_timer(&port->cmd_timer, 964 jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD)); 965 } 966 967 /* 968 * Handle a set device bits interrupt 969 */ 970 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group, 971 u32 completed) 972 { 973 struct driver_data *dd = port->dd; 974 int tag, bit; 975 struct mtip_cmd *command; 976 977 if (!completed) { 978 WARN_ON_ONCE(!completed); 979 return; 980 } 981 /* clear completed status register in the hardware.*/ 982 writel(completed, port->completed[group]); 983 984 /* Process completed commands. */ 985 for (bit = 0; (bit < 32) && completed; bit++) { 986 if (completed & 0x01) { 987 tag = (group << 5) | bit; 988 989 /* skip internal command slot. */ 990 if (unlikely(tag == MTIP_TAG_INTERNAL)) 991 continue; 992 993 command = &port->commands[tag]; 994 /* make internal callback */ 995 if (likely(command->comp_func)) { 996 command->comp_func( 997 port, 998 tag, 999 command->comp_data, 1000 0); 1001 } else { 1002 dev_warn(&dd->pdev->dev, 1003 "Null completion " 1004 "for tag %d", 1005 tag); 1006 1007 if (mtip_check_surprise_removal( 1008 dd->pdev)) { 1009 mtip_command_cleanup(dd); 1010 return; 1011 } 1012 } 1013 } 1014 completed >>= 1; 1015 } 1016 1017 /* If last, re-enable interrupts */ 1018 if (atomic_dec_return(&dd->irq_workers_active) == 0) 1019 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); 1020 } 1021 1022 /* 1023 * Process legacy pio and d2h interrupts 1024 */ 1025 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) 1026 { 1027 struct mtip_port *port = dd->port; 1028 struct mtip_cmd *cmd = &port->commands[MTIP_TAG_INTERNAL]; 1029 1030 if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && 1031 (cmd != NULL) && !(readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 1032 & (1 << MTIP_TAG_INTERNAL))) { 1033 if (cmd->comp_func) { 1034 cmd->comp_func(port, 1035 MTIP_TAG_INTERNAL, 1036 cmd->comp_data, 1037 0); 1038 return; 1039 } 1040 } 1041 1042 return; 1043 } 1044 1045 /* 1046 * Demux and handle errors 1047 */ 1048 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) 1049 { 1050 if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) 1051 mtip_handle_tfe(dd); 1052 1053 if (unlikely(port_stat & PORT_IRQ_CONNECT)) { 1054 dev_warn(&dd->pdev->dev, 1055 "Clearing PxSERR.DIAG.x\n"); 1056 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); 1057 } 1058 1059 if (unlikely(port_stat & PORT_IRQ_PHYRDY)) { 1060 dev_warn(&dd->pdev->dev, 1061 "Clearing PxSERR.DIAG.n\n"); 1062 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); 1063 } 1064 1065 if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) { 1066 dev_warn(&dd->pdev->dev, 1067 "Port stat errors %x unhandled\n", 1068 (port_stat & ~PORT_IRQ_HANDLED)); 1069 } 1070 } 1071 1072 static inline irqreturn_t mtip_handle_irq(struct driver_data *data) 1073 { 1074 struct driver_data *dd = (struct driver_data *) data; 1075 struct mtip_port *port = dd->port; 1076 u32 hba_stat, port_stat; 1077 int rv = IRQ_NONE; 1078 int do_irq_enable = 1, i, workers; 1079 struct mtip_work *twork; 1080 1081 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); 1082 if (hba_stat) { 1083 rv = IRQ_HANDLED; 1084 1085 /* Acknowledge the interrupt status on the port.*/ 1086 port_stat = readl(port->mmio + PORT_IRQ_STAT); 1087 writel(port_stat, port->mmio + PORT_IRQ_STAT); 1088 1089 /* Demux port status */ 1090 if (likely(port_stat & PORT_IRQ_SDB_FIS)) { 1091 do_irq_enable = 0; 1092 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); 1093 1094 /* Start at 1: group zero is always local? */ 1095 for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS; 1096 i++) { 1097 twork = &dd->work[i]; 1098 twork->completed = readl(port->completed[i]); 1099 if (twork->completed) 1100 workers++; 1101 } 1102 1103 atomic_set(&dd->irq_workers_active, workers); 1104 if (workers) { 1105 for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) { 1106 twork = &dd->work[i]; 1107 if (twork->completed) 1108 queue_work_on( 1109 twork->cpu_binding, 1110 dd->isr_workq, 1111 &twork->work); 1112 } 1113 1114 if (likely(dd->work[0].completed)) 1115 mtip_workq_sdbfx(port, 0, 1116 dd->work[0].completed); 1117 1118 } else { 1119 /* 1120 * Chip quirk: SDB interrupt but nothing 1121 * to complete 1122 */ 1123 do_irq_enable = 1; 1124 } 1125 } 1126 1127 if (unlikely(port_stat & PORT_IRQ_ERR)) { 1128 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 1129 mtip_command_cleanup(dd); 1130 /* don't proceed further */ 1131 return IRQ_HANDLED; 1132 } 1133 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1134 &dd->dd_flag)) 1135 return rv; 1136 1137 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); 1138 } 1139 1140 if (unlikely(port_stat & PORT_IRQ_LEGACY)) 1141 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); 1142 } 1143 1144 /* acknowledge interrupt */ 1145 if (unlikely(do_irq_enable)) 1146 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); 1147 1148 return rv; 1149 } 1150 1151 /* 1152 * HBA interrupt subroutine. 1153 * 1154 * @irq IRQ number. 1155 * @instance Pointer to the driver data structure. 1156 * 1157 * return value 1158 * IRQ_HANDLED A HBA interrupt was pending and handled. 1159 * IRQ_NONE This interrupt was not for the HBA. 1160 */ 1161 static irqreturn_t mtip_irq_handler(int irq, void *instance) 1162 { 1163 struct driver_data *dd = instance; 1164 1165 return mtip_handle_irq(dd); 1166 } 1167 1168 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag) 1169 { 1170 atomic_set(&port->commands[tag].active, 1); 1171 writel(1 << MTIP_TAG_BIT(tag), 1172 port->cmd_issue[MTIP_TAG_INDEX(tag)]); 1173 } 1174 1175 static bool mtip_pause_ncq(struct mtip_port *port, 1176 struct host_to_dev_fis *fis) 1177 { 1178 struct host_to_dev_fis *reply; 1179 unsigned long task_file_data; 1180 1181 reply = port->rxfis + RX_FIS_D2H_REG; 1182 task_file_data = readl(port->mmio+PORT_TFDATA); 1183 1184 if (fis->command == ATA_CMD_SEC_ERASE_UNIT) 1185 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1186 1187 if ((task_file_data & 1)) 1188 return false; 1189 1190 if (fis->command == ATA_CMD_SEC_ERASE_PREP) { 1191 set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 1192 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); 1193 port->ic_pause_timer = jiffies; 1194 return true; 1195 } else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) && 1196 (fis->features == 0x03)) { 1197 set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 1198 port->ic_pause_timer = jiffies; 1199 return true; 1200 } else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) || 1201 ((fis->command == 0xFC) && 1202 (fis->features == 0x27 || fis->features == 0x72 || 1203 fis->features == 0x62 || fis->features == 0x26))) { 1204 /* Com reset after secure erase or lowlevel format */ 1205 mtip_restart_port(port); 1206 return false; 1207 } 1208 1209 return false; 1210 } 1211 1212 /* 1213 * Wait for port to quiesce 1214 * 1215 * @port Pointer to port data structure 1216 * @timeout Max duration to wait (ms) 1217 * 1218 * return value 1219 * 0 Success 1220 * -EBUSY Commands still active 1221 */ 1222 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout) 1223 { 1224 unsigned long to; 1225 unsigned int n; 1226 unsigned int active = 1; 1227 1228 to = jiffies + msecs_to_jiffies(timeout); 1229 do { 1230 if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) && 1231 test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 1232 msleep(20); 1233 continue; /* svc thd is actively issuing commands */ 1234 } 1235 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 1236 return -EFAULT; 1237 /* 1238 * Ignore s_active bit 0 of array element 0. 1239 * This bit will always be set 1240 */ 1241 active = readl(port->s_active[0]) & 0xFFFFFFFE; 1242 for (n = 1; n < port->dd->slot_groups; n++) 1243 active |= readl(port->s_active[n]); 1244 1245 if (!active) 1246 break; 1247 1248 msleep(20); 1249 } while (time_before(jiffies, to)); 1250 1251 return active ? -EBUSY : 0; 1252 } 1253 1254 /* 1255 * Execute an internal command and wait for the completion. 1256 * 1257 * @port Pointer to the port data structure. 1258 * @fis Pointer to the FIS that describes the command. 1259 * @fis_len Length in WORDS of the FIS. 1260 * @buffer DMA accessible for command data. 1261 * @buf_len Length, in bytes, of the data buffer. 1262 * @opts Command header options, excluding the FIS length 1263 * and the number of PRD entries. 1264 * @timeout Time in ms to wait for the command to complete. 1265 * 1266 * return value 1267 * 0 Command completed successfully. 1268 * -EFAULT The buffer address is not correctly aligned. 1269 * -EBUSY Internal command or other IO in progress. 1270 * -EAGAIN Time out waiting for command to complete. 1271 */ 1272 static int mtip_exec_internal_command(struct mtip_port *port, 1273 struct host_to_dev_fis *fis, 1274 int fis_len, 1275 dma_addr_t buffer, 1276 int buf_len, 1277 u32 opts, 1278 gfp_t atomic, 1279 unsigned long timeout) 1280 { 1281 struct mtip_cmd_sg *command_sg; 1282 DECLARE_COMPLETION_ONSTACK(wait); 1283 int rv = 0, ready2go = 1; 1284 struct mtip_cmd *int_cmd = &port->commands[MTIP_TAG_INTERNAL]; 1285 unsigned long to; 1286 1287 /* Make sure the buffer is 8 byte aligned. This is asic specific. */ 1288 if (buffer & 0x00000007) { 1289 dev_err(&port->dd->pdev->dev, 1290 "SG buffer is not 8 byte aligned\n"); 1291 return -EFAULT; 1292 } 1293 1294 to = jiffies + msecs_to_jiffies(timeout); 1295 do { 1296 ready2go = !test_and_set_bit(MTIP_TAG_INTERNAL, 1297 port->allocated); 1298 if (ready2go) 1299 break; 1300 mdelay(100); 1301 } while (time_before(jiffies, to)); 1302 if (!ready2go) { 1303 dev_warn(&port->dd->pdev->dev, 1304 "Internal cmd active. new cmd [%02X]\n", fis->command); 1305 return -EBUSY; 1306 } 1307 set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1308 port->ic_pause_timer = 0; 1309 1310 if (fis->command == ATA_CMD_SEC_ERASE_UNIT) 1311 clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags); 1312 else if (fis->command == ATA_CMD_DOWNLOAD_MICRO) 1313 clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags); 1314 1315 if (atomic == GFP_KERNEL) { 1316 if (fis->command != ATA_CMD_STANDBYNOW1) { 1317 /* wait for io to complete if non atomic */ 1318 if (mtip_quiesce_io(port, 5000) < 0) { 1319 dev_warn(&port->dd->pdev->dev, 1320 "Failed to quiesce IO\n"); 1321 release_slot(port, MTIP_TAG_INTERNAL); 1322 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1323 wake_up_interruptible(&port->svc_wait); 1324 return -EBUSY; 1325 } 1326 } 1327 1328 /* Set the completion function and data for the command. */ 1329 int_cmd->comp_data = &wait; 1330 int_cmd->comp_func = mtip_completion; 1331 1332 } else { 1333 /* Clear completion - we're going to poll */ 1334 int_cmd->comp_data = NULL; 1335 int_cmd->comp_func = mtip_null_completion; 1336 } 1337 1338 /* Copy the command to the command table */ 1339 memcpy(int_cmd->command, fis, fis_len*4); 1340 1341 /* Populate the SG list */ 1342 int_cmd->command_header->opts = 1343 __force_bit2int cpu_to_le32(opts | fis_len); 1344 if (buf_len) { 1345 command_sg = int_cmd->command + AHCI_CMD_TBL_HDR_SZ; 1346 1347 command_sg->info = 1348 __force_bit2int cpu_to_le32((buf_len-1) & 0x3FFFFF); 1349 command_sg->dba = 1350 __force_bit2int cpu_to_le32(buffer & 0xFFFFFFFF); 1351 command_sg->dba_upper = 1352 __force_bit2int cpu_to_le32((buffer >> 16) >> 16); 1353 1354 int_cmd->command_header->opts |= 1355 __force_bit2int cpu_to_le32((1 << 16)); 1356 } 1357 1358 /* Populate the command header */ 1359 int_cmd->command_header->byte_count = 0; 1360 1361 /* Issue the command to the hardware */ 1362 mtip_issue_non_ncq_command(port, MTIP_TAG_INTERNAL); 1363 1364 /* Poll if atomic, wait_for_completion otherwise */ 1365 if (atomic == GFP_KERNEL) { 1366 /* Wait for the command to complete or timeout. */ 1367 if (wait_for_completion_timeout( 1368 &wait, 1369 msecs_to_jiffies(timeout)) == 0) { 1370 dev_err(&port->dd->pdev->dev, 1371 "Internal command did not complete [%d] " 1372 "within timeout of %lu ms\n", 1373 atomic, timeout); 1374 if (mtip_check_surprise_removal(port->dd->pdev) || 1375 test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1376 &port->dd->dd_flag)) { 1377 rv = -ENXIO; 1378 goto exec_ic_exit; 1379 } 1380 rv = -EAGAIN; 1381 } 1382 } else { 1383 /* Spin for <timeout> checking if command still outstanding */ 1384 timeout = jiffies + msecs_to_jiffies(timeout); 1385 while ((readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 1386 & (1 << MTIP_TAG_INTERNAL)) 1387 && time_before(jiffies, timeout)) { 1388 if (mtip_check_surprise_removal(port->dd->pdev)) { 1389 rv = -ENXIO; 1390 goto exec_ic_exit; 1391 } 1392 if ((fis->command != ATA_CMD_STANDBYNOW1) && 1393 test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1394 &port->dd->dd_flag)) { 1395 rv = -ENXIO; 1396 goto exec_ic_exit; 1397 } 1398 if (readl(port->mmio + PORT_IRQ_STAT) & PORT_IRQ_ERR) { 1399 atomic_inc(&int_cmd->active); /* error */ 1400 break; 1401 } 1402 } 1403 } 1404 1405 if (atomic_read(&int_cmd->active) > 1) { 1406 dev_err(&port->dd->pdev->dev, 1407 "Internal command [%02X] failed\n", fis->command); 1408 rv = -EIO; 1409 } 1410 if (readl(port->cmd_issue[MTIP_TAG_INTERNAL]) 1411 & (1 << MTIP_TAG_INTERNAL)) { 1412 rv = -ENXIO; 1413 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 1414 &port->dd->dd_flag)) { 1415 mtip_restart_port(port); 1416 rv = -EAGAIN; 1417 } 1418 } 1419 exec_ic_exit: 1420 /* Clear the allocated and active bits for the internal command. */ 1421 atomic_set(&int_cmd->active, 0); 1422 release_slot(port, MTIP_TAG_INTERNAL); 1423 if (rv >= 0 && mtip_pause_ncq(port, fis)) { 1424 /* NCQ paused */ 1425 return rv; 1426 } 1427 clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags); 1428 wake_up_interruptible(&port->svc_wait); 1429 1430 return rv; 1431 } 1432 1433 /* 1434 * Byte-swap ATA ID strings. 1435 * 1436 * ATA identify data contains strings in byte-swapped 16-bit words. 1437 * They must be swapped (on all architectures) to be usable as C strings. 1438 * This function swaps bytes in-place. 1439 * 1440 * @buf The buffer location of the string 1441 * @len The number of bytes to swap 1442 * 1443 * return value 1444 * None 1445 */ 1446 static inline void ata_swap_string(u16 *buf, unsigned int len) 1447 { 1448 int i; 1449 for (i = 0; i < (len/2); i++) 1450 be16_to_cpus(&buf[i]); 1451 } 1452 1453 /* 1454 * Request the device identity information. 1455 * 1456 * If a user space buffer is not specified, i.e. is NULL, the 1457 * identify information is still read from the drive and placed 1458 * into the identify data buffer (@e port->identify) in the 1459 * port data structure. 1460 * When the identify buffer contains valid identify information @e 1461 * port->identify_valid is non-zero. 1462 * 1463 * @port Pointer to the port structure. 1464 * @user_buffer A user space buffer where the identify data should be 1465 * copied. 1466 * 1467 * return value 1468 * 0 Command completed successfully. 1469 * -EFAULT An error occurred while coping data to the user buffer. 1470 * -1 Command failed. 1471 */ 1472 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer) 1473 { 1474 int rv = 0; 1475 struct host_to_dev_fis fis; 1476 1477 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) 1478 return -EFAULT; 1479 1480 /* Build the FIS. */ 1481 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1482 fis.type = 0x27; 1483 fis.opts = 1 << 7; 1484 fis.command = ATA_CMD_ID_ATA; 1485 1486 /* Set the identify information as invalid. */ 1487 port->identify_valid = 0; 1488 1489 /* Clear the identify information. */ 1490 memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS); 1491 1492 /* Execute the command. */ 1493 if (mtip_exec_internal_command(port, 1494 &fis, 1495 5, 1496 port->identify_dma, 1497 sizeof(u16) * ATA_ID_WORDS, 1498 0, 1499 GFP_KERNEL, 1500 MTIP_INTERNAL_COMMAND_TIMEOUT_MS) 1501 < 0) { 1502 rv = -1; 1503 goto out; 1504 } 1505 1506 /* 1507 * Perform any necessary byte-swapping. Yes, the kernel does in fact 1508 * perform field-sensitive swapping on the string fields. 1509 * See the kernel use of ata_id_string() for proof of this. 1510 */ 1511 #ifdef __LITTLE_ENDIAN 1512 ata_swap_string(port->identify + 27, 40); /* model string*/ 1513 ata_swap_string(port->identify + 23, 8); /* firmware string*/ 1514 ata_swap_string(port->identify + 10, 20); /* serial# string*/ 1515 #else 1516 { 1517 int i; 1518 for (i = 0; i < ATA_ID_WORDS; i++) 1519 port->identify[i] = le16_to_cpu(port->identify[i]); 1520 } 1521 #endif 1522 1523 /* Demux ID.DRAT & ID.RZAT to determine trim support */ 1524 if (port->identify[69] & (1 << 14) && port->identify[69] & (1 << 5)) 1525 port->dd->trim_supp = true; 1526 else 1527 port->dd->trim_supp = false; 1528 1529 /* Set the identify buffer as valid. */ 1530 port->identify_valid = 1; 1531 1532 if (user_buffer) { 1533 if (copy_to_user( 1534 user_buffer, 1535 port->identify, 1536 ATA_ID_WORDS * sizeof(u16))) { 1537 rv = -EFAULT; 1538 goto out; 1539 } 1540 } 1541 1542 out: 1543 return rv; 1544 } 1545 1546 /* 1547 * Issue a standby immediate command to the device. 1548 * 1549 * @port Pointer to the port structure. 1550 * 1551 * return value 1552 * 0 Command was executed successfully. 1553 * -1 An error occurred while executing the command. 1554 */ 1555 static int mtip_standby_immediate(struct mtip_port *port) 1556 { 1557 int rv; 1558 struct host_to_dev_fis fis; 1559 unsigned long start; 1560 1561 /* Build the FIS. */ 1562 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1563 fis.type = 0x27; 1564 fis.opts = 1 << 7; 1565 fis.command = ATA_CMD_STANDBYNOW1; 1566 1567 start = jiffies; 1568 rv = mtip_exec_internal_command(port, 1569 &fis, 1570 5, 1571 0, 1572 0, 1573 0, 1574 GFP_ATOMIC, 1575 15000); 1576 dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n", 1577 jiffies_to_msecs(jiffies - start)); 1578 if (rv) 1579 dev_warn(&port->dd->pdev->dev, 1580 "STANDBY IMMEDIATE command failed.\n"); 1581 1582 return rv; 1583 } 1584 1585 /* 1586 * Issue a READ LOG EXT command to the device. 1587 * 1588 * @port pointer to the port structure. 1589 * @page page number to fetch 1590 * @buffer pointer to buffer 1591 * @buffer_dma dma address corresponding to @buffer 1592 * @sectors page length to fetch, in sectors 1593 * 1594 * return value 1595 * @rv return value from mtip_exec_internal_command() 1596 */ 1597 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer, 1598 dma_addr_t buffer_dma, unsigned int sectors) 1599 { 1600 struct host_to_dev_fis fis; 1601 1602 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1603 fis.type = 0x27; 1604 fis.opts = 1 << 7; 1605 fis.command = ATA_CMD_READ_LOG_EXT; 1606 fis.sect_count = sectors & 0xFF; 1607 fis.sect_cnt_ex = (sectors >> 8) & 0xFF; 1608 fis.lba_low = page; 1609 fis.lba_mid = 0; 1610 fis.device = ATA_DEVICE_OBS; 1611 1612 memset(buffer, 0, sectors * ATA_SECT_SIZE); 1613 1614 return mtip_exec_internal_command(port, 1615 &fis, 1616 5, 1617 buffer_dma, 1618 sectors * ATA_SECT_SIZE, 1619 0, 1620 GFP_ATOMIC, 1621 MTIP_INTERNAL_COMMAND_TIMEOUT_MS); 1622 } 1623 1624 /* 1625 * Issue a SMART READ DATA command to the device. 1626 * 1627 * @port pointer to the port structure. 1628 * @buffer pointer to buffer 1629 * @buffer_dma dma address corresponding to @buffer 1630 * 1631 * return value 1632 * @rv return value from mtip_exec_internal_command() 1633 */ 1634 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer, 1635 dma_addr_t buffer_dma) 1636 { 1637 struct host_to_dev_fis fis; 1638 1639 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1640 fis.type = 0x27; 1641 fis.opts = 1 << 7; 1642 fis.command = ATA_CMD_SMART; 1643 fis.features = 0xD0; 1644 fis.sect_count = 1; 1645 fis.lba_mid = 0x4F; 1646 fis.lba_hi = 0xC2; 1647 fis.device = ATA_DEVICE_OBS; 1648 1649 return mtip_exec_internal_command(port, 1650 &fis, 1651 5, 1652 buffer_dma, 1653 ATA_SECT_SIZE, 1654 0, 1655 GFP_ATOMIC, 1656 15000); 1657 } 1658 1659 /* 1660 * Get the value of a smart attribute 1661 * 1662 * @port pointer to the port structure 1663 * @id attribute number 1664 * @attrib pointer to return attrib information corresponding to @id 1665 * 1666 * return value 1667 * -EINVAL NULL buffer passed or unsupported attribute @id. 1668 * -EPERM Identify data not valid, SMART not supported or not enabled 1669 */ 1670 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id, 1671 struct smart_attr *attrib) 1672 { 1673 int rv, i; 1674 struct smart_attr *pattr; 1675 1676 if (!attrib) 1677 return -EINVAL; 1678 1679 if (!port->identify_valid) { 1680 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); 1681 return -EPERM; 1682 } 1683 if (!(port->identify[82] & 0x1)) { 1684 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); 1685 return -EPERM; 1686 } 1687 if (!(port->identify[85] & 0x1)) { 1688 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); 1689 return -EPERM; 1690 } 1691 1692 memset(port->smart_buf, 0, ATA_SECT_SIZE); 1693 rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma); 1694 if (rv) { 1695 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); 1696 return rv; 1697 } 1698 1699 pattr = (struct smart_attr *)(port->smart_buf + 2); 1700 for (i = 0; i < 29; i++, pattr++) 1701 if (pattr->attr_id == id) { 1702 memcpy(attrib, pattr, sizeof(struct smart_attr)); 1703 break; 1704 } 1705 1706 if (i == 29) { 1707 dev_warn(&port->dd->pdev->dev, 1708 "Query for invalid SMART attribute ID\n"); 1709 rv = -EINVAL; 1710 } 1711 1712 return rv; 1713 } 1714 1715 /* 1716 * Trim unused sectors 1717 * 1718 * @dd pointer to driver_data structure 1719 * @lba starting lba 1720 * @len # of 512b sectors to trim 1721 * 1722 * return value 1723 * -ENOMEM Out of dma memory 1724 * -EINVAL Invalid parameters passed in, trim not supported 1725 * -EIO Error submitting trim request to hw 1726 */ 1727 static int mtip_send_trim(struct driver_data *dd, unsigned int lba, unsigned int len) 1728 { 1729 int i, rv = 0; 1730 u64 tlba, tlen, sect_left; 1731 struct mtip_trim_entry *buf; 1732 dma_addr_t dma_addr; 1733 struct host_to_dev_fis fis; 1734 1735 if (!len || dd->trim_supp == false) 1736 return -EINVAL; 1737 1738 /* Trim request too big */ 1739 WARN_ON(len > (MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES)); 1740 1741 /* Trim request not aligned on 4k boundary */ 1742 WARN_ON(len % 8 != 0); 1743 1744 /* Warn if vu_trim structure is too big */ 1745 WARN_ON(sizeof(struct mtip_trim) > ATA_SECT_SIZE); 1746 1747 /* Allocate a DMA buffer for the trim structure */ 1748 buf = dmam_alloc_coherent(&dd->pdev->dev, ATA_SECT_SIZE, &dma_addr, 1749 GFP_KERNEL); 1750 if (!buf) 1751 return -ENOMEM; 1752 memset(buf, 0, ATA_SECT_SIZE); 1753 1754 for (i = 0, sect_left = len, tlba = lba; 1755 i < MTIP_MAX_TRIM_ENTRIES && sect_left; 1756 i++) { 1757 tlen = (sect_left >= MTIP_MAX_TRIM_ENTRY_LEN ? 1758 MTIP_MAX_TRIM_ENTRY_LEN : 1759 sect_left); 1760 buf[i].lba = __force_bit2int cpu_to_le32(tlba); 1761 buf[i].range = __force_bit2int cpu_to_le16(tlen); 1762 tlba += tlen; 1763 sect_left -= tlen; 1764 } 1765 WARN_ON(sect_left != 0); 1766 1767 /* Build the fis */ 1768 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1769 fis.type = 0x27; 1770 fis.opts = 1 << 7; 1771 fis.command = 0xfb; 1772 fis.features = 0x60; 1773 fis.sect_count = 1; 1774 fis.device = ATA_DEVICE_OBS; 1775 1776 if (mtip_exec_internal_command(dd->port, 1777 &fis, 1778 5, 1779 dma_addr, 1780 ATA_SECT_SIZE, 1781 0, 1782 GFP_KERNEL, 1783 MTIP_TRIM_TIMEOUT_MS) < 0) 1784 rv = -EIO; 1785 1786 dmam_free_coherent(&dd->pdev->dev, ATA_SECT_SIZE, buf, dma_addr); 1787 return rv; 1788 } 1789 1790 /* 1791 * Get the drive capacity. 1792 * 1793 * @dd Pointer to the device data structure. 1794 * @sectors Pointer to the variable that will receive the sector count. 1795 * 1796 * return value 1797 * 1 Capacity was returned successfully. 1798 * 0 The identify information is invalid. 1799 */ 1800 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) 1801 { 1802 struct mtip_port *port = dd->port; 1803 u64 total, raw0, raw1, raw2, raw3; 1804 raw0 = port->identify[100]; 1805 raw1 = port->identify[101]; 1806 raw2 = port->identify[102]; 1807 raw3 = port->identify[103]; 1808 total = raw0 | raw1<<16 | raw2<<32 | raw3<<48; 1809 *sectors = total; 1810 return (bool) !!port->identify_valid; 1811 } 1812 1813 /* 1814 * Reset the HBA. 1815 * 1816 * Resets the HBA by setting the HBA Reset bit in the Global 1817 * HBA Control register. After setting the HBA Reset bit the 1818 * function waits for 1 second before reading the HBA Reset 1819 * bit to make sure it has cleared. If HBA Reset is not clear 1820 * an error is returned. Cannot be used in non-blockable 1821 * context. 1822 * 1823 * @dd Pointer to the driver data structure. 1824 * 1825 * return value 1826 * 0 The reset was successful. 1827 * -1 The HBA Reset bit did not clear. 1828 */ 1829 static int mtip_hba_reset(struct driver_data *dd) 1830 { 1831 mtip_deinit_port(dd->port); 1832 1833 /* Set the reset bit */ 1834 writel(HOST_RESET, dd->mmio + HOST_CTL); 1835 1836 /* Flush */ 1837 readl(dd->mmio + HOST_CTL); 1838 1839 /* Wait for reset to clear */ 1840 ssleep(1); 1841 1842 /* Check the bit has cleared */ 1843 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) { 1844 dev_err(&dd->pdev->dev, 1845 "Reset bit did not clear.\n"); 1846 return -1; 1847 } 1848 1849 return 0; 1850 } 1851 1852 /* 1853 * Display the identify command data. 1854 * 1855 * @port Pointer to the port data structure. 1856 * 1857 * return value 1858 * None 1859 */ 1860 static void mtip_dump_identify(struct mtip_port *port) 1861 { 1862 sector_t sectors; 1863 unsigned short revid; 1864 char cbuf[42]; 1865 1866 if (!port->identify_valid) 1867 return; 1868 1869 strlcpy(cbuf, (char *)(port->identify+10), 21); 1870 dev_info(&port->dd->pdev->dev, 1871 "Serial No.: %s\n", cbuf); 1872 1873 strlcpy(cbuf, (char *)(port->identify+23), 9); 1874 dev_info(&port->dd->pdev->dev, 1875 "Firmware Ver.: %s\n", cbuf); 1876 1877 strlcpy(cbuf, (char *)(port->identify+27), 41); 1878 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); 1879 1880 if (mtip_hw_get_capacity(port->dd, §ors)) 1881 dev_info(&port->dd->pdev->dev, 1882 "Capacity: %llu sectors (%llu MB)\n", 1883 (u64)sectors, 1884 ((u64)sectors) * ATA_SECT_SIZE >> 20); 1885 1886 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); 1887 switch (revid & 0xFF) { 1888 case 0x1: 1889 strlcpy(cbuf, "A0", 3); 1890 break; 1891 case 0x3: 1892 strlcpy(cbuf, "A2", 3); 1893 break; 1894 default: 1895 strlcpy(cbuf, "?", 2); 1896 break; 1897 } 1898 dev_info(&port->dd->pdev->dev, 1899 "Card Type: %s\n", cbuf); 1900 } 1901 1902 /* 1903 * Map the commands scatter list into the command table. 1904 * 1905 * @command Pointer to the command. 1906 * @nents Number of scatter list entries. 1907 * 1908 * return value 1909 * None 1910 */ 1911 static inline void fill_command_sg(struct driver_data *dd, 1912 struct mtip_cmd *command, 1913 int nents) 1914 { 1915 int n; 1916 unsigned int dma_len; 1917 struct mtip_cmd_sg *command_sg; 1918 struct scatterlist *sg = command->sg; 1919 1920 command_sg = command->command + AHCI_CMD_TBL_HDR_SZ; 1921 1922 for (n = 0; n < nents; n++) { 1923 dma_len = sg_dma_len(sg); 1924 if (dma_len > 0x400000) 1925 dev_err(&dd->pdev->dev, 1926 "DMA segment length truncated\n"); 1927 command_sg->info = __force_bit2int 1928 cpu_to_le32((dma_len-1) & 0x3FFFFF); 1929 command_sg->dba = __force_bit2int 1930 cpu_to_le32(sg_dma_address(sg)); 1931 command_sg->dba_upper = __force_bit2int 1932 cpu_to_le32((sg_dma_address(sg) >> 16) >> 16); 1933 command_sg++; 1934 sg++; 1935 } 1936 } 1937 1938 /* 1939 * @brief Execute a drive command. 1940 * 1941 * return value 0 The command completed successfully. 1942 * return value -1 An error occurred while executing the command. 1943 */ 1944 static int exec_drive_task(struct mtip_port *port, u8 *command) 1945 { 1946 struct host_to_dev_fis fis; 1947 struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG); 1948 1949 /* Build the FIS. */ 1950 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 1951 fis.type = 0x27; 1952 fis.opts = 1 << 7; 1953 fis.command = command[0]; 1954 fis.features = command[1]; 1955 fis.sect_count = command[2]; 1956 fis.sector = command[3]; 1957 fis.cyl_low = command[4]; 1958 fis.cyl_hi = command[5]; 1959 fis.device = command[6] & ~0x10; /* Clear the dev bit*/ 1960 1961 dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n", 1962 __func__, 1963 command[0], 1964 command[1], 1965 command[2], 1966 command[3], 1967 command[4], 1968 command[5], 1969 command[6]); 1970 1971 /* Execute the command. */ 1972 if (mtip_exec_internal_command(port, 1973 &fis, 1974 5, 1975 0, 1976 0, 1977 0, 1978 GFP_KERNEL, 1979 MTIP_IOCTL_COMMAND_TIMEOUT_MS) < 0) { 1980 return -1; 1981 } 1982 1983 command[0] = reply->command; /* Status*/ 1984 command[1] = reply->features; /* Error*/ 1985 command[4] = reply->cyl_low; 1986 command[5] = reply->cyl_hi; 1987 1988 dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n", 1989 __func__, 1990 command[0], 1991 command[1], 1992 command[4], 1993 command[5]); 1994 1995 return 0; 1996 } 1997 1998 /* 1999 * @brief Execute a drive command. 2000 * 2001 * @param port Pointer to the port data structure. 2002 * @param command Pointer to the user specified command parameters. 2003 * @param user_buffer Pointer to the user space buffer where read sector 2004 * data should be copied. 2005 * 2006 * return value 0 The command completed successfully. 2007 * return value -EFAULT An error occurred while copying the completion 2008 * data to the user space buffer. 2009 * return value -1 An error occurred while executing the command. 2010 */ 2011 static int exec_drive_command(struct mtip_port *port, u8 *command, 2012 void __user *user_buffer) 2013 { 2014 struct host_to_dev_fis fis; 2015 struct host_to_dev_fis *reply; 2016 u8 *buf = NULL; 2017 dma_addr_t dma_addr = 0; 2018 int rv = 0, xfer_sz = command[3]; 2019 2020 if (xfer_sz) { 2021 if (!user_buffer) 2022 return -EFAULT; 2023 2024 buf = dmam_alloc_coherent(&port->dd->pdev->dev, 2025 ATA_SECT_SIZE * xfer_sz, 2026 &dma_addr, 2027 GFP_KERNEL); 2028 if (!buf) { 2029 dev_err(&port->dd->pdev->dev, 2030 "Memory allocation failed (%d bytes)\n", 2031 ATA_SECT_SIZE * xfer_sz); 2032 return -ENOMEM; 2033 } 2034 memset(buf, 0, ATA_SECT_SIZE * xfer_sz); 2035 } 2036 2037 /* Build the FIS. */ 2038 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 2039 fis.type = 0x27; 2040 fis.opts = 1 << 7; 2041 fis.command = command[0]; 2042 fis.features = command[2]; 2043 fis.sect_count = command[3]; 2044 if (fis.command == ATA_CMD_SMART) { 2045 fis.sector = command[1]; 2046 fis.cyl_low = 0x4F; 2047 fis.cyl_hi = 0xC2; 2048 } 2049 2050 if (xfer_sz) 2051 reply = (port->rxfis + RX_FIS_PIO_SETUP); 2052 else 2053 reply = (port->rxfis + RX_FIS_D2H_REG); 2054 2055 dbg_printk(MTIP_DRV_NAME 2056 " %s: User Command: cmd %x, sect %x, " 2057 "feat %x, sectcnt %x\n", 2058 __func__, 2059 command[0], 2060 command[1], 2061 command[2], 2062 command[3]); 2063 2064 /* Execute the command. */ 2065 if (mtip_exec_internal_command(port, 2066 &fis, 2067 5, 2068 (xfer_sz ? dma_addr : 0), 2069 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0), 2070 0, 2071 GFP_KERNEL, 2072 MTIP_IOCTL_COMMAND_TIMEOUT_MS) 2073 < 0) { 2074 rv = -EFAULT; 2075 goto exit_drive_command; 2076 } 2077 2078 /* Collect the completion status. */ 2079 command[0] = reply->command; /* Status*/ 2080 command[1] = reply->features; /* Error*/ 2081 command[2] = reply->sect_count; 2082 2083 dbg_printk(MTIP_DRV_NAME 2084 " %s: Completion Status: stat %x, " 2085 "err %x, nsect %x\n", 2086 __func__, 2087 command[0], 2088 command[1], 2089 command[2]); 2090 2091 if (xfer_sz) { 2092 if (copy_to_user(user_buffer, 2093 buf, 2094 ATA_SECT_SIZE * command[3])) { 2095 rv = -EFAULT; 2096 goto exit_drive_command; 2097 } 2098 } 2099 exit_drive_command: 2100 if (buf) 2101 dmam_free_coherent(&port->dd->pdev->dev, 2102 ATA_SECT_SIZE * xfer_sz, buf, dma_addr); 2103 return rv; 2104 } 2105 2106 /* 2107 * Indicates whether a command has a single sector payload. 2108 * 2109 * @command passed to the device to perform the certain event. 2110 * @features passed to the device to perform the certain event. 2111 * 2112 * return value 2113 * 1 command is one that always has a single sector payload, 2114 * regardless of the value in the Sector Count field. 2115 * 0 otherwise 2116 * 2117 */ 2118 static unsigned int implicit_sector(unsigned char command, 2119 unsigned char features) 2120 { 2121 unsigned int rv = 0; 2122 2123 /* list of commands that have an implicit sector count of 1 */ 2124 switch (command) { 2125 case ATA_CMD_SEC_SET_PASS: 2126 case ATA_CMD_SEC_UNLOCK: 2127 case ATA_CMD_SEC_ERASE_PREP: 2128 case ATA_CMD_SEC_ERASE_UNIT: 2129 case ATA_CMD_SEC_FREEZE_LOCK: 2130 case ATA_CMD_SEC_DISABLE_PASS: 2131 case ATA_CMD_PMP_READ: 2132 case ATA_CMD_PMP_WRITE: 2133 rv = 1; 2134 break; 2135 case ATA_CMD_SET_MAX: 2136 if (features == ATA_SET_MAX_UNLOCK) 2137 rv = 1; 2138 break; 2139 case ATA_CMD_SMART: 2140 if ((features == ATA_SMART_READ_VALUES) || 2141 (features == ATA_SMART_READ_THRESHOLDS)) 2142 rv = 1; 2143 break; 2144 case ATA_CMD_CONF_OVERLAY: 2145 if ((features == ATA_DCO_IDENTIFY) || 2146 (features == ATA_DCO_SET)) 2147 rv = 1; 2148 break; 2149 } 2150 return rv; 2151 } 2152 static void mtip_set_timeout(struct driver_data *dd, 2153 struct host_to_dev_fis *fis, 2154 unsigned int *timeout, u8 erasemode) 2155 { 2156 switch (fis->command) { 2157 case ATA_CMD_DOWNLOAD_MICRO: 2158 *timeout = 120000; /* 2 minutes */ 2159 break; 2160 case ATA_CMD_SEC_ERASE_UNIT: 2161 case 0xFC: 2162 if (erasemode) 2163 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); 2164 else 2165 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); 2166 break; 2167 case ATA_CMD_STANDBYNOW1: 2168 *timeout = 120000; /* 2 minutes */ 2169 break; 2170 case 0xF7: 2171 case 0xFA: 2172 *timeout = 60000; /* 60 seconds */ 2173 break; 2174 case ATA_CMD_SMART: 2175 *timeout = 15000; /* 15 seconds */ 2176 break; 2177 default: 2178 *timeout = MTIP_IOCTL_COMMAND_TIMEOUT_MS; 2179 break; 2180 } 2181 } 2182 2183 /* 2184 * Executes a taskfile 2185 * See ide_taskfile_ioctl() for derivation 2186 */ 2187 static int exec_drive_taskfile(struct driver_data *dd, 2188 void __user *buf, 2189 ide_task_request_t *req_task, 2190 int outtotal) 2191 { 2192 struct host_to_dev_fis fis; 2193 struct host_to_dev_fis *reply; 2194 u8 *outbuf = NULL; 2195 u8 *inbuf = NULL; 2196 dma_addr_t outbuf_dma = 0; 2197 dma_addr_t inbuf_dma = 0; 2198 dma_addr_t dma_buffer = 0; 2199 int err = 0; 2200 unsigned int taskin = 0; 2201 unsigned int taskout = 0; 2202 u8 nsect = 0; 2203 unsigned int timeout; 2204 unsigned int force_single_sector; 2205 unsigned int transfer_size; 2206 unsigned long task_file_data; 2207 int intotal = outtotal + req_task->out_size; 2208 int erasemode = 0; 2209 2210 taskout = req_task->out_size; 2211 taskin = req_task->in_size; 2212 /* 130560 = 512 * 0xFF*/ 2213 if (taskin > 130560 || taskout > 130560) { 2214 err = -EINVAL; 2215 goto abort; 2216 } 2217 2218 if (taskout) { 2219 outbuf = kzalloc(taskout, GFP_KERNEL); 2220 if (outbuf == NULL) { 2221 err = -ENOMEM; 2222 goto abort; 2223 } 2224 if (copy_from_user(outbuf, buf + outtotal, taskout)) { 2225 err = -EFAULT; 2226 goto abort; 2227 } 2228 outbuf_dma = pci_map_single(dd->pdev, 2229 outbuf, 2230 taskout, 2231 DMA_TO_DEVICE); 2232 if (outbuf_dma == 0) { 2233 err = -ENOMEM; 2234 goto abort; 2235 } 2236 dma_buffer = outbuf_dma; 2237 } 2238 2239 if (taskin) { 2240 inbuf = kzalloc(taskin, GFP_KERNEL); 2241 if (inbuf == NULL) { 2242 err = -ENOMEM; 2243 goto abort; 2244 } 2245 2246 if (copy_from_user(inbuf, buf + intotal, taskin)) { 2247 err = -EFAULT; 2248 goto abort; 2249 } 2250 inbuf_dma = pci_map_single(dd->pdev, 2251 inbuf, 2252 taskin, DMA_FROM_DEVICE); 2253 if (inbuf_dma == 0) { 2254 err = -ENOMEM; 2255 goto abort; 2256 } 2257 dma_buffer = inbuf_dma; 2258 } 2259 2260 /* only supports PIO and non-data commands from this ioctl. */ 2261 switch (req_task->data_phase) { 2262 case TASKFILE_OUT: 2263 nsect = taskout / ATA_SECT_SIZE; 2264 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 2265 break; 2266 case TASKFILE_IN: 2267 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); 2268 break; 2269 case TASKFILE_NO_DATA: 2270 reply = (dd->port->rxfis + RX_FIS_D2H_REG); 2271 break; 2272 default: 2273 err = -EINVAL; 2274 goto abort; 2275 } 2276 2277 /* Build the FIS. */ 2278 memset(&fis, 0, sizeof(struct host_to_dev_fis)); 2279 2280 fis.type = 0x27; 2281 fis.opts = 1 << 7; 2282 fis.command = req_task->io_ports[7]; 2283 fis.features = req_task->io_ports[1]; 2284 fis.sect_count = req_task->io_ports[2]; 2285 fis.lba_low = req_task->io_ports[3]; 2286 fis.lba_mid = req_task->io_ports[4]; 2287 fis.lba_hi = req_task->io_ports[5]; 2288 /* Clear the dev bit*/ 2289 fis.device = req_task->io_ports[6] & ~0x10; 2290 2291 if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) { 2292 req_task->in_flags.all = 2293 IDE_TASKFILE_STD_IN_FLAGS | 2294 (IDE_HOB_STD_IN_FLAGS << 8); 2295 fis.lba_low_ex = req_task->hob_ports[3]; 2296 fis.lba_mid_ex = req_task->hob_ports[4]; 2297 fis.lba_hi_ex = req_task->hob_ports[5]; 2298 fis.features_ex = req_task->hob_ports[1]; 2299 fis.sect_cnt_ex = req_task->hob_ports[2]; 2300 2301 } else { 2302 req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS; 2303 } 2304 2305 force_single_sector = implicit_sector(fis.command, fis.features); 2306 2307 if ((taskin || taskout) && (!fis.sect_count)) { 2308 if (nsect) 2309 fis.sect_count = nsect; 2310 else { 2311 if (!force_single_sector) { 2312 dev_warn(&dd->pdev->dev, 2313 "data movement but " 2314 "sect_count is 0\n"); 2315 err = -EINVAL; 2316 goto abort; 2317 } 2318 } 2319 } 2320 2321 dbg_printk(MTIP_DRV_NAME 2322 " %s: cmd %x, feat %x, nsect %x," 2323 " sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x," 2324 " head/dev %x\n", 2325 __func__, 2326 fis.command, 2327 fis.features, 2328 fis.sect_count, 2329 fis.lba_low, 2330 fis.lba_mid, 2331 fis.lba_hi, 2332 fis.device); 2333 2334 /* check for erase mode support during secure erase.*/ 2335 if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf && 2336 (outbuf[0] & MTIP_SEC_ERASE_MODE)) { 2337 erasemode = 1; 2338 } 2339 2340 mtip_set_timeout(dd, &fis, &timeout, erasemode); 2341 2342 /* Determine the correct transfer size.*/ 2343 if (force_single_sector) 2344 transfer_size = ATA_SECT_SIZE; 2345 else 2346 transfer_size = ATA_SECT_SIZE * fis.sect_count; 2347 2348 /* Execute the command.*/ 2349 if (mtip_exec_internal_command(dd->port, 2350 &fis, 2351 5, 2352 dma_buffer, 2353 transfer_size, 2354 0, 2355 GFP_KERNEL, 2356 timeout) < 0) { 2357 err = -EIO; 2358 goto abort; 2359 } 2360 2361 task_file_data = readl(dd->port->mmio+PORT_TFDATA); 2362 2363 if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) { 2364 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; 2365 req_task->io_ports[7] = reply->control; 2366 } else { 2367 reply = dd->port->rxfis + RX_FIS_D2H_REG; 2368 req_task->io_ports[7] = reply->command; 2369 } 2370 2371 /* reclaim the DMA buffers.*/ 2372 if (inbuf_dma) 2373 pci_unmap_single(dd->pdev, inbuf_dma, 2374 taskin, DMA_FROM_DEVICE); 2375 if (outbuf_dma) 2376 pci_unmap_single(dd->pdev, outbuf_dma, 2377 taskout, DMA_TO_DEVICE); 2378 inbuf_dma = 0; 2379 outbuf_dma = 0; 2380 2381 /* return the ATA registers to the caller.*/ 2382 req_task->io_ports[1] = reply->features; 2383 req_task->io_ports[2] = reply->sect_count; 2384 req_task->io_ports[3] = reply->lba_low; 2385 req_task->io_ports[4] = reply->lba_mid; 2386 req_task->io_ports[5] = reply->lba_hi; 2387 req_task->io_ports[6] = reply->device; 2388 2389 if (req_task->out_flags.all & 1) { 2390 2391 req_task->hob_ports[3] = reply->lba_low_ex; 2392 req_task->hob_ports[4] = reply->lba_mid_ex; 2393 req_task->hob_ports[5] = reply->lba_hi_ex; 2394 req_task->hob_ports[1] = reply->features_ex; 2395 req_task->hob_ports[2] = reply->sect_cnt_ex; 2396 } 2397 dbg_printk(MTIP_DRV_NAME 2398 " %s: Completion: stat %x," 2399 "err %x, sect_cnt %x, lbalo %x," 2400 "lbamid %x, lbahi %x, dev %x\n", 2401 __func__, 2402 req_task->io_ports[7], 2403 req_task->io_ports[1], 2404 req_task->io_ports[2], 2405 req_task->io_ports[3], 2406 req_task->io_ports[4], 2407 req_task->io_ports[5], 2408 req_task->io_ports[6]); 2409 2410 if (taskout) { 2411 if (copy_to_user(buf + outtotal, outbuf, taskout)) { 2412 err = -EFAULT; 2413 goto abort; 2414 } 2415 } 2416 if (taskin) { 2417 if (copy_to_user(buf + intotal, inbuf, taskin)) { 2418 err = -EFAULT; 2419 goto abort; 2420 } 2421 } 2422 abort: 2423 if (inbuf_dma) 2424 pci_unmap_single(dd->pdev, inbuf_dma, 2425 taskin, DMA_FROM_DEVICE); 2426 if (outbuf_dma) 2427 pci_unmap_single(dd->pdev, outbuf_dma, 2428 taskout, DMA_TO_DEVICE); 2429 kfree(outbuf); 2430 kfree(inbuf); 2431 2432 return err; 2433 } 2434 2435 /* 2436 * Handle IOCTL calls from the Block Layer. 2437 * 2438 * This function is called by the Block Layer when it receives an IOCTL 2439 * command that it does not understand. If the IOCTL command is not supported 2440 * this function returns -ENOTTY. 2441 * 2442 * @dd Pointer to the driver data structure. 2443 * @cmd IOCTL command passed from the Block Layer. 2444 * @arg IOCTL argument passed from the Block Layer. 2445 * 2446 * return value 2447 * 0 The IOCTL completed successfully. 2448 * -ENOTTY The specified command is not supported. 2449 * -EFAULT An error occurred copying data to a user space buffer. 2450 * -EIO An error occurred while executing the command. 2451 */ 2452 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, 2453 unsigned long arg) 2454 { 2455 switch (cmd) { 2456 case HDIO_GET_IDENTITY: 2457 { 2458 if (copy_to_user((void __user *)arg, dd->port->identify, 2459 sizeof(u16) * ATA_ID_WORDS)) 2460 return -EFAULT; 2461 break; 2462 } 2463 case HDIO_DRIVE_CMD: 2464 { 2465 u8 drive_command[4]; 2466 2467 /* Copy the user command info to our buffer. */ 2468 if (copy_from_user(drive_command, 2469 (void __user *) arg, 2470 sizeof(drive_command))) 2471 return -EFAULT; 2472 2473 /* Execute the drive command. */ 2474 if (exec_drive_command(dd->port, 2475 drive_command, 2476 (void __user *) (arg+4))) 2477 return -EIO; 2478 2479 /* Copy the status back to the users buffer. */ 2480 if (copy_to_user((void __user *) arg, 2481 drive_command, 2482 sizeof(drive_command))) 2483 return -EFAULT; 2484 2485 break; 2486 } 2487 case HDIO_DRIVE_TASK: 2488 { 2489 u8 drive_command[7]; 2490 2491 /* Copy the user command info to our buffer. */ 2492 if (copy_from_user(drive_command, 2493 (void __user *) arg, 2494 sizeof(drive_command))) 2495 return -EFAULT; 2496 2497 /* Execute the drive command. */ 2498 if (exec_drive_task(dd->port, drive_command)) 2499 return -EIO; 2500 2501 /* Copy the status back to the users buffer. */ 2502 if (copy_to_user((void __user *) arg, 2503 drive_command, 2504 sizeof(drive_command))) 2505 return -EFAULT; 2506 2507 break; 2508 } 2509 case HDIO_DRIVE_TASKFILE: { 2510 ide_task_request_t req_task; 2511 int ret, outtotal; 2512 2513 if (copy_from_user(&req_task, (void __user *) arg, 2514 sizeof(req_task))) 2515 return -EFAULT; 2516 2517 outtotal = sizeof(req_task); 2518 2519 ret = exec_drive_taskfile(dd, (void __user *) arg, 2520 &req_task, outtotal); 2521 2522 if (copy_to_user((void __user *) arg, &req_task, 2523 sizeof(req_task))) 2524 return -EFAULT; 2525 2526 return ret; 2527 } 2528 2529 default: 2530 return -EINVAL; 2531 } 2532 return 0; 2533 } 2534 2535 /* 2536 * Submit an IO to the hw 2537 * 2538 * This function is called by the block layer to issue an io 2539 * to the device. Upon completion, the callback function will 2540 * be called with the data parameter passed as the callback data. 2541 * 2542 * @dd Pointer to the driver data structure. 2543 * @start First sector to read. 2544 * @nsect Number of sectors to read. 2545 * @nents Number of entries in scatter list for the read command. 2546 * @tag The tag of this read command. 2547 * @callback Pointer to the function that should be called 2548 * when the read completes. 2549 * @data Callback data passed to the callback function 2550 * when the read completes. 2551 * @dir Direction (read or write) 2552 * 2553 * return value 2554 * None 2555 */ 2556 static void mtip_hw_submit_io(struct driver_data *dd, sector_t sector, 2557 int nsect, int nents, int tag, void *callback, 2558 void *data, int dir) 2559 { 2560 struct host_to_dev_fis *fis; 2561 struct mtip_port *port = dd->port; 2562 struct mtip_cmd *command = &port->commands[tag]; 2563 int dma_dir = (dir == READ) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 2564 u64 start = sector; 2565 2566 /* Map the scatter list for DMA access */ 2567 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); 2568 2569 command->scatter_ents = nents; 2570 2571 /* 2572 * The number of retries for this command before it is 2573 * reported as a failure to the upper layers. 2574 */ 2575 command->retries = MTIP_MAX_RETRIES; 2576 2577 /* Fill out fis */ 2578 fis = command->command; 2579 fis->type = 0x27; 2580 fis->opts = 1 << 7; 2581 fis->command = 2582 (dir == READ ? ATA_CMD_FPDMA_READ : ATA_CMD_FPDMA_WRITE); 2583 fis->lba_low = start & 0xFF; 2584 fis->lba_mid = (start >> 8) & 0xFF; 2585 fis->lba_hi = (start >> 16) & 0xFF; 2586 fis->lba_low_ex = (start >> 24) & 0xFF; 2587 fis->lba_mid_ex = (start >> 32) & 0xFF; 2588 fis->lba_hi_ex = (start >> 40) & 0xFF; 2589 fis->device = 1 << 6; 2590 fis->features = nsect & 0xFF; 2591 fis->features_ex = (nsect >> 8) & 0xFF; 2592 fis->sect_count = ((tag << 3) | (tag >> 5)); 2593 fis->sect_cnt_ex = 0; 2594 fis->control = 0; 2595 fis->res2 = 0; 2596 fis->res3 = 0; 2597 fill_command_sg(dd, command, nents); 2598 2599 /* Populate the command header */ 2600 command->command_header->opts = 2601 __force_bit2int cpu_to_le32( 2602 (nents << 16) | 5 | AHCI_CMD_PREFETCH); 2603 command->command_header->byte_count = 0; 2604 2605 /* 2606 * Set the completion function and data for the command 2607 * within this layer. 2608 */ 2609 command->comp_data = dd; 2610 command->comp_func = mtip_async_complete; 2611 command->direction = dma_dir; 2612 2613 /* 2614 * Set the completion function and data for the command passed 2615 * from the upper layer. 2616 */ 2617 command->async_data = data; 2618 command->async_callback = callback; 2619 2620 /* 2621 * To prevent this command from being issued 2622 * if an internal command is in progress or error handling is active. 2623 */ 2624 if (port->flags & MTIP_PF_PAUSE_IO) { 2625 set_bit(tag, port->cmds_to_issue); 2626 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 2627 return; 2628 } 2629 2630 /* Issue the command to the hardware */ 2631 mtip_issue_ncq_command(port, tag); 2632 2633 return; 2634 } 2635 2636 /* 2637 * Release a command slot. 2638 * 2639 * @dd Pointer to the driver data structure. 2640 * @tag Slot tag 2641 * 2642 * return value 2643 * None 2644 */ 2645 static void mtip_hw_release_scatterlist(struct driver_data *dd, int tag) 2646 { 2647 release_slot(dd->port, tag); 2648 } 2649 2650 /* 2651 * Obtain a command slot and return its associated scatter list. 2652 * 2653 * @dd Pointer to the driver data structure. 2654 * @tag Pointer to an int that will receive the allocated command 2655 * slot tag. 2656 * 2657 * return value 2658 * Pointer to the scatter list for the allocated command slot 2659 * or NULL if no command slots are available. 2660 */ 2661 static struct scatterlist *mtip_hw_get_scatterlist(struct driver_data *dd, 2662 int *tag) 2663 { 2664 /* 2665 * It is possible that, even with this semaphore, a thread 2666 * may think that no command slots are available. Therefore, we 2667 * need to make an attempt to get_slot(). 2668 */ 2669 down(&dd->port->cmd_slot); 2670 *tag = get_slot(dd->port); 2671 2672 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { 2673 up(&dd->port->cmd_slot); 2674 return NULL; 2675 } 2676 if (unlikely(*tag < 0)) { 2677 up(&dd->port->cmd_slot); 2678 return NULL; 2679 } 2680 2681 return dd->port->commands[*tag].sg; 2682 } 2683 2684 /* 2685 * Sysfs status dump. 2686 * 2687 * @dev Pointer to the device structure, passed by the kernrel. 2688 * @attr Pointer to the device_attribute structure passed by the kernel. 2689 * @buf Pointer to the char buffer that will receive the stats info. 2690 * 2691 * return value 2692 * The size, in bytes, of the data copied into buf. 2693 */ 2694 static ssize_t mtip_hw_show_status(struct device *dev, 2695 struct device_attribute *attr, 2696 char *buf) 2697 { 2698 struct driver_data *dd = dev_to_disk(dev)->private_data; 2699 int size = 0; 2700 2701 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) 2702 size += sprintf(buf, "%s", "thermal_shutdown\n"); 2703 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) 2704 size += sprintf(buf, "%s", "write_protect\n"); 2705 else 2706 size += sprintf(buf, "%s", "online\n"); 2707 2708 return size; 2709 } 2710 2711 static DEVICE_ATTR(status, S_IRUGO, mtip_hw_show_status, NULL); 2712 2713 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf, 2714 size_t len, loff_t *offset) 2715 { 2716 struct driver_data *dd = (struct driver_data *)f->private_data; 2717 char buf[MTIP_DFS_MAX_BUF_SIZE]; 2718 u32 group_allocated; 2719 int size = *offset; 2720 int n; 2721 2722 if (!len || size) 2723 return 0; 2724 2725 size += sprintf(&buf[size], "H/ S ACTive : [ 0x"); 2726 2727 for (n = dd->slot_groups-1; n >= 0; n--) 2728 size += sprintf(&buf[size], "%08X ", 2729 readl(dd->port->s_active[n])); 2730 2731 size += sprintf(&buf[size], "]\n"); 2732 size += sprintf(&buf[size], "H/ Command Issue : [ 0x"); 2733 2734 for (n = dd->slot_groups-1; n >= 0; n--) 2735 size += sprintf(&buf[size], "%08X ", 2736 readl(dd->port->cmd_issue[n])); 2737 2738 size += sprintf(&buf[size], "]\n"); 2739 size += sprintf(&buf[size], "H/ Completed : [ 0x"); 2740 2741 for (n = dd->slot_groups-1; n >= 0; n--) 2742 size += sprintf(&buf[size], "%08X ", 2743 readl(dd->port->completed[n])); 2744 2745 size += sprintf(&buf[size], "]\n"); 2746 size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n", 2747 readl(dd->port->mmio + PORT_IRQ_STAT)); 2748 size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n", 2749 readl(dd->mmio + HOST_IRQ_STAT)); 2750 size += sprintf(&buf[size], "\n"); 2751 2752 size += sprintf(&buf[size], "L/ Allocated : [ 0x"); 2753 2754 for (n = dd->slot_groups-1; n >= 0; n--) { 2755 if (sizeof(long) > sizeof(u32)) 2756 group_allocated = 2757 dd->port->allocated[n/2] >> (32*(n&1)); 2758 else 2759 group_allocated = dd->port->allocated[n]; 2760 size += sprintf(&buf[size], "%08X ", group_allocated); 2761 } 2762 size += sprintf(&buf[size], "]\n"); 2763 2764 size += sprintf(&buf[size], "L/ Commands in Q : [ 0x"); 2765 2766 for (n = dd->slot_groups-1; n >= 0; n--) { 2767 if (sizeof(long) > sizeof(u32)) 2768 group_allocated = 2769 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); 2770 else 2771 group_allocated = dd->port->cmds_to_issue[n]; 2772 size += sprintf(&buf[size], "%08X ", group_allocated); 2773 } 2774 size += sprintf(&buf[size], "]\n"); 2775 2776 *offset = size <= len ? size : len; 2777 size = copy_to_user(ubuf, buf, *offset); 2778 if (size) 2779 return -EFAULT; 2780 2781 return *offset; 2782 } 2783 2784 static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf, 2785 size_t len, loff_t *offset) 2786 { 2787 struct driver_data *dd = (struct driver_data *)f->private_data; 2788 char buf[MTIP_DFS_MAX_BUF_SIZE]; 2789 int size = *offset; 2790 2791 if (!len || size) 2792 return 0; 2793 2794 size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n", 2795 dd->port->flags); 2796 size += sprintf(&buf[size], "Flag-dd : [ %08lX ]\n", 2797 dd->dd_flag); 2798 2799 *offset = size <= len ? size : len; 2800 size = copy_to_user(ubuf, buf, *offset); 2801 if (size) 2802 return -EFAULT; 2803 2804 return *offset; 2805 } 2806 2807 static const struct file_operations mtip_regs_fops = { 2808 .owner = THIS_MODULE, 2809 .open = simple_open, 2810 .read = mtip_hw_read_registers, 2811 .llseek = no_llseek, 2812 }; 2813 2814 static const struct file_operations mtip_flags_fops = { 2815 .owner = THIS_MODULE, 2816 .open = simple_open, 2817 .read = mtip_hw_read_flags, 2818 .llseek = no_llseek, 2819 }; 2820 2821 /* 2822 * Create the sysfs related attributes. 2823 * 2824 * @dd Pointer to the driver data structure. 2825 * @kobj Pointer to the kobj for the block device. 2826 * 2827 * return value 2828 * 0 Operation completed successfully. 2829 * -EINVAL Invalid parameter. 2830 */ 2831 static int mtip_hw_sysfs_init(struct driver_data *dd, struct kobject *kobj) 2832 { 2833 if (!kobj || !dd) 2834 return -EINVAL; 2835 2836 if (sysfs_create_file(kobj, &dev_attr_status.attr)) 2837 dev_warn(&dd->pdev->dev, 2838 "Error creating 'status' sysfs entry\n"); 2839 return 0; 2840 } 2841 2842 /* 2843 * Remove the sysfs related attributes. 2844 * 2845 * @dd Pointer to the driver data structure. 2846 * @kobj Pointer to the kobj for the block device. 2847 * 2848 * return value 2849 * 0 Operation completed successfully. 2850 * -EINVAL Invalid parameter. 2851 */ 2852 static int mtip_hw_sysfs_exit(struct driver_data *dd, struct kobject *kobj) 2853 { 2854 if (!kobj || !dd) 2855 return -EINVAL; 2856 2857 sysfs_remove_file(kobj, &dev_attr_status.attr); 2858 2859 return 0; 2860 } 2861 2862 static int mtip_hw_debugfs_init(struct driver_data *dd) 2863 { 2864 if (!dfs_parent) 2865 return -1; 2866 2867 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); 2868 if (IS_ERR_OR_NULL(dd->dfs_node)) { 2869 dev_warn(&dd->pdev->dev, 2870 "Error creating node %s under debugfs\n", 2871 dd->disk->disk_name); 2872 dd->dfs_node = NULL; 2873 return -1; 2874 } 2875 2876 debugfs_create_file("flags", S_IRUGO, dd->dfs_node, dd, 2877 &mtip_flags_fops); 2878 debugfs_create_file("registers", S_IRUGO, dd->dfs_node, dd, 2879 &mtip_regs_fops); 2880 2881 return 0; 2882 } 2883 2884 static void mtip_hw_debugfs_exit(struct driver_data *dd) 2885 { 2886 debugfs_remove_recursive(dd->dfs_node); 2887 } 2888 2889 2890 /* 2891 * Perform any init/resume time hardware setup 2892 * 2893 * @dd Pointer to the driver data structure. 2894 * 2895 * return value 2896 * None 2897 */ 2898 static inline void hba_setup(struct driver_data *dd) 2899 { 2900 u32 hwdata; 2901 hwdata = readl(dd->mmio + HOST_HSORG); 2902 2903 /* interrupt bug workaround: use only 1 IS bit.*/ 2904 writel(hwdata | 2905 HSORG_DISABLE_SLOTGRP_INTR | 2906 HSORG_DISABLE_SLOTGRP_PXIS, 2907 dd->mmio + HOST_HSORG); 2908 } 2909 2910 /* 2911 * Detect the details of the product, and store anything needed 2912 * into the driver data structure. This includes product type and 2913 * version and number of slot groups. 2914 * 2915 * @dd Pointer to the driver data structure. 2916 * 2917 * return value 2918 * None 2919 */ 2920 static void mtip_detect_product(struct driver_data *dd) 2921 { 2922 u32 hwdata; 2923 unsigned int rev, slotgroups; 2924 2925 /* 2926 * HBA base + 0xFC [15:0] - vendor-specific hardware interface 2927 * info register: 2928 * [15:8] hardware/software interface rev# 2929 * [ 3] asic-style interface 2930 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style). 2931 */ 2932 hwdata = readl(dd->mmio + HOST_HSORG); 2933 2934 dd->product_type = MTIP_PRODUCT_UNKNOWN; 2935 dd->slot_groups = 1; 2936 2937 if (hwdata & 0x8) { 2938 dd->product_type = MTIP_PRODUCT_ASICFPGA; 2939 rev = (hwdata & HSORG_HWREV) >> 8; 2940 slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1; 2941 dev_info(&dd->pdev->dev, 2942 "ASIC-FPGA design, HS rev 0x%x, " 2943 "%i slot groups [%i slots]\n", 2944 rev, 2945 slotgroups, 2946 slotgroups * 32); 2947 2948 if (slotgroups > MTIP_MAX_SLOT_GROUPS) { 2949 dev_warn(&dd->pdev->dev, 2950 "Warning: driver only supports " 2951 "%i slot groups.\n", MTIP_MAX_SLOT_GROUPS); 2952 slotgroups = MTIP_MAX_SLOT_GROUPS; 2953 } 2954 dd->slot_groups = slotgroups; 2955 return; 2956 } 2957 2958 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); 2959 } 2960 2961 /* 2962 * Blocking wait for FTL rebuild to complete 2963 * 2964 * @dd Pointer to the DRIVER_DATA structure. 2965 * 2966 * return value 2967 * 0 FTL rebuild completed successfully 2968 * -EFAULT FTL rebuild error/timeout/interruption 2969 */ 2970 static int mtip_ftl_rebuild_poll(struct driver_data *dd) 2971 { 2972 unsigned long timeout, cnt = 0, start; 2973 2974 dev_warn(&dd->pdev->dev, 2975 "FTL rebuild in progress. Polling for completion.\n"); 2976 2977 start = jiffies; 2978 timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS); 2979 2980 do { 2981 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 2982 &dd->dd_flag))) 2983 return -EFAULT; 2984 if (mtip_check_surprise_removal(dd->pdev)) 2985 return -EFAULT; 2986 2987 if (mtip_get_identify(dd->port, NULL) < 0) 2988 return -EFAULT; 2989 2990 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 2991 MTIP_FTL_REBUILD_MAGIC) { 2992 ssleep(1); 2993 /* Print message every 3 minutes */ 2994 if (cnt++ >= 180) { 2995 dev_warn(&dd->pdev->dev, 2996 "FTL rebuild in progress (%d secs).\n", 2997 jiffies_to_msecs(jiffies - start) / 1000); 2998 cnt = 0; 2999 } 3000 } else { 3001 dev_warn(&dd->pdev->dev, 3002 "FTL rebuild complete (%d secs).\n", 3003 jiffies_to_msecs(jiffies - start) / 1000); 3004 mtip_block_initialize(dd); 3005 return 0; 3006 } 3007 ssleep(10); 3008 } while (time_before(jiffies, timeout)); 3009 3010 /* Check for timeout */ 3011 dev_err(&dd->pdev->dev, 3012 "Timed out waiting for FTL rebuild to complete (%d secs).\n", 3013 jiffies_to_msecs(jiffies - start) / 1000); 3014 return -EFAULT; 3015 } 3016 3017 /* 3018 * service thread to issue queued commands 3019 * 3020 * @data Pointer to the driver data structure. 3021 * 3022 * return value 3023 * 0 3024 */ 3025 3026 static int mtip_service_thread(void *data) 3027 { 3028 struct driver_data *dd = (struct driver_data *)data; 3029 unsigned long slot, slot_start, slot_wrap; 3030 unsigned int num_cmd_slots = dd->slot_groups * 32; 3031 struct mtip_port *port = dd->port; 3032 3033 while (1) { 3034 /* 3035 * the condition is to check neither an internal command is 3036 * is in progress nor error handling is active 3037 */ 3038 wait_event_interruptible(port->svc_wait, (port->flags) && 3039 !(port->flags & MTIP_PF_PAUSE_IO)); 3040 3041 if (kthread_should_stop()) 3042 break; 3043 3044 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 3045 &dd->dd_flag))) 3046 break; 3047 3048 set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 3049 if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) { 3050 slot = 1; 3051 /* used to restrict the loop to one iteration */ 3052 slot_start = num_cmd_slots; 3053 slot_wrap = 0; 3054 while (1) { 3055 slot = find_next_bit(port->cmds_to_issue, 3056 num_cmd_slots, slot); 3057 if (slot_wrap == 1) { 3058 if ((slot_start >= slot) || 3059 (slot >= num_cmd_slots)) 3060 break; 3061 } 3062 if (unlikely(slot_start == num_cmd_slots)) 3063 slot_start = slot; 3064 3065 if (unlikely(slot == num_cmd_slots)) { 3066 slot = 1; 3067 slot_wrap = 1; 3068 continue; 3069 } 3070 3071 /* Issue the command to the hardware */ 3072 mtip_issue_ncq_command(port, slot); 3073 3074 clear_bit(slot, port->cmds_to_issue); 3075 } 3076 3077 clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags); 3078 } else if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) { 3079 if (!mtip_ftl_rebuild_poll(dd)) 3080 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, 3081 &dd->dd_flag); 3082 clear_bit(MTIP_PF_REBUILD_BIT, &port->flags); 3083 } 3084 clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags); 3085 3086 if (test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags)) 3087 break; 3088 } 3089 return 0; 3090 } 3091 3092 /* 3093 * Called once for each card. 3094 * 3095 * @dd Pointer to the driver data structure. 3096 * 3097 * return value 3098 * 0 on success, else an error code. 3099 */ 3100 static int mtip_hw_init(struct driver_data *dd) 3101 { 3102 int i; 3103 int rv; 3104 unsigned int num_command_slots; 3105 unsigned long timeout, timetaken; 3106 unsigned char *buf; 3107 struct smart_attr attr242; 3108 3109 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; 3110 3111 mtip_detect_product(dd); 3112 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { 3113 rv = -EIO; 3114 goto out1; 3115 } 3116 num_command_slots = dd->slot_groups * 32; 3117 3118 hba_setup(dd); 3119 3120 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, 3121 dd->numa_node); 3122 if (!dd->port) { 3123 dev_err(&dd->pdev->dev, 3124 "Memory allocation: port structure\n"); 3125 return -ENOMEM; 3126 } 3127 3128 /* Continue workqueue setup */ 3129 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3130 dd->work[i].port = dd->port; 3131 3132 /* Counting semaphore to track command slot usage */ 3133 sema_init(&dd->port->cmd_slot, num_command_slots - 1); 3134 3135 /* Spinlock to prevent concurrent issue */ 3136 for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++) 3137 spin_lock_init(&dd->port->cmd_issue_lock[i]); 3138 3139 /* Set the port mmio base address. */ 3140 dd->port->mmio = dd->mmio + PORT_OFFSET; 3141 dd->port->dd = dd; 3142 3143 /* Allocate memory for the command list. */ 3144 dd->port->command_list = 3145 dmam_alloc_coherent(&dd->pdev->dev, 3146 HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 4), 3147 &dd->port->command_list_dma, 3148 GFP_KERNEL); 3149 if (!dd->port->command_list) { 3150 dev_err(&dd->pdev->dev, 3151 "Memory allocation: command list\n"); 3152 rv = -ENOMEM; 3153 goto out1; 3154 } 3155 3156 /* Clear the memory we have allocated. */ 3157 memset(dd->port->command_list, 3158 0, 3159 HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 4)); 3160 3161 /* Setup the addresse of the RX FIS. */ 3162 dd->port->rxfis = dd->port->command_list + HW_CMD_SLOT_SZ; 3163 dd->port->rxfis_dma = dd->port->command_list_dma + HW_CMD_SLOT_SZ; 3164 3165 /* Setup the address of the command tables. */ 3166 dd->port->command_table = dd->port->rxfis + AHCI_RX_FIS_SZ; 3167 dd->port->command_tbl_dma = dd->port->rxfis_dma + AHCI_RX_FIS_SZ; 3168 3169 /* Setup the address of the identify data. */ 3170 dd->port->identify = dd->port->command_table + 3171 HW_CMD_TBL_AR_SZ; 3172 dd->port->identify_dma = dd->port->command_tbl_dma + 3173 HW_CMD_TBL_AR_SZ; 3174 3175 /* Setup the address of the sector buffer - for some non-ncq cmds */ 3176 dd->port->sector_buffer = (void *) dd->port->identify + ATA_SECT_SIZE; 3177 dd->port->sector_buffer_dma = dd->port->identify_dma + ATA_SECT_SIZE; 3178 3179 /* Setup the address of the log buf - for read log command */ 3180 dd->port->log_buf = (void *)dd->port->sector_buffer + ATA_SECT_SIZE; 3181 dd->port->log_buf_dma = dd->port->sector_buffer_dma + ATA_SECT_SIZE; 3182 3183 /* Setup the address of the smart buf - for smart read data command */ 3184 dd->port->smart_buf = (void *)dd->port->log_buf + ATA_SECT_SIZE; 3185 dd->port->smart_buf_dma = dd->port->log_buf_dma + ATA_SECT_SIZE; 3186 3187 3188 /* Point the command headers at the command tables. */ 3189 for (i = 0; i < num_command_slots; i++) { 3190 dd->port->commands[i].command_header = 3191 dd->port->command_list + 3192 (sizeof(struct mtip_cmd_hdr) * i); 3193 dd->port->commands[i].command_header_dma = 3194 dd->port->command_list_dma + 3195 (sizeof(struct mtip_cmd_hdr) * i); 3196 3197 dd->port->commands[i].command = 3198 dd->port->command_table + (HW_CMD_TBL_SZ * i); 3199 dd->port->commands[i].command_dma = 3200 dd->port->command_tbl_dma + (HW_CMD_TBL_SZ * i); 3201 3202 if (readl(dd->mmio + HOST_CAP) & HOST_CAP_64) 3203 dd->port->commands[i].command_header->ctbau = 3204 __force_bit2int cpu_to_le32( 3205 (dd->port->commands[i].command_dma >> 16) >> 16); 3206 dd->port->commands[i].command_header->ctba = 3207 __force_bit2int cpu_to_le32( 3208 dd->port->commands[i].command_dma & 0xFFFFFFFF); 3209 3210 /* 3211 * If this is not done, a bug is reported by the stock 3212 * FC11 i386. Due to the fact that it has lots of kernel 3213 * debugging enabled. 3214 */ 3215 sg_init_table(dd->port->commands[i].sg, MTIP_MAX_SG); 3216 3217 /* Mark all commands as currently inactive.*/ 3218 atomic_set(&dd->port->commands[i].active, 0); 3219 } 3220 3221 /* Setup the pointers to the extended s_active and CI registers. */ 3222 for (i = 0; i < dd->slot_groups; i++) { 3223 dd->port->s_active[i] = 3224 dd->port->mmio + i*0x80 + PORT_SCR_ACT; 3225 dd->port->cmd_issue[i] = 3226 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; 3227 dd->port->completed[i] = 3228 dd->port->mmio + i*0x80 + PORT_SDBV; 3229 } 3230 3231 timetaken = jiffies; 3232 timeout = jiffies + msecs_to_jiffies(30000); 3233 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && 3234 time_before(jiffies, timeout)) { 3235 mdelay(100); 3236 } 3237 if (unlikely(mtip_check_surprise_removal(dd->pdev))) { 3238 timetaken = jiffies - timetaken; 3239 dev_warn(&dd->pdev->dev, 3240 "Surprise removal detected at %u ms\n", 3241 jiffies_to_msecs(timetaken)); 3242 rv = -ENODEV; 3243 goto out2 ; 3244 } 3245 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { 3246 timetaken = jiffies - timetaken; 3247 dev_warn(&dd->pdev->dev, 3248 "Removal detected at %u ms\n", 3249 jiffies_to_msecs(timetaken)); 3250 rv = -EFAULT; 3251 goto out2; 3252 } 3253 3254 /* Conditionally reset the HBA. */ 3255 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { 3256 if (mtip_hba_reset(dd) < 0) { 3257 dev_err(&dd->pdev->dev, 3258 "Card did not reset within timeout\n"); 3259 rv = -EIO; 3260 goto out2; 3261 } 3262 } else { 3263 /* Clear any pending interrupts on the HBA */ 3264 writel(readl(dd->mmio + HOST_IRQ_STAT), 3265 dd->mmio + HOST_IRQ_STAT); 3266 } 3267 3268 mtip_init_port(dd->port); 3269 mtip_start_port(dd->port); 3270 3271 /* Setup the ISR and enable interrupts. */ 3272 rv = devm_request_irq(&dd->pdev->dev, 3273 dd->pdev->irq, 3274 mtip_irq_handler, 3275 IRQF_SHARED, 3276 dev_driver_string(&dd->pdev->dev), 3277 dd); 3278 3279 if (rv) { 3280 dev_err(&dd->pdev->dev, 3281 "Unable to allocate IRQ %d\n", dd->pdev->irq); 3282 goto out2; 3283 } 3284 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); 3285 3286 /* Enable interrupts on the HBA. */ 3287 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3288 dd->mmio + HOST_CTL); 3289 3290 init_timer(&dd->port->cmd_timer); 3291 init_waitqueue_head(&dd->port->svc_wait); 3292 3293 dd->port->cmd_timer.data = (unsigned long int) dd->port; 3294 dd->port->cmd_timer.function = mtip_timeout_function; 3295 mod_timer(&dd->port->cmd_timer, 3296 jiffies + msecs_to_jiffies(MTIP_TIMEOUT_CHECK_PERIOD)); 3297 3298 3299 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { 3300 rv = -EFAULT; 3301 goto out3; 3302 } 3303 3304 if (mtip_get_identify(dd->port, NULL) < 0) { 3305 rv = -EFAULT; 3306 goto out3; 3307 } 3308 3309 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == 3310 MTIP_FTL_REBUILD_MAGIC) { 3311 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); 3312 return MTIP_FTL_REBUILD_MAGIC; 3313 } 3314 mtip_dump_identify(dd->port); 3315 3316 /* check write protect, over temp and rebuild statuses */ 3317 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, 3318 dd->port->log_buf, 3319 dd->port->log_buf_dma, 1); 3320 if (rv) { 3321 dev_warn(&dd->pdev->dev, 3322 "Error in READ LOG EXT (10h) command\n"); 3323 /* non-critical error, don't fail the load */ 3324 } else { 3325 buf = (unsigned char *)dd->port->log_buf; 3326 if (buf[259] & 0x1) { 3327 dev_info(&dd->pdev->dev, 3328 "Write protect bit is set.\n"); 3329 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); 3330 } 3331 if (buf[288] == 0xF7) { 3332 dev_info(&dd->pdev->dev, 3333 "Exceeded Tmax, drive in thermal shutdown.\n"); 3334 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); 3335 } 3336 if (buf[288] == 0xBF) { 3337 dev_info(&dd->pdev->dev, 3338 "Drive indicates rebuild has failed.\n"); 3339 /* TODO */ 3340 } 3341 } 3342 3343 /* get write protect progess */ 3344 memset(&attr242, 0, sizeof(struct smart_attr)); 3345 if (mtip_get_smart_attr(dd->port, 242, &attr242)) 3346 dev_warn(&dd->pdev->dev, 3347 "Unable to check write protect progress\n"); 3348 else 3349 dev_info(&dd->pdev->dev, 3350 "Write protect progress: %u%% (%u blocks)\n", 3351 attr242.cur, le32_to_cpu(attr242.data)); 3352 return rv; 3353 3354 out3: 3355 del_timer_sync(&dd->port->cmd_timer); 3356 3357 /* Disable interrupts on the HBA. */ 3358 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3359 dd->mmio + HOST_CTL); 3360 3361 /* Release the IRQ. */ 3362 irq_set_affinity_hint(dd->pdev->irq, NULL); 3363 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3364 3365 out2: 3366 mtip_deinit_port(dd->port); 3367 3368 /* Free the command/command header memory. */ 3369 dmam_free_coherent(&dd->pdev->dev, 3370 HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 4), 3371 dd->port->command_list, 3372 dd->port->command_list_dma); 3373 out1: 3374 /* Free the memory allocated for the for structure. */ 3375 kfree(dd->port); 3376 3377 return rv; 3378 } 3379 3380 /* 3381 * Called to deinitialize an interface. 3382 * 3383 * @dd Pointer to the driver data structure. 3384 * 3385 * return value 3386 * 0 3387 */ 3388 static int mtip_hw_exit(struct driver_data *dd) 3389 { 3390 /* 3391 * Send standby immediate (E0h) to the drive so that it 3392 * saves its state. 3393 */ 3394 if (!test_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag)) { 3395 3396 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) 3397 if (mtip_standby_immediate(dd->port)) 3398 dev_warn(&dd->pdev->dev, 3399 "STANDBY IMMEDIATE failed\n"); 3400 3401 /* de-initialize the port. */ 3402 mtip_deinit_port(dd->port); 3403 3404 /* Disable interrupts on the HBA. */ 3405 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3406 dd->mmio + HOST_CTL); 3407 } 3408 3409 del_timer_sync(&dd->port->cmd_timer); 3410 3411 /* Release the IRQ. */ 3412 irq_set_affinity_hint(dd->pdev->irq, NULL); 3413 devm_free_irq(&dd->pdev->dev, dd->pdev->irq, dd); 3414 3415 /* Free the command/command header memory. */ 3416 dmam_free_coherent(&dd->pdev->dev, 3417 HW_PORT_PRIV_DMA_SZ + (ATA_SECT_SIZE * 4), 3418 dd->port->command_list, 3419 dd->port->command_list_dma); 3420 /* Free the memory allocated for the for structure. */ 3421 kfree(dd->port); 3422 3423 return 0; 3424 } 3425 3426 /* 3427 * Issue a Standby Immediate command to the device. 3428 * 3429 * This function is called by the Block Layer just before the 3430 * system powers off during a shutdown. 3431 * 3432 * @dd Pointer to the driver data structure. 3433 * 3434 * return value 3435 * 0 3436 */ 3437 static int mtip_hw_shutdown(struct driver_data *dd) 3438 { 3439 /* 3440 * Send standby immediate (E0h) to the drive so that it 3441 * saves its state. 3442 */ 3443 mtip_standby_immediate(dd->port); 3444 3445 return 0; 3446 } 3447 3448 /* 3449 * Suspend function 3450 * 3451 * This function is called by the Block Layer just before the 3452 * system hibernates. 3453 * 3454 * @dd Pointer to the driver data structure. 3455 * 3456 * return value 3457 * 0 Suspend was successful 3458 * -EFAULT Suspend was not successful 3459 */ 3460 static int mtip_hw_suspend(struct driver_data *dd) 3461 { 3462 /* 3463 * Send standby immediate (E0h) to the drive 3464 * so that it saves its state. 3465 */ 3466 if (mtip_standby_immediate(dd->port) != 0) { 3467 dev_err(&dd->pdev->dev, 3468 "Failed standby-immediate command\n"); 3469 return -EFAULT; 3470 } 3471 3472 /* Disable interrupts on the HBA.*/ 3473 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, 3474 dd->mmio + HOST_CTL); 3475 mtip_deinit_port(dd->port); 3476 3477 return 0; 3478 } 3479 3480 /* 3481 * Resume function 3482 * 3483 * This function is called by the Block Layer as the 3484 * system resumes. 3485 * 3486 * @dd Pointer to the driver data structure. 3487 * 3488 * return value 3489 * 0 Resume was successful 3490 * -EFAULT Resume was not successful 3491 */ 3492 static int mtip_hw_resume(struct driver_data *dd) 3493 { 3494 /* Perform any needed hardware setup steps */ 3495 hba_setup(dd); 3496 3497 /* Reset the HBA */ 3498 if (mtip_hba_reset(dd) != 0) { 3499 dev_err(&dd->pdev->dev, 3500 "Unable to reset the HBA\n"); 3501 return -EFAULT; 3502 } 3503 3504 /* 3505 * Enable the port, DMA engine, and FIS reception specific 3506 * h/w in controller. 3507 */ 3508 mtip_init_port(dd->port); 3509 mtip_start_port(dd->port); 3510 3511 /* Enable interrupts on the HBA.*/ 3512 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, 3513 dd->mmio + HOST_CTL); 3514 3515 return 0; 3516 } 3517 3518 /* 3519 * Helper function for reusing disk name 3520 * upon hot insertion. 3521 */ 3522 static int rssd_disk_name_format(char *prefix, 3523 int index, 3524 char *buf, 3525 int buflen) 3526 { 3527 const int base = 'z' - 'a' + 1; 3528 char *begin = buf + strlen(prefix); 3529 char *end = buf + buflen; 3530 char *p; 3531 int unit; 3532 3533 p = end - 1; 3534 *p = '\0'; 3535 unit = base; 3536 do { 3537 if (p == begin) 3538 return -EINVAL; 3539 *--p = 'a' + (index % unit); 3540 index = (index / unit) - 1; 3541 } while (index >= 0); 3542 3543 memmove(begin, p, end - p); 3544 memcpy(buf, prefix, strlen(prefix)); 3545 3546 return 0; 3547 } 3548 3549 /* 3550 * Block layer IOCTL handler. 3551 * 3552 * @dev Pointer to the block_device structure. 3553 * @mode ignored 3554 * @cmd IOCTL command passed from the user application. 3555 * @arg Argument passed from the user application. 3556 * 3557 * return value 3558 * 0 IOCTL completed successfully. 3559 * -ENOTTY IOCTL not supported or invalid driver data 3560 * structure pointer. 3561 */ 3562 static int mtip_block_ioctl(struct block_device *dev, 3563 fmode_t mode, 3564 unsigned cmd, 3565 unsigned long arg) 3566 { 3567 struct driver_data *dd = dev->bd_disk->private_data; 3568 3569 if (!capable(CAP_SYS_ADMIN)) 3570 return -EACCES; 3571 3572 if (!dd) 3573 return -ENOTTY; 3574 3575 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3576 return -ENOTTY; 3577 3578 switch (cmd) { 3579 case BLKFLSBUF: 3580 return -ENOTTY; 3581 default: 3582 return mtip_hw_ioctl(dd, cmd, arg); 3583 } 3584 } 3585 3586 #ifdef CONFIG_COMPAT 3587 /* 3588 * Block layer compat IOCTL handler. 3589 * 3590 * @dev Pointer to the block_device structure. 3591 * @mode ignored 3592 * @cmd IOCTL command passed from the user application. 3593 * @arg Argument passed from the user application. 3594 * 3595 * return value 3596 * 0 IOCTL completed successfully. 3597 * -ENOTTY IOCTL not supported or invalid driver data 3598 * structure pointer. 3599 */ 3600 static int mtip_block_compat_ioctl(struct block_device *dev, 3601 fmode_t mode, 3602 unsigned cmd, 3603 unsigned long arg) 3604 { 3605 struct driver_data *dd = dev->bd_disk->private_data; 3606 3607 if (!capable(CAP_SYS_ADMIN)) 3608 return -EACCES; 3609 3610 if (!dd) 3611 return -ENOTTY; 3612 3613 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) 3614 return -ENOTTY; 3615 3616 switch (cmd) { 3617 case BLKFLSBUF: 3618 return -ENOTTY; 3619 case HDIO_DRIVE_TASKFILE: { 3620 struct mtip_compat_ide_task_request_s __user *compat_req_task; 3621 ide_task_request_t req_task; 3622 int compat_tasksize, outtotal, ret; 3623 3624 compat_tasksize = 3625 sizeof(struct mtip_compat_ide_task_request_s); 3626 3627 compat_req_task = 3628 (struct mtip_compat_ide_task_request_s __user *) arg; 3629 3630 if (copy_from_user(&req_task, (void __user *) arg, 3631 compat_tasksize - (2 * sizeof(compat_long_t)))) 3632 return -EFAULT; 3633 3634 if (get_user(req_task.out_size, &compat_req_task->out_size)) 3635 return -EFAULT; 3636 3637 if (get_user(req_task.in_size, &compat_req_task->in_size)) 3638 return -EFAULT; 3639 3640 outtotal = sizeof(struct mtip_compat_ide_task_request_s); 3641 3642 ret = exec_drive_taskfile(dd, (void __user *) arg, 3643 &req_task, outtotal); 3644 3645 if (copy_to_user((void __user *) arg, &req_task, 3646 compat_tasksize - 3647 (2 * sizeof(compat_long_t)))) 3648 return -EFAULT; 3649 3650 if (put_user(req_task.out_size, &compat_req_task->out_size)) 3651 return -EFAULT; 3652 3653 if (put_user(req_task.in_size, &compat_req_task->in_size)) 3654 return -EFAULT; 3655 3656 return ret; 3657 } 3658 default: 3659 return mtip_hw_ioctl(dd, cmd, arg); 3660 } 3661 } 3662 #endif 3663 3664 /* 3665 * Obtain the geometry of the device. 3666 * 3667 * You may think that this function is obsolete, but some applications, 3668 * fdisk for example still used CHS values. This function describes the 3669 * device as having 224 heads and 56 sectors per cylinder. These values are 3670 * chosen so that each cylinder is aligned on a 4KB boundary. Since a 3671 * partition is described in terms of a start and end cylinder this means 3672 * that each partition is also 4KB aligned. Non-aligned partitions adversely 3673 * affects performance. 3674 * 3675 * @dev Pointer to the block_device strucutre. 3676 * @geo Pointer to a hd_geometry structure. 3677 * 3678 * return value 3679 * 0 Operation completed successfully. 3680 * -ENOTTY An error occurred while reading the drive capacity. 3681 */ 3682 static int mtip_block_getgeo(struct block_device *dev, 3683 struct hd_geometry *geo) 3684 { 3685 struct driver_data *dd = dev->bd_disk->private_data; 3686 sector_t capacity; 3687 3688 if (!dd) 3689 return -ENOTTY; 3690 3691 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3692 dev_warn(&dd->pdev->dev, 3693 "Could not get drive capacity.\n"); 3694 return -ENOTTY; 3695 } 3696 3697 geo->heads = 224; 3698 geo->sectors = 56; 3699 sector_div(capacity, (geo->heads * geo->sectors)); 3700 geo->cylinders = capacity; 3701 return 0; 3702 } 3703 3704 /* 3705 * Block device operation function. 3706 * 3707 * This structure contains pointers to the functions required by the block 3708 * layer. 3709 */ 3710 static const struct block_device_operations mtip_block_ops = { 3711 .ioctl = mtip_block_ioctl, 3712 #ifdef CONFIG_COMPAT 3713 .compat_ioctl = mtip_block_compat_ioctl, 3714 #endif 3715 .getgeo = mtip_block_getgeo, 3716 .owner = THIS_MODULE 3717 }; 3718 3719 /* 3720 * Block layer make request function. 3721 * 3722 * This function is called by the kernel to process a BIO for 3723 * the P320 device. 3724 * 3725 * @queue Pointer to the request queue. Unused other than to obtain 3726 * the driver data structure. 3727 * @bio Pointer to the BIO. 3728 * 3729 */ 3730 static void mtip_make_request(struct request_queue *queue, struct bio *bio) 3731 { 3732 struct driver_data *dd = queue->queuedata; 3733 struct scatterlist *sg; 3734 struct bio_vec *bvec; 3735 int nents = 0; 3736 int tag = 0; 3737 3738 if (unlikely(dd->dd_flag & MTIP_DDF_STOP_IO)) { 3739 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, 3740 &dd->dd_flag))) { 3741 bio_endio(bio, -ENXIO); 3742 return; 3743 } 3744 if (unlikely(test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))) { 3745 bio_endio(bio, -ENODATA); 3746 return; 3747 } 3748 if (unlikely(test_bit(MTIP_DDF_WRITE_PROTECT_BIT, 3749 &dd->dd_flag) && 3750 bio_data_dir(bio))) { 3751 bio_endio(bio, -ENODATA); 3752 return; 3753 } 3754 if (unlikely(test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))) { 3755 bio_endio(bio, -ENODATA); 3756 return; 3757 } 3758 } 3759 3760 if (unlikely(bio->bi_rw & REQ_DISCARD)) { 3761 bio_endio(bio, mtip_send_trim(dd, bio->bi_sector, 3762 bio_sectors(bio))); 3763 return; 3764 } 3765 3766 if (unlikely(!bio_has_data(bio))) { 3767 blk_queue_flush(queue, 0); 3768 bio_endio(bio, 0); 3769 return; 3770 } 3771 3772 sg = mtip_hw_get_scatterlist(dd, &tag); 3773 if (likely(sg != NULL)) { 3774 blk_queue_bounce(queue, &bio); 3775 3776 if (unlikely((bio)->bi_vcnt > MTIP_MAX_SG)) { 3777 dev_warn(&dd->pdev->dev, 3778 "Maximum number of SGL entries exceeded\n"); 3779 bio_io_error(bio); 3780 mtip_hw_release_scatterlist(dd, tag); 3781 return; 3782 } 3783 3784 /* Create the scatter list for this bio. */ 3785 bio_for_each_segment(bvec, bio, nents) { 3786 sg_set_page(&sg[nents], 3787 bvec->bv_page, 3788 bvec->bv_len, 3789 bvec->bv_offset); 3790 } 3791 3792 /* Issue the read/write. */ 3793 mtip_hw_submit_io(dd, 3794 bio->bi_sector, 3795 bio_sectors(bio), 3796 nents, 3797 tag, 3798 bio_endio, 3799 bio, 3800 bio_data_dir(bio)); 3801 } else 3802 bio_io_error(bio); 3803 } 3804 3805 /* 3806 * Block layer initialization function. 3807 * 3808 * This function is called once by the PCI layer for each P320 3809 * device that is connected to the system. 3810 * 3811 * @dd Pointer to the driver data structure. 3812 * 3813 * return value 3814 * 0 on success else an error code. 3815 */ 3816 static int mtip_block_initialize(struct driver_data *dd) 3817 { 3818 int rv = 0, wait_for_rebuild = 0; 3819 sector_t capacity; 3820 unsigned int index = 0; 3821 struct kobject *kobj; 3822 unsigned char thd_name[16]; 3823 3824 if (dd->disk) 3825 goto skip_create_disk; /* hw init done, before rebuild */ 3826 3827 /* Initialize the protocol layer. */ 3828 wait_for_rebuild = mtip_hw_init(dd); 3829 if (wait_for_rebuild < 0) { 3830 dev_err(&dd->pdev->dev, 3831 "Protocol layer initialization failed\n"); 3832 rv = -EINVAL; 3833 goto protocol_init_error; 3834 } 3835 3836 dd->disk = alloc_disk_node(MTIP_MAX_MINORS, dd->numa_node); 3837 if (dd->disk == NULL) { 3838 dev_err(&dd->pdev->dev, 3839 "Unable to allocate gendisk structure\n"); 3840 rv = -EINVAL; 3841 goto alloc_disk_error; 3842 } 3843 3844 /* Generate the disk name, implemented same as in sd.c */ 3845 do { 3846 if (!ida_pre_get(&rssd_index_ida, GFP_KERNEL)) 3847 goto ida_get_error; 3848 3849 spin_lock(&rssd_index_lock); 3850 rv = ida_get_new(&rssd_index_ida, &index); 3851 spin_unlock(&rssd_index_lock); 3852 } while (rv == -EAGAIN); 3853 3854 if (rv) 3855 goto ida_get_error; 3856 3857 rv = rssd_disk_name_format("rssd", 3858 index, 3859 dd->disk->disk_name, 3860 DISK_NAME_LEN); 3861 if (rv) 3862 goto disk_index_error; 3863 3864 dd->disk->driverfs_dev = &dd->pdev->dev; 3865 dd->disk->major = dd->major; 3866 dd->disk->first_minor = dd->instance * MTIP_MAX_MINORS; 3867 dd->disk->fops = &mtip_block_ops; 3868 dd->disk->private_data = dd; 3869 dd->index = index; 3870 3871 /* 3872 * if rebuild pending, start the service thread, and delay the block 3873 * queue creation and add_disk() 3874 */ 3875 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3876 goto start_service_thread; 3877 3878 skip_create_disk: 3879 /* Allocate the request queue. */ 3880 dd->queue = blk_alloc_queue_node(GFP_KERNEL, dd->numa_node); 3881 if (dd->queue == NULL) { 3882 dev_err(&dd->pdev->dev, 3883 "Unable to allocate request queue\n"); 3884 rv = -ENOMEM; 3885 goto block_queue_alloc_init_error; 3886 } 3887 3888 /* Attach our request function to the request queue. */ 3889 blk_queue_make_request(dd->queue, mtip_make_request); 3890 3891 dd->disk->queue = dd->queue; 3892 dd->queue->queuedata = dd; 3893 3894 /* Set device limits. */ 3895 set_bit(QUEUE_FLAG_NONROT, &dd->queue->queue_flags); 3896 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); 3897 blk_queue_physical_block_size(dd->queue, 4096); 3898 blk_queue_max_hw_sectors(dd->queue, 0xffff); 3899 blk_queue_max_segment_size(dd->queue, 0x400000); 3900 blk_queue_io_min(dd->queue, 4096); 3901 3902 /* 3903 * write back cache is not supported in the device. FUA depends on 3904 * write back cache support, hence setting flush support to zero. 3905 */ 3906 blk_queue_flush(dd->queue, 0); 3907 3908 /* Signal trim support */ 3909 if (dd->trim_supp == true) { 3910 set_bit(QUEUE_FLAG_DISCARD, &dd->queue->queue_flags); 3911 dd->queue->limits.discard_granularity = 4096; 3912 blk_queue_max_discard_sectors(dd->queue, 3913 MTIP_MAX_TRIM_ENTRY_LEN * MTIP_MAX_TRIM_ENTRIES); 3914 dd->queue->limits.discard_zeroes_data = 0; 3915 } 3916 3917 /* Set the capacity of the device in 512 byte sectors. */ 3918 if (!(mtip_hw_get_capacity(dd, &capacity))) { 3919 dev_warn(&dd->pdev->dev, 3920 "Could not read drive capacity\n"); 3921 rv = -EIO; 3922 goto read_capacity_error; 3923 } 3924 set_capacity(dd->disk, capacity); 3925 3926 /* Enable the block device and add it to /dev */ 3927 add_disk(dd->disk); 3928 3929 /* 3930 * Now that the disk is active, initialize any sysfs attributes 3931 * managed by the protocol layer. 3932 */ 3933 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 3934 if (kobj) { 3935 mtip_hw_sysfs_init(dd, kobj); 3936 kobject_put(kobj); 3937 } 3938 mtip_hw_debugfs_init(dd); 3939 3940 if (dd->mtip_svc_handler) { 3941 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 3942 return rv; /* service thread created for handling rebuild */ 3943 } 3944 3945 start_service_thread: 3946 sprintf(thd_name, "mtip_svc_thd_%02d", index); 3947 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, 3948 dd, dd->numa_node, thd_name); 3949 3950 if (IS_ERR(dd->mtip_svc_handler)) { 3951 dev_err(&dd->pdev->dev, "service thread failed to start\n"); 3952 dd->mtip_svc_handler = NULL; 3953 rv = -EFAULT; 3954 goto kthread_run_error; 3955 } 3956 wake_up_process(dd->mtip_svc_handler); 3957 if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) 3958 rv = wait_for_rebuild; 3959 3960 return rv; 3961 3962 kthread_run_error: 3963 mtip_hw_debugfs_exit(dd); 3964 3965 /* Delete our gendisk. This also removes the device from /dev */ 3966 del_gendisk(dd->disk); 3967 3968 read_capacity_error: 3969 blk_cleanup_queue(dd->queue); 3970 3971 block_queue_alloc_init_error: 3972 disk_index_error: 3973 spin_lock(&rssd_index_lock); 3974 ida_remove(&rssd_index_ida, index); 3975 spin_unlock(&rssd_index_lock); 3976 3977 ida_get_error: 3978 put_disk(dd->disk); 3979 3980 alloc_disk_error: 3981 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ 3982 3983 protocol_init_error: 3984 return rv; 3985 } 3986 3987 /* 3988 * Block layer deinitialization function. 3989 * 3990 * Called by the PCI layer as each P320 device is removed. 3991 * 3992 * @dd Pointer to the driver data structure. 3993 * 3994 * return value 3995 * 0 3996 */ 3997 static int mtip_block_remove(struct driver_data *dd) 3998 { 3999 struct kobject *kobj; 4000 4001 if (dd->mtip_svc_handler) { 4002 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); 4003 wake_up_interruptible(&dd->port->svc_wait); 4004 kthread_stop(dd->mtip_svc_handler); 4005 } 4006 4007 /* Clean up the sysfs attributes, if created */ 4008 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) { 4009 kobj = kobject_get(&disk_to_dev(dd->disk)->kobj); 4010 if (kobj) { 4011 mtip_hw_sysfs_exit(dd, kobj); 4012 kobject_put(kobj); 4013 } 4014 } 4015 mtip_hw_debugfs_exit(dd); 4016 4017 /* 4018 * Delete our gendisk structure. This also removes the device 4019 * from /dev 4020 */ 4021 if (dd->disk) { 4022 if (dd->disk->queue) 4023 del_gendisk(dd->disk); 4024 else 4025 put_disk(dd->disk); 4026 } 4027 4028 spin_lock(&rssd_index_lock); 4029 ida_remove(&rssd_index_ida, dd->index); 4030 spin_unlock(&rssd_index_lock); 4031 4032 blk_cleanup_queue(dd->queue); 4033 dd->disk = NULL; 4034 dd->queue = NULL; 4035 4036 /* De-initialize the protocol layer. */ 4037 mtip_hw_exit(dd); 4038 4039 return 0; 4040 } 4041 4042 /* 4043 * Function called by the PCI layer when just before the 4044 * machine shuts down. 4045 * 4046 * If a protocol layer shutdown function is present it will be called 4047 * by this function. 4048 * 4049 * @dd Pointer to the driver data structure. 4050 * 4051 * return value 4052 * 0 4053 */ 4054 static int mtip_block_shutdown(struct driver_data *dd) 4055 { 4056 dev_info(&dd->pdev->dev, 4057 "Shutting down %s ...\n", dd->disk->disk_name); 4058 4059 /* Delete our gendisk structure, and cleanup the blk queue. */ 4060 if (dd->disk) { 4061 if (dd->disk->queue) 4062 del_gendisk(dd->disk); 4063 else 4064 put_disk(dd->disk); 4065 } 4066 4067 4068 spin_lock(&rssd_index_lock); 4069 ida_remove(&rssd_index_ida, dd->index); 4070 spin_unlock(&rssd_index_lock); 4071 4072 blk_cleanup_queue(dd->queue); 4073 dd->disk = NULL; 4074 dd->queue = NULL; 4075 4076 mtip_hw_shutdown(dd); 4077 return 0; 4078 } 4079 4080 static int mtip_block_suspend(struct driver_data *dd) 4081 { 4082 dev_info(&dd->pdev->dev, 4083 "Suspending %s ...\n", dd->disk->disk_name); 4084 mtip_hw_suspend(dd); 4085 return 0; 4086 } 4087 4088 static int mtip_block_resume(struct driver_data *dd) 4089 { 4090 dev_info(&dd->pdev->dev, "Resuming %s ...\n", 4091 dd->disk->disk_name); 4092 mtip_hw_resume(dd); 4093 return 0; 4094 } 4095 4096 static void drop_cpu(int cpu) 4097 { 4098 cpu_use[cpu]--; 4099 } 4100 4101 static int get_least_used_cpu_on_node(int node) 4102 { 4103 int cpu, least_used_cpu, least_cnt; 4104 const struct cpumask *node_mask; 4105 4106 node_mask = cpumask_of_node(node); 4107 least_used_cpu = cpumask_first(node_mask); 4108 least_cnt = cpu_use[least_used_cpu]; 4109 cpu = least_used_cpu; 4110 4111 for_each_cpu(cpu, node_mask) { 4112 if (cpu_use[cpu] < least_cnt) { 4113 least_used_cpu = cpu; 4114 least_cnt = cpu_use[cpu]; 4115 } 4116 } 4117 cpu_use[least_used_cpu]++; 4118 return least_used_cpu; 4119 } 4120 4121 /* Helper for selecting a node in round robin mode */ 4122 static inline int mtip_get_next_rr_node(void) 4123 { 4124 static int next_node = -1; 4125 4126 if (next_node == -1) { 4127 next_node = first_online_node; 4128 return next_node; 4129 } 4130 4131 next_node = next_online_node(next_node); 4132 if (next_node == MAX_NUMNODES) 4133 next_node = first_online_node; 4134 return next_node; 4135 } 4136 4137 static DEFINE_HANDLER(0); 4138 static DEFINE_HANDLER(1); 4139 static DEFINE_HANDLER(2); 4140 static DEFINE_HANDLER(3); 4141 static DEFINE_HANDLER(4); 4142 static DEFINE_HANDLER(5); 4143 static DEFINE_HANDLER(6); 4144 static DEFINE_HANDLER(7); 4145 4146 /* 4147 * Called for each supported PCI device detected. 4148 * 4149 * This function allocates the private data structure, enables the 4150 * PCI device and then calls the block layer initialization function. 4151 * 4152 * return value 4153 * 0 on success else an error code. 4154 */ 4155 static int mtip_pci_probe(struct pci_dev *pdev, 4156 const struct pci_device_id *ent) 4157 { 4158 int rv = 0; 4159 struct driver_data *dd = NULL; 4160 char cpu_list[256]; 4161 const struct cpumask *node_mask; 4162 int cpu, i = 0, j = 0; 4163 int my_node = NUMA_NO_NODE; 4164 4165 /* Allocate memory for this devices private data. */ 4166 my_node = pcibus_to_node(pdev->bus); 4167 if (my_node != NUMA_NO_NODE) { 4168 if (!node_online(my_node)) 4169 my_node = mtip_get_next_rr_node(); 4170 } else { 4171 dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n"); 4172 my_node = mtip_get_next_rr_node(); 4173 } 4174 dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n", 4175 my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev), 4176 cpu_to_node(smp_processor_id()), smp_processor_id()); 4177 4178 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); 4179 if (dd == NULL) { 4180 dev_err(&pdev->dev, 4181 "Unable to allocate memory for driver data\n"); 4182 return -ENOMEM; 4183 } 4184 4185 /* Attach the private data to this PCI device. */ 4186 pci_set_drvdata(pdev, dd); 4187 4188 rv = pcim_enable_device(pdev); 4189 if (rv < 0) { 4190 dev_err(&pdev->dev, "Unable to enable device\n"); 4191 goto iomap_err; 4192 } 4193 4194 /* Map BAR5 to memory. */ 4195 rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME); 4196 if (rv < 0) { 4197 dev_err(&pdev->dev, "Unable to map regions\n"); 4198 goto iomap_err; 4199 } 4200 4201 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 4202 rv = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4203 4204 if (rv) { 4205 rv = pci_set_consistent_dma_mask(pdev, 4206 DMA_BIT_MASK(32)); 4207 if (rv) { 4208 dev_warn(&pdev->dev, 4209 "64-bit DMA enable failed\n"); 4210 goto setmask_err; 4211 } 4212 } 4213 } 4214 4215 /* Copy the info we may need later into the private data structure. */ 4216 dd->major = mtip_major; 4217 dd->instance = instance; 4218 dd->pdev = pdev; 4219 dd->numa_node = my_node; 4220 4221 memset(dd->workq_name, 0, 32); 4222 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); 4223 4224 dd->isr_workq = create_workqueue(dd->workq_name); 4225 if (!dd->isr_workq) { 4226 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); 4227 goto block_initialize_err; 4228 } 4229 4230 memset(cpu_list, 0, sizeof(cpu_list)); 4231 4232 node_mask = cpumask_of_node(dd->numa_node); 4233 if (!cpumask_empty(node_mask)) { 4234 for_each_cpu(cpu, node_mask) 4235 { 4236 snprintf(&cpu_list[j], 256 - j, "%d ", cpu); 4237 j = strlen(cpu_list); 4238 } 4239 4240 dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n", 4241 dd->numa_node, 4242 topology_physical_package_id(cpumask_first(node_mask)), 4243 nr_cpus_node(dd->numa_node), 4244 cpu_list); 4245 } else 4246 dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n"); 4247 4248 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); 4249 dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n", 4250 cpu_to_node(dd->isr_binding), dd->isr_binding); 4251 4252 /* first worker context always runs in ISR */ 4253 dd->work[0].cpu_binding = dd->isr_binding; 4254 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4255 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); 4256 dd->work[3].cpu_binding = dd->work[0].cpu_binding; 4257 dd->work[4].cpu_binding = dd->work[1].cpu_binding; 4258 dd->work[5].cpu_binding = dd->work[2].cpu_binding; 4259 dd->work[6].cpu_binding = dd->work[2].cpu_binding; 4260 dd->work[7].cpu_binding = dd->work[1].cpu_binding; 4261 4262 /* Log the bindings */ 4263 for_each_present_cpu(cpu) { 4264 memset(cpu_list, 0, sizeof(cpu_list)); 4265 for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) { 4266 if (dd->work[i].cpu_binding == cpu) { 4267 snprintf(&cpu_list[j], 256 - j, "%d ", i); 4268 j = strlen(cpu_list); 4269 } 4270 } 4271 if (j) 4272 dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list); 4273 } 4274 4275 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); 4276 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); 4277 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); 4278 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); 4279 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); 4280 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); 4281 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); 4282 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); 4283 4284 pci_set_master(pdev); 4285 if (pci_enable_msi(pdev)) { 4286 dev_warn(&pdev->dev, 4287 "Unable to enable MSI interrupt.\n"); 4288 goto block_initialize_err; 4289 } 4290 4291 /* Initialize the block layer. */ 4292 rv = mtip_block_initialize(dd); 4293 if (rv < 0) { 4294 dev_err(&pdev->dev, 4295 "Unable to initialize block layer\n"); 4296 goto block_initialize_err; 4297 } 4298 4299 /* 4300 * Increment the instance count so that each device has a unique 4301 * instance number. 4302 */ 4303 instance++; 4304 if (rv != MTIP_FTL_REBUILD_MAGIC) 4305 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); 4306 goto done; 4307 4308 block_initialize_err: 4309 pci_disable_msi(pdev); 4310 if (dd->isr_workq) { 4311 flush_workqueue(dd->isr_workq); 4312 destroy_workqueue(dd->isr_workq); 4313 drop_cpu(dd->work[0].cpu_binding); 4314 drop_cpu(dd->work[1].cpu_binding); 4315 drop_cpu(dd->work[2].cpu_binding); 4316 } 4317 setmask_err: 4318 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4319 4320 iomap_err: 4321 kfree(dd); 4322 pci_set_drvdata(pdev, NULL); 4323 return rv; 4324 done: 4325 return rv; 4326 } 4327 4328 /* 4329 * Called for each probed device when the device is removed or the 4330 * driver is unloaded. 4331 * 4332 * return value 4333 * None 4334 */ 4335 static void mtip_pci_remove(struct pci_dev *pdev) 4336 { 4337 struct driver_data *dd = pci_get_drvdata(pdev); 4338 int counter = 0; 4339 4340 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); 4341 4342 if (mtip_check_surprise_removal(pdev)) { 4343 while (!test_bit(MTIP_DDF_CLEANUP_BIT, &dd->dd_flag)) { 4344 counter++; 4345 msleep(20); 4346 if (counter == 10) { 4347 /* Cleanup the outstanding commands */ 4348 mtip_command_cleanup(dd); 4349 break; 4350 } 4351 } 4352 } 4353 4354 /* Clean up the block layer. */ 4355 mtip_block_remove(dd); 4356 4357 if (dd->isr_workq) { 4358 flush_workqueue(dd->isr_workq); 4359 destroy_workqueue(dd->isr_workq); 4360 drop_cpu(dd->work[0].cpu_binding); 4361 drop_cpu(dd->work[1].cpu_binding); 4362 drop_cpu(dd->work[2].cpu_binding); 4363 } 4364 4365 pci_disable_msi(pdev); 4366 4367 kfree(dd); 4368 pcim_iounmap_regions(pdev, 1 << MTIP_ABAR); 4369 } 4370 4371 /* 4372 * Called for each probed device when the device is suspended. 4373 * 4374 * return value 4375 * 0 Success 4376 * <0 Error 4377 */ 4378 static int mtip_pci_suspend(struct pci_dev *pdev, pm_message_t mesg) 4379 { 4380 int rv = 0; 4381 struct driver_data *dd = pci_get_drvdata(pdev); 4382 4383 if (!dd) { 4384 dev_err(&pdev->dev, 4385 "Driver private datastructure is NULL\n"); 4386 return -EFAULT; 4387 } 4388 4389 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4390 4391 /* Disable ports & interrupts then send standby immediate */ 4392 rv = mtip_block_suspend(dd); 4393 if (rv < 0) { 4394 dev_err(&pdev->dev, 4395 "Failed to suspend controller\n"); 4396 return rv; 4397 } 4398 4399 /* 4400 * Save the pci config space to pdev structure & 4401 * disable the device 4402 */ 4403 pci_save_state(pdev); 4404 pci_disable_device(pdev); 4405 4406 /* Move to Low power state*/ 4407 pci_set_power_state(pdev, PCI_D3hot); 4408 4409 return rv; 4410 } 4411 4412 /* 4413 * Called for each probed device when the device is resumed. 4414 * 4415 * return value 4416 * 0 Success 4417 * <0 Error 4418 */ 4419 static int mtip_pci_resume(struct pci_dev *pdev) 4420 { 4421 int rv = 0; 4422 struct driver_data *dd; 4423 4424 dd = pci_get_drvdata(pdev); 4425 if (!dd) { 4426 dev_err(&pdev->dev, 4427 "Driver private datastructure is NULL\n"); 4428 return -EFAULT; 4429 } 4430 4431 /* Move the device to active State */ 4432 pci_set_power_state(pdev, PCI_D0); 4433 4434 /* Restore PCI configuration space */ 4435 pci_restore_state(pdev); 4436 4437 /* Enable the PCI device*/ 4438 rv = pcim_enable_device(pdev); 4439 if (rv < 0) { 4440 dev_err(&pdev->dev, 4441 "Failed to enable card during resume\n"); 4442 goto err; 4443 } 4444 pci_set_master(pdev); 4445 4446 /* 4447 * Calls hbaReset, initPort, & startPort function 4448 * then enables interrupts 4449 */ 4450 rv = mtip_block_resume(dd); 4451 if (rv < 0) 4452 dev_err(&pdev->dev, "Unable to resume\n"); 4453 4454 err: 4455 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); 4456 4457 return rv; 4458 } 4459 4460 /* 4461 * Shutdown routine 4462 * 4463 * return value 4464 * None 4465 */ 4466 static void mtip_pci_shutdown(struct pci_dev *pdev) 4467 { 4468 struct driver_data *dd = pci_get_drvdata(pdev); 4469 if (dd) 4470 mtip_block_shutdown(dd); 4471 } 4472 4473 /* Table of device ids supported by this driver. */ 4474 static DEFINE_PCI_DEVICE_TABLE(mtip_pci_tbl) = { 4475 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) }, 4476 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) }, 4477 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) }, 4478 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) }, 4479 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) }, 4480 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) }, 4481 { PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) }, 4482 { 0 } 4483 }; 4484 4485 /* Structure that describes the PCI driver functions. */ 4486 static struct pci_driver mtip_pci_driver = { 4487 .name = MTIP_DRV_NAME, 4488 .id_table = mtip_pci_tbl, 4489 .probe = mtip_pci_probe, 4490 .remove = mtip_pci_remove, 4491 .suspend = mtip_pci_suspend, 4492 .resume = mtip_pci_resume, 4493 .shutdown = mtip_pci_shutdown, 4494 }; 4495 4496 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl); 4497 4498 /* 4499 * Module initialization function. 4500 * 4501 * Called once when the module is loaded. This function allocates a major 4502 * block device number to the Cyclone devices and registers the PCI layer 4503 * of the driver. 4504 * 4505 * Return value 4506 * 0 on success else error code. 4507 */ 4508 static int __init mtip_init(void) 4509 { 4510 int error; 4511 4512 pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n"); 4513 4514 /* Allocate a major block device number to use with this driver. */ 4515 error = register_blkdev(0, MTIP_DRV_NAME); 4516 if (error <= 0) { 4517 pr_err("Unable to register block device (%d)\n", 4518 error); 4519 return -EBUSY; 4520 } 4521 mtip_major = error; 4522 4523 if (!dfs_parent) { 4524 dfs_parent = debugfs_create_dir("rssd", NULL); 4525 if (IS_ERR_OR_NULL(dfs_parent)) { 4526 pr_warn("Error creating debugfs parent\n"); 4527 dfs_parent = NULL; 4528 } 4529 } 4530 4531 /* Register our PCI operations. */ 4532 error = pci_register_driver(&mtip_pci_driver); 4533 if (error) { 4534 debugfs_remove(dfs_parent); 4535 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4536 } 4537 4538 return error; 4539 } 4540 4541 /* 4542 * Module de-initialization function. 4543 * 4544 * Called once when the module is unloaded. This function deallocates 4545 * the major block device number allocated by mtip_init() and 4546 * unregisters the PCI layer of the driver. 4547 * 4548 * Return value 4549 * none 4550 */ 4551 static void __exit mtip_exit(void) 4552 { 4553 debugfs_remove_recursive(dfs_parent); 4554 4555 /* Release the allocated major block device number. */ 4556 unregister_blkdev(mtip_major, MTIP_DRV_NAME); 4557 4558 /* Unregister the PCI driver. */ 4559 pci_unregister_driver(&mtip_pci_driver); 4560 } 4561 4562 MODULE_AUTHOR("Micron Technology, Inc"); 4563 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver"); 4564 MODULE_LICENSE("GPL"); 4565 MODULE_VERSION(MTIP_DRV_VERSION); 4566 4567 module_init(mtip_init); 4568 module_exit(mtip_exit); 4569