1 /* 2 * ahci.c - AHCI SATA support 3 * 4 * Maintained by: Jeff Garzik <jgarzik@pobox.com> 5 * Please ALWAYS copy linux-ide@vger.kernel.org 6 * on emails. 7 * 8 * Copyright 2004-2005 Red Hat, Inc. 9 * 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; see the file COPYING. If not, write to 23 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 24 * 25 * 26 * libata documentation is available via 'make {ps|pdf}docs', 27 * as Documentation/DocBook/libata.* 28 * 29 * AHCI hardware documentation: 30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf 31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf 32 * 33 */ 34 35 #include <linux/kernel.h> 36 #include <linux/module.h> 37 #include <linux/pci.h> 38 #include <linux/init.h> 39 #include <linux/blkdev.h> 40 #include <linux/delay.h> 41 #include <linux/interrupt.h> 42 #include <linux/dma-mapping.h> 43 #include <linux/device.h> 44 #include <scsi/scsi_host.h> 45 #include <scsi/scsi_cmnd.h> 46 #include <linux/libata.h> 47 48 #define DRV_NAME "ahci" 49 #define DRV_VERSION "2.1" 50 51 52 enum { 53 AHCI_PCI_BAR = 5, 54 AHCI_MAX_PORTS = 32, 55 AHCI_MAX_SG = 168, /* hardware max is 64K */ 56 AHCI_DMA_BOUNDARY = 0xffffffff, 57 AHCI_USE_CLUSTERING = 0, 58 AHCI_MAX_CMDS = 32, 59 AHCI_CMD_SZ = 32, 60 AHCI_CMD_SLOT_SZ = AHCI_MAX_CMDS * AHCI_CMD_SZ, 61 AHCI_RX_FIS_SZ = 256, 62 AHCI_CMD_TBL_CDB = 0x40, 63 AHCI_CMD_TBL_HDR_SZ = 0x80, 64 AHCI_CMD_TBL_SZ = AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16), 65 AHCI_CMD_TBL_AR_SZ = AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS, 66 AHCI_PORT_PRIV_DMA_SZ = AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ + 67 AHCI_RX_FIS_SZ, 68 AHCI_IRQ_ON_SG = (1 << 31), 69 AHCI_CMD_ATAPI = (1 << 5), 70 AHCI_CMD_WRITE = (1 << 6), 71 AHCI_CMD_PREFETCH = (1 << 7), 72 AHCI_CMD_RESET = (1 << 8), 73 AHCI_CMD_CLR_BUSY = (1 << 10), 74 75 RX_FIS_D2H_REG = 0x40, /* offset of D2H Register FIS data */ 76 RX_FIS_SDB = 0x58, /* offset of SDB FIS data */ 77 RX_FIS_UNK = 0x60, /* offset of Unknown FIS data */ 78 79 board_ahci = 0, 80 board_ahci_pi = 1, 81 board_ahci_vt8251 = 2, 82 board_ahci_ign_iferr = 3, 83 board_ahci_sb600 = 4, 84 85 /* global controller registers */ 86 HOST_CAP = 0x00, /* host capabilities */ 87 HOST_CTL = 0x04, /* global host control */ 88 HOST_IRQ_STAT = 0x08, /* interrupt status */ 89 HOST_PORTS_IMPL = 0x0c, /* bitmap of implemented ports */ 90 HOST_VERSION = 0x10, /* AHCI spec. version compliancy */ 91 92 /* HOST_CTL bits */ 93 HOST_RESET = (1 << 0), /* reset controller; self-clear */ 94 HOST_IRQ_EN = (1 << 1), /* global IRQ enable */ 95 HOST_AHCI_EN = (1 << 31), /* AHCI enabled */ 96 97 /* HOST_CAP bits */ 98 HOST_CAP_SSC = (1 << 14), /* Slumber capable */ 99 HOST_CAP_CLO = (1 << 24), /* Command List Override support */ 100 HOST_CAP_SSS = (1 << 27), /* Staggered Spin-up */ 101 HOST_CAP_NCQ = (1 << 30), /* Native Command Queueing */ 102 HOST_CAP_64 = (1 << 31), /* PCI DAC (64-bit DMA) support */ 103 104 /* registers for each SATA port */ 105 PORT_LST_ADDR = 0x00, /* command list DMA addr */ 106 PORT_LST_ADDR_HI = 0x04, /* command list DMA addr hi */ 107 PORT_FIS_ADDR = 0x08, /* FIS rx buf addr */ 108 PORT_FIS_ADDR_HI = 0x0c, /* FIS rx buf addr hi */ 109 PORT_IRQ_STAT = 0x10, /* interrupt status */ 110 PORT_IRQ_MASK = 0x14, /* interrupt enable/disable mask */ 111 PORT_CMD = 0x18, /* port command */ 112 PORT_TFDATA = 0x20, /* taskfile data */ 113 PORT_SIG = 0x24, /* device TF signature */ 114 PORT_CMD_ISSUE = 0x38, /* command issue */ 115 PORT_SCR = 0x28, /* SATA phy register block */ 116 PORT_SCR_STAT = 0x28, /* SATA phy register: SStatus */ 117 PORT_SCR_CTL = 0x2c, /* SATA phy register: SControl */ 118 PORT_SCR_ERR = 0x30, /* SATA phy register: SError */ 119 PORT_SCR_ACT = 0x34, /* SATA phy register: SActive */ 120 121 /* PORT_IRQ_{STAT,MASK} bits */ 122 PORT_IRQ_COLD_PRES = (1 << 31), /* cold presence detect */ 123 PORT_IRQ_TF_ERR = (1 << 30), /* task file error */ 124 PORT_IRQ_HBUS_ERR = (1 << 29), /* host bus fatal error */ 125 PORT_IRQ_HBUS_DATA_ERR = (1 << 28), /* host bus data error */ 126 PORT_IRQ_IF_ERR = (1 << 27), /* interface fatal error */ 127 PORT_IRQ_IF_NONFATAL = (1 << 26), /* interface non-fatal error */ 128 PORT_IRQ_OVERFLOW = (1 << 24), /* xfer exhausted available S/G */ 129 PORT_IRQ_BAD_PMP = (1 << 23), /* incorrect port multiplier */ 130 131 PORT_IRQ_PHYRDY = (1 << 22), /* PhyRdy changed */ 132 PORT_IRQ_DEV_ILCK = (1 << 7), /* device interlock */ 133 PORT_IRQ_CONNECT = (1 << 6), /* port connect change status */ 134 PORT_IRQ_SG_DONE = (1 << 5), /* descriptor processed */ 135 PORT_IRQ_UNK_FIS = (1 << 4), /* unknown FIS rx'd */ 136 PORT_IRQ_SDB_FIS = (1 << 3), /* Set Device Bits FIS rx'd */ 137 PORT_IRQ_DMAS_FIS = (1 << 2), /* DMA Setup FIS rx'd */ 138 PORT_IRQ_PIOS_FIS = (1 << 1), /* PIO Setup FIS rx'd */ 139 PORT_IRQ_D2H_REG_FIS = (1 << 0), /* D2H Register FIS rx'd */ 140 141 PORT_IRQ_FREEZE = PORT_IRQ_HBUS_ERR | 142 PORT_IRQ_IF_ERR | 143 PORT_IRQ_CONNECT | 144 PORT_IRQ_PHYRDY | 145 PORT_IRQ_UNK_FIS, 146 PORT_IRQ_ERROR = PORT_IRQ_FREEZE | 147 PORT_IRQ_TF_ERR | 148 PORT_IRQ_HBUS_DATA_ERR, 149 DEF_PORT_IRQ = PORT_IRQ_ERROR | PORT_IRQ_SG_DONE | 150 PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS | 151 PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS, 152 153 /* PORT_CMD bits */ 154 PORT_CMD_ATAPI = (1 << 24), /* Device is ATAPI */ 155 PORT_CMD_LIST_ON = (1 << 15), /* cmd list DMA engine running */ 156 PORT_CMD_FIS_ON = (1 << 14), /* FIS DMA engine running */ 157 PORT_CMD_FIS_RX = (1 << 4), /* Enable FIS receive DMA engine */ 158 PORT_CMD_CLO = (1 << 3), /* Command list override */ 159 PORT_CMD_POWER_ON = (1 << 2), /* Power up device */ 160 PORT_CMD_SPIN_UP = (1 << 1), /* Spin up device */ 161 PORT_CMD_START = (1 << 0), /* Enable port DMA engine */ 162 163 PORT_CMD_ICC_MASK = (0xf << 28), /* i/f ICC state mask */ 164 PORT_CMD_ICC_ACTIVE = (0x1 << 28), /* Put i/f in active state */ 165 PORT_CMD_ICC_PARTIAL = (0x2 << 28), /* Put i/f in partial state */ 166 PORT_CMD_ICC_SLUMBER = (0x6 << 28), /* Put i/f in slumber state */ 167 168 /* ap->flags bits */ 169 AHCI_FLAG_NO_NCQ = (1 << 24), 170 AHCI_FLAG_IGN_IRQ_IF_ERR = (1 << 25), /* ignore IRQ_IF_ERR */ 171 AHCI_FLAG_HONOR_PI = (1 << 26), /* honor PORTS_IMPL */ 172 AHCI_FLAG_IGN_SERR_INTERNAL = (1 << 27), /* ignore SERR_INTERNAL */ 173 174 AHCI_FLAG_COMMON = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY | 175 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA | 176 ATA_FLAG_SKIP_D2H_BSY, 177 }; 178 179 struct ahci_cmd_hdr { 180 u32 opts; 181 u32 status; 182 u32 tbl_addr; 183 u32 tbl_addr_hi; 184 u32 reserved[4]; 185 }; 186 187 struct ahci_sg { 188 u32 addr; 189 u32 addr_hi; 190 u32 reserved; 191 u32 flags_size; 192 }; 193 194 struct ahci_host_priv { 195 u32 cap; /* cap to use */ 196 u32 port_map; /* port map to use */ 197 u32 saved_cap; /* saved initial cap */ 198 u32 saved_port_map; /* saved initial port_map */ 199 }; 200 201 struct ahci_port_priv { 202 struct ahci_cmd_hdr *cmd_slot; 203 dma_addr_t cmd_slot_dma; 204 void *cmd_tbl; 205 dma_addr_t cmd_tbl_dma; 206 void *rx_fis; 207 dma_addr_t rx_fis_dma; 208 /* for NCQ spurious interrupt analysis */ 209 unsigned int ncq_saw_d2h:1; 210 unsigned int ncq_saw_dmas:1; 211 unsigned int ncq_saw_sdb:1; 212 }; 213 214 static u32 ahci_scr_read (struct ata_port *ap, unsigned int sc_reg); 215 static void ahci_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val); 216 static int ahci_init_one (struct pci_dev *pdev, const struct pci_device_id *ent); 217 static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc); 218 static void ahci_irq_clear(struct ata_port *ap); 219 static int ahci_port_start(struct ata_port *ap); 220 static void ahci_port_stop(struct ata_port *ap); 221 static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf); 222 static void ahci_qc_prep(struct ata_queued_cmd *qc); 223 static u8 ahci_check_status(struct ata_port *ap); 224 static void ahci_freeze(struct ata_port *ap); 225 static void ahci_thaw(struct ata_port *ap); 226 static void ahci_error_handler(struct ata_port *ap); 227 static void ahci_vt8251_error_handler(struct ata_port *ap); 228 static void ahci_post_internal_cmd(struct ata_queued_cmd *qc); 229 #ifdef CONFIG_PM 230 static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg); 231 static int ahci_port_resume(struct ata_port *ap); 232 static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg); 233 static int ahci_pci_device_resume(struct pci_dev *pdev); 234 #endif 235 236 static struct scsi_host_template ahci_sht = { 237 .module = THIS_MODULE, 238 .name = DRV_NAME, 239 .ioctl = ata_scsi_ioctl, 240 .queuecommand = ata_scsi_queuecmd, 241 .change_queue_depth = ata_scsi_change_queue_depth, 242 .can_queue = AHCI_MAX_CMDS - 1, 243 .this_id = ATA_SHT_THIS_ID, 244 .sg_tablesize = AHCI_MAX_SG, 245 .cmd_per_lun = ATA_SHT_CMD_PER_LUN, 246 .emulated = ATA_SHT_EMULATED, 247 .use_clustering = AHCI_USE_CLUSTERING, 248 .proc_name = DRV_NAME, 249 .dma_boundary = AHCI_DMA_BOUNDARY, 250 .slave_configure = ata_scsi_slave_config, 251 .slave_destroy = ata_scsi_slave_destroy, 252 .bios_param = ata_std_bios_param, 253 }; 254 255 static const struct ata_port_operations ahci_ops = { 256 .port_disable = ata_port_disable, 257 258 .check_status = ahci_check_status, 259 .check_altstatus = ahci_check_status, 260 .dev_select = ata_noop_dev_select, 261 262 .tf_read = ahci_tf_read, 263 264 .qc_prep = ahci_qc_prep, 265 .qc_issue = ahci_qc_issue, 266 267 .irq_clear = ahci_irq_clear, 268 .irq_on = ata_dummy_irq_on, 269 .irq_ack = ata_dummy_irq_ack, 270 271 .scr_read = ahci_scr_read, 272 .scr_write = ahci_scr_write, 273 274 .freeze = ahci_freeze, 275 .thaw = ahci_thaw, 276 277 .error_handler = ahci_error_handler, 278 .post_internal_cmd = ahci_post_internal_cmd, 279 280 #ifdef CONFIG_PM 281 .port_suspend = ahci_port_suspend, 282 .port_resume = ahci_port_resume, 283 #endif 284 285 .port_start = ahci_port_start, 286 .port_stop = ahci_port_stop, 287 }; 288 289 static const struct ata_port_operations ahci_vt8251_ops = { 290 .port_disable = ata_port_disable, 291 292 .check_status = ahci_check_status, 293 .check_altstatus = ahci_check_status, 294 .dev_select = ata_noop_dev_select, 295 296 .tf_read = ahci_tf_read, 297 298 .qc_prep = ahci_qc_prep, 299 .qc_issue = ahci_qc_issue, 300 301 .irq_clear = ahci_irq_clear, 302 .irq_on = ata_dummy_irq_on, 303 .irq_ack = ata_dummy_irq_ack, 304 305 .scr_read = ahci_scr_read, 306 .scr_write = ahci_scr_write, 307 308 .freeze = ahci_freeze, 309 .thaw = ahci_thaw, 310 311 .error_handler = ahci_vt8251_error_handler, 312 .post_internal_cmd = ahci_post_internal_cmd, 313 314 #ifdef CONFIG_PM 315 .port_suspend = ahci_port_suspend, 316 .port_resume = ahci_port_resume, 317 #endif 318 319 .port_start = ahci_port_start, 320 .port_stop = ahci_port_stop, 321 }; 322 323 static const struct ata_port_info ahci_port_info[] = { 324 /* board_ahci */ 325 { 326 .flags = AHCI_FLAG_COMMON, 327 .pio_mask = 0x1f, /* pio0-4 */ 328 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 329 .port_ops = &ahci_ops, 330 }, 331 /* board_ahci_pi */ 332 { 333 .flags = AHCI_FLAG_COMMON | AHCI_FLAG_HONOR_PI, 334 .pio_mask = 0x1f, /* pio0-4 */ 335 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 336 .port_ops = &ahci_ops, 337 }, 338 /* board_ahci_vt8251 */ 339 { 340 .flags = AHCI_FLAG_COMMON | ATA_FLAG_HRST_TO_RESUME | 341 AHCI_FLAG_NO_NCQ, 342 .pio_mask = 0x1f, /* pio0-4 */ 343 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 344 .port_ops = &ahci_vt8251_ops, 345 }, 346 /* board_ahci_ign_iferr */ 347 { 348 .flags = AHCI_FLAG_COMMON | AHCI_FLAG_IGN_IRQ_IF_ERR, 349 .pio_mask = 0x1f, /* pio0-4 */ 350 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 351 .port_ops = &ahci_ops, 352 }, 353 /* board_ahci_sb600 */ 354 { 355 .flags = AHCI_FLAG_COMMON | 356 AHCI_FLAG_IGN_SERR_INTERNAL, 357 .pio_mask = 0x1f, /* pio0-4 */ 358 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 359 .port_ops = &ahci_ops, 360 }, 361 }; 362 363 static const struct pci_device_id ahci_pci_tbl[] = { 364 /* Intel */ 365 { PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */ 366 { PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */ 367 { PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */ 368 { PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */ 369 { PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */ 370 { PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */ 371 { PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */ 372 { PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */ 373 { PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */ 374 { PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */ 375 { PCI_VDEVICE(INTEL, 0x2821), board_ahci_pi }, /* ICH8 */ 376 { PCI_VDEVICE(INTEL, 0x2822), board_ahci_pi }, /* ICH8 */ 377 { PCI_VDEVICE(INTEL, 0x2824), board_ahci_pi }, /* ICH8 */ 378 { PCI_VDEVICE(INTEL, 0x2829), board_ahci_pi }, /* ICH8M */ 379 { PCI_VDEVICE(INTEL, 0x282a), board_ahci_pi }, /* ICH8M */ 380 { PCI_VDEVICE(INTEL, 0x2922), board_ahci_pi }, /* ICH9 */ 381 { PCI_VDEVICE(INTEL, 0x2923), board_ahci_pi }, /* ICH9 */ 382 { PCI_VDEVICE(INTEL, 0x2924), board_ahci_pi }, /* ICH9 */ 383 { PCI_VDEVICE(INTEL, 0x2925), board_ahci_pi }, /* ICH9 */ 384 { PCI_VDEVICE(INTEL, 0x2927), board_ahci_pi }, /* ICH9 */ 385 { PCI_VDEVICE(INTEL, 0x2929), board_ahci_pi }, /* ICH9M */ 386 { PCI_VDEVICE(INTEL, 0x292a), board_ahci_pi }, /* ICH9M */ 387 { PCI_VDEVICE(INTEL, 0x292b), board_ahci_pi }, /* ICH9M */ 388 { PCI_VDEVICE(INTEL, 0x292c), board_ahci_pi }, /* ICH9M */ 389 { PCI_VDEVICE(INTEL, 0x292f), board_ahci_pi }, /* ICH9M */ 390 { PCI_VDEVICE(INTEL, 0x294d), board_ahci_pi }, /* ICH9 */ 391 { PCI_VDEVICE(INTEL, 0x294e), board_ahci_pi }, /* ICH9M */ 392 393 /* JMicron 360/1/3/5/6, match class to avoid IDE function */ 394 { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 395 PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr }, 396 397 /* ATI */ 398 { PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */ 399 { PCI_VDEVICE(ATI, 0x4390), board_ahci_sb600 }, /* ATI SB700 */ 400 401 /* VIA */ 402 { PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */ 403 { PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */ 404 405 /* NVIDIA */ 406 { PCI_VDEVICE(NVIDIA, 0x044c), board_ahci }, /* MCP65 */ 407 { PCI_VDEVICE(NVIDIA, 0x044d), board_ahci }, /* MCP65 */ 408 { PCI_VDEVICE(NVIDIA, 0x044e), board_ahci }, /* MCP65 */ 409 { PCI_VDEVICE(NVIDIA, 0x044f), board_ahci }, /* MCP65 */ 410 { PCI_VDEVICE(NVIDIA, 0x045c), board_ahci }, /* MCP65 */ 411 { PCI_VDEVICE(NVIDIA, 0x045d), board_ahci }, /* MCP65 */ 412 { PCI_VDEVICE(NVIDIA, 0x045e), board_ahci }, /* MCP65 */ 413 { PCI_VDEVICE(NVIDIA, 0x045f), board_ahci }, /* MCP65 */ 414 { PCI_VDEVICE(NVIDIA, 0x0550), board_ahci }, /* MCP67 */ 415 { PCI_VDEVICE(NVIDIA, 0x0551), board_ahci }, /* MCP67 */ 416 { PCI_VDEVICE(NVIDIA, 0x0552), board_ahci }, /* MCP67 */ 417 { PCI_VDEVICE(NVIDIA, 0x0553), board_ahci }, /* MCP67 */ 418 { PCI_VDEVICE(NVIDIA, 0x0554), board_ahci }, /* MCP67 */ 419 { PCI_VDEVICE(NVIDIA, 0x0555), board_ahci }, /* MCP67 */ 420 { PCI_VDEVICE(NVIDIA, 0x0556), board_ahci }, /* MCP67 */ 421 { PCI_VDEVICE(NVIDIA, 0x0557), board_ahci }, /* MCP67 */ 422 { PCI_VDEVICE(NVIDIA, 0x0558), board_ahci }, /* MCP67 */ 423 { PCI_VDEVICE(NVIDIA, 0x0559), board_ahci }, /* MCP67 */ 424 { PCI_VDEVICE(NVIDIA, 0x055a), board_ahci }, /* MCP67 */ 425 { PCI_VDEVICE(NVIDIA, 0x055b), board_ahci }, /* MCP67 */ 426 427 /* SiS */ 428 { PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */ 429 { PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */ 430 { PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */ 431 432 /* Generic, PCI class code for AHCI */ 433 { PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 434 PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci }, 435 436 { } /* terminate list */ 437 }; 438 439 440 static struct pci_driver ahci_pci_driver = { 441 .name = DRV_NAME, 442 .id_table = ahci_pci_tbl, 443 .probe = ahci_init_one, 444 .remove = ata_pci_remove_one, 445 #ifdef CONFIG_PM 446 .suspend = ahci_pci_device_suspend, 447 .resume = ahci_pci_device_resume, 448 #endif 449 }; 450 451 452 static inline int ahci_nr_ports(u32 cap) 453 { 454 return (cap & 0x1f) + 1; 455 } 456 457 static inline void __iomem *ahci_port_base(struct ata_port *ap) 458 { 459 void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR]; 460 461 return mmio + 0x100 + (ap->port_no * 0x80); 462 } 463 464 /** 465 * ahci_save_initial_config - Save and fixup initial config values 466 * @pdev: target PCI device 467 * @pi: associated ATA port info 468 * @hpriv: host private area to store config values 469 * 470 * Some registers containing configuration info might be setup by 471 * BIOS and might be cleared on reset. This function saves the 472 * initial values of those registers into @hpriv such that they 473 * can be restored after controller reset. 474 * 475 * If inconsistent, config values are fixed up by this function. 476 * 477 * LOCKING: 478 * None. 479 */ 480 static void ahci_save_initial_config(struct pci_dev *pdev, 481 const struct ata_port_info *pi, 482 struct ahci_host_priv *hpriv) 483 { 484 void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR]; 485 u32 cap, port_map; 486 int i; 487 488 /* Values prefixed with saved_ are written back to host after 489 * reset. Values without are used for driver operation. 490 */ 491 hpriv->saved_cap = cap = readl(mmio + HOST_CAP); 492 hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL); 493 494 /* fixup zero port_map */ 495 if (!port_map) { 496 port_map = (1 << ahci_nr_ports(hpriv->cap)) - 1; 497 dev_printk(KERN_WARNING, &pdev->dev, 498 "PORTS_IMPL is zero, forcing 0x%x\n", port_map); 499 500 /* write the fixed up value to the PI register */ 501 hpriv->saved_port_map = port_map; 502 } 503 504 /* cross check port_map and cap.n_ports */ 505 if (pi->flags & AHCI_FLAG_HONOR_PI) { 506 u32 tmp_port_map = port_map; 507 int n_ports = ahci_nr_ports(cap); 508 509 for (i = 0; i < AHCI_MAX_PORTS && n_ports; i++) { 510 if (tmp_port_map & (1 << i)) { 511 n_ports--; 512 tmp_port_map &= ~(1 << i); 513 } 514 } 515 516 /* Whine if inconsistent. No need to update cap. 517 * port_map is used to determine number of ports. 518 */ 519 if (n_ports || tmp_port_map) 520 dev_printk(KERN_WARNING, &pdev->dev, 521 "nr_ports (%u) and implemented port map " 522 "(0x%x) don't match\n", 523 ahci_nr_ports(cap), port_map); 524 } else { 525 /* fabricate port_map from cap.nr_ports */ 526 port_map = (1 << ahci_nr_ports(cap)) - 1; 527 } 528 529 /* record values to use during operation */ 530 hpriv->cap = cap; 531 hpriv->port_map = port_map; 532 } 533 534 /** 535 * ahci_restore_initial_config - Restore initial config 536 * @host: target ATA host 537 * 538 * Restore initial config stored by ahci_save_initial_config(). 539 * 540 * LOCKING: 541 * None. 542 */ 543 static void ahci_restore_initial_config(struct ata_host *host) 544 { 545 struct ahci_host_priv *hpriv = host->private_data; 546 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 547 548 writel(hpriv->saved_cap, mmio + HOST_CAP); 549 writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL); 550 (void) readl(mmio + HOST_PORTS_IMPL); /* flush */ 551 } 552 553 static u32 ahci_scr_read (struct ata_port *ap, unsigned int sc_reg_in) 554 { 555 unsigned int sc_reg; 556 557 switch (sc_reg_in) { 558 case SCR_STATUS: sc_reg = 0; break; 559 case SCR_CONTROL: sc_reg = 1; break; 560 case SCR_ERROR: sc_reg = 2; break; 561 case SCR_ACTIVE: sc_reg = 3; break; 562 default: 563 return 0xffffffffU; 564 } 565 566 return readl(ap->ioaddr.scr_addr + (sc_reg * 4)); 567 } 568 569 570 static void ahci_scr_write (struct ata_port *ap, unsigned int sc_reg_in, 571 u32 val) 572 { 573 unsigned int sc_reg; 574 575 switch (sc_reg_in) { 576 case SCR_STATUS: sc_reg = 0; break; 577 case SCR_CONTROL: sc_reg = 1; break; 578 case SCR_ERROR: sc_reg = 2; break; 579 case SCR_ACTIVE: sc_reg = 3; break; 580 default: 581 return; 582 } 583 584 writel(val, ap->ioaddr.scr_addr + (sc_reg * 4)); 585 } 586 587 static void ahci_start_engine(struct ata_port *ap) 588 { 589 void __iomem *port_mmio = ahci_port_base(ap); 590 u32 tmp; 591 592 /* start DMA */ 593 tmp = readl(port_mmio + PORT_CMD); 594 tmp |= PORT_CMD_START; 595 writel(tmp, port_mmio + PORT_CMD); 596 readl(port_mmio + PORT_CMD); /* flush */ 597 } 598 599 static int ahci_stop_engine(struct ata_port *ap) 600 { 601 void __iomem *port_mmio = ahci_port_base(ap); 602 u32 tmp; 603 604 tmp = readl(port_mmio + PORT_CMD); 605 606 /* check if the HBA is idle */ 607 if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0) 608 return 0; 609 610 /* setting HBA to idle */ 611 tmp &= ~PORT_CMD_START; 612 writel(tmp, port_mmio + PORT_CMD); 613 614 /* wait for engine to stop. This could be as long as 500 msec */ 615 tmp = ata_wait_register(port_mmio + PORT_CMD, 616 PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500); 617 if (tmp & PORT_CMD_LIST_ON) 618 return -EIO; 619 620 return 0; 621 } 622 623 static void ahci_start_fis_rx(struct ata_port *ap) 624 { 625 void __iomem *port_mmio = ahci_port_base(ap); 626 struct ahci_host_priv *hpriv = ap->host->private_data; 627 struct ahci_port_priv *pp = ap->private_data; 628 u32 tmp; 629 630 /* set FIS registers */ 631 if (hpriv->cap & HOST_CAP_64) 632 writel((pp->cmd_slot_dma >> 16) >> 16, 633 port_mmio + PORT_LST_ADDR_HI); 634 writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR); 635 636 if (hpriv->cap & HOST_CAP_64) 637 writel((pp->rx_fis_dma >> 16) >> 16, 638 port_mmio + PORT_FIS_ADDR_HI); 639 writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR); 640 641 /* enable FIS reception */ 642 tmp = readl(port_mmio + PORT_CMD); 643 tmp |= PORT_CMD_FIS_RX; 644 writel(tmp, port_mmio + PORT_CMD); 645 646 /* flush */ 647 readl(port_mmio + PORT_CMD); 648 } 649 650 static int ahci_stop_fis_rx(struct ata_port *ap) 651 { 652 void __iomem *port_mmio = ahci_port_base(ap); 653 u32 tmp; 654 655 /* disable FIS reception */ 656 tmp = readl(port_mmio + PORT_CMD); 657 tmp &= ~PORT_CMD_FIS_RX; 658 writel(tmp, port_mmio + PORT_CMD); 659 660 /* wait for completion, spec says 500ms, give it 1000 */ 661 tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON, 662 PORT_CMD_FIS_ON, 10, 1000); 663 if (tmp & PORT_CMD_FIS_ON) 664 return -EBUSY; 665 666 return 0; 667 } 668 669 static void ahci_power_up(struct ata_port *ap) 670 { 671 struct ahci_host_priv *hpriv = ap->host->private_data; 672 void __iomem *port_mmio = ahci_port_base(ap); 673 u32 cmd; 674 675 cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK; 676 677 /* spin up device */ 678 if (hpriv->cap & HOST_CAP_SSS) { 679 cmd |= PORT_CMD_SPIN_UP; 680 writel(cmd, port_mmio + PORT_CMD); 681 } 682 683 /* wake up link */ 684 writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD); 685 } 686 687 #ifdef CONFIG_PM 688 static void ahci_power_down(struct ata_port *ap) 689 { 690 struct ahci_host_priv *hpriv = ap->host->private_data; 691 void __iomem *port_mmio = ahci_port_base(ap); 692 u32 cmd, scontrol; 693 694 if (!(hpriv->cap & HOST_CAP_SSS)) 695 return; 696 697 /* put device into listen mode, first set PxSCTL.DET to 0 */ 698 scontrol = readl(port_mmio + PORT_SCR_CTL); 699 scontrol &= ~0xf; 700 writel(scontrol, port_mmio + PORT_SCR_CTL); 701 702 /* then set PxCMD.SUD to 0 */ 703 cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK; 704 cmd &= ~PORT_CMD_SPIN_UP; 705 writel(cmd, port_mmio + PORT_CMD); 706 } 707 #endif 708 709 static void ahci_init_port(struct ata_port *ap) 710 { 711 /* enable FIS reception */ 712 ahci_start_fis_rx(ap); 713 714 /* enable DMA */ 715 ahci_start_engine(ap); 716 } 717 718 static int ahci_deinit_port(struct ata_port *ap, const char **emsg) 719 { 720 int rc; 721 722 /* disable DMA */ 723 rc = ahci_stop_engine(ap); 724 if (rc) { 725 *emsg = "failed to stop engine"; 726 return rc; 727 } 728 729 /* disable FIS reception */ 730 rc = ahci_stop_fis_rx(ap); 731 if (rc) { 732 *emsg = "failed stop FIS RX"; 733 return rc; 734 } 735 736 return 0; 737 } 738 739 static int ahci_reset_controller(struct ata_host *host) 740 { 741 struct pci_dev *pdev = to_pci_dev(host->dev); 742 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 743 u32 tmp; 744 745 /* global controller reset */ 746 tmp = readl(mmio + HOST_CTL); 747 if ((tmp & HOST_RESET) == 0) { 748 writel(tmp | HOST_RESET, mmio + HOST_CTL); 749 readl(mmio + HOST_CTL); /* flush */ 750 } 751 752 /* reset must complete within 1 second, or 753 * the hardware should be considered fried. 754 */ 755 ssleep(1); 756 757 tmp = readl(mmio + HOST_CTL); 758 if (tmp & HOST_RESET) { 759 dev_printk(KERN_ERR, host->dev, 760 "controller reset failed (0x%x)\n", tmp); 761 return -EIO; 762 } 763 764 /* turn on AHCI mode */ 765 writel(HOST_AHCI_EN, mmio + HOST_CTL); 766 (void) readl(mmio + HOST_CTL); /* flush */ 767 768 /* some registers might be cleared on reset. restore initial values */ 769 ahci_restore_initial_config(host); 770 771 if (pdev->vendor == PCI_VENDOR_ID_INTEL) { 772 u16 tmp16; 773 774 /* configure PCS */ 775 pci_read_config_word(pdev, 0x92, &tmp16); 776 tmp16 |= 0xf; 777 pci_write_config_word(pdev, 0x92, tmp16); 778 } 779 780 return 0; 781 } 782 783 static void ahci_init_controller(struct ata_host *host) 784 { 785 struct pci_dev *pdev = to_pci_dev(host->dev); 786 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 787 int i, rc; 788 u32 tmp; 789 790 for (i = 0; i < host->n_ports; i++) { 791 struct ata_port *ap = host->ports[i]; 792 void __iomem *port_mmio = ahci_port_base(ap); 793 const char *emsg = NULL; 794 795 if (ata_port_is_dummy(ap)) 796 continue; 797 798 /* make sure port is not active */ 799 rc = ahci_deinit_port(ap, &emsg); 800 if (rc) 801 dev_printk(KERN_WARNING, &pdev->dev, 802 "%s (%d)\n", emsg, rc); 803 804 /* clear SError */ 805 tmp = readl(port_mmio + PORT_SCR_ERR); 806 VPRINTK("PORT_SCR_ERR 0x%x\n", tmp); 807 writel(tmp, port_mmio + PORT_SCR_ERR); 808 809 /* clear port IRQ */ 810 tmp = readl(port_mmio + PORT_IRQ_STAT); 811 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp); 812 if (tmp) 813 writel(tmp, port_mmio + PORT_IRQ_STAT); 814 815 writel(1 << i, mmio + HOST_IRQ_STAT); 816 } 817 818 tmp = readl(mmio + HOST_CTL); 819 VPRINTK("HOST_CTL 0x%x\n", tmp); 820 writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL); 821 tmp = readl(mmio + HOST_CTL); 822 VPRINTK("HOST_CTL 0x%x\n", tmp); 823 } 824 825 static unsigned int ahci_dev_classify(struct ata_port *ap) 826 { 827 void __iomem *port_mmio = ahci_port_base(ap); 828 struct ata_taskfile tf; 829 u32 tmp; 830 831 tmp = readl(port_mmio + PORT_SIG); 832 tf.lbah = (tmp >> 24) & 0xff; 833 tf.lbam = (tmp >> 16) & 0xff; 834 tf.lbal = (tmp >> 8) & 0xff; 835 tf.nsect = (tmp) & 0xff; 836 837 return ata_dev_classify(&tf); 838 } 839 840 static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag, 841 u32 opts) 842 { 843 dma_addr_t cmd_tbl_dma; 844 845 cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ; 846 847 pp->cmd_slot[tag].opts = cpu_to_le32(opts); 848 pp->cmd_slot[tag].status = 0; 849 pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff); 850 pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16); 851 } 852 853 static int ahci_clo(struct ata_port *ap) 854 { 855 void __iomem *port_mmio = ap->ioaddr.cmd_addr; 856 struct ahci_host_priv *hpriv = ap->host->private_data; 857 u32 tmp; 858 859 if (!(hpriv->cap & HOST_CAP_CLO)) 860 return -EOPNOTSUPP; 861 862 tmp = readl(port_mmio + PORT_CMD); 863 tmp |= PORT_CMD_CLO; 864 writel(tmp, port_mmio + PORT_CMD); 865 866 tmp = ata_wait_register(port_mmio + PORT_CMD, 867 PORT_CMD_CLO, PORT_CMD_CLO, 1, 500); 868 if (tmp & PORT_CMD_CLO) 869 return -EIO; 870 871 return 0; 872 } 873 874 static int ahci_softreset(struct ata_port *ap, unsigned int *class, 875 unsigned long deadline) 876 { 877 struct ahci_port_priv *pp = ap->private_data; 878 void __iomem *port_mmio = ahci_port_base(ap); 879 const u32 cmd_fis_len = 5; /* five dwords */ 880 const char *reason = NULL; 881 struct ata_taskfile tf; 882 u32 tmp; 883 u8 *fis; 884 int rc; 885 886 DPRINTK("ENTER\n"); 887 888 if (ata_port_offline(ap)) { 889 DPRINTK("PHY reports no device\n"); 890 *class = ATA_DEV_NONE; 891 return 0; 892 } 893 894 /* prepare for SRST (AHCI-1.1 10.4.1) */ 895 rc = ahci_stop_engine(ap); 896 if (rc) { 897 reason = "failed to stop engine"; 898 goto fail_restart; 899 } 900 901 /* check BUSY/DRQ, perform Command List Override if necessary */ 902 if (ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ)) { 903 rc = ahci_clo(ap); 904 905 if (rc == -EOPNOTSUPP) { 906 reason = "port busy but CLO unavailable"; 907 goto fail_restart; 908 } else if (rc) { 909 reason = "port busy but CLO failed"; 910 goto fail_restart; 911 } 912 } 913 914 /* restart engine */ 915 ahci_start_engine(ap); 916 917 ata_tf_init(ap->device, &tf); 918 fis = pp->cmd_tbl; 919 920 /* issue the first D2H Register FIS */ 921 ahci_fill_cmd_slot(pp, 0, 922 cmd_fis_len | AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY); 923 924 tf.ctl |= ATA_SRST; 925 ata_tf_to_fis(&tf, fis, 0); 926 fis[1] &= ~(1 << 7); /* turn off Command FIS bit */ 927 928 writel(1, port_mmio + PORT_CMD_ISSUE); 929 930 tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1, 1, 500); 931 if (tmp & 0x1) { 932 rc = -EIO; 933 reason = "1st FIS failed"; 934 goto fail; 935 } 936 937 /* spec says at least 5us, but be generous and sleep for 1ms */ 938 msleep(1); 939 940 /* issue the second D2H Register FIS */ 941 ahci_fill_cmd_slot(pp, 0, cmd_fis_len); 942 943 tf.ctl &= ~ATA_SRST; 944 ata_tf_to_fis(&tf, fis, 0); 945 fis[1] &= ~(1 << 7); /* turn off Command FIS bit */ 946 947 writel(1, port_mmio + PORT_CMD_ISSUE); 948 readl(port_mmio + PORT_CMD_ISSUE); /* flush */ 949 950 /* spec mandates ">= 2ms" before checking status. 951 * We wait 150ms, because that was the magic delay used for 952 * ATAPI devices in Hale Landis's ATADRVR, for the period of time 953 * between when the ATA command register is written, and then 954 * status is checked. Because waiting for "a while" before 955 * checking status is fine, post SRST, we perform this magic 956 * delay here as well. 957 */ 958 msleep(150); 959 960 rc = ata_wait_ready(ap, deadline); 961 /* link occupied, -ENODEV too is an error */ 962 if (rc) { 963 reason = "device not ready"; 964 goto fail; 965 } 966 *class = ahci_dev_classify(ap); 967 968 DPRINTK("EXIT, class=%u\n", *class); 969 return 0; 970 971 fail_restart: 972 ahci_start_engine(ap); 973 fail: 974 ata_port_printk(ap, KERN_ERR, "softreset failed (%s)\n", reason); 975 return rc; 976 } 977 978 static int ahci_hardreset(struct ata_port *ap, unsigned int *class, 979 unsigned long deadline) 980 { 981 struct ahci_port_priv *pp = ap->private_data; 982 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG; 983 struct ata_taskfile tf; 984 int rc; 985 986 DPRINTK("ENTER\n"); 987 988 ahci_stop_engine(ap); 989 990 /* clear D2H reception area to properly wait for D2H FIS */ 991 ata_tf_init(ap->device, &tf); 992 tf.command = 0x80; 993 ata_tf_to_fis(&tf, d2h_fis, 0); 994 995 rc = sata_std_hardreset(ap, class, deadline); 996 997 ahci_start_engine(ap); 998 999 if (rc == 0 && ata_port_online(ap)) 1000 *class = ahci_dev_classify(ap); 1001 if (*class == ATA_DEV_UNKNOWN) 1002 *class = ATA_DEV_NONE; 1003 1004 DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class); 1005 return rc; 1006 } 1007 1008 static int ahci_vt8251_hardreset(struct ata_port *ap, unsigned int *class, 1009 unsigned long deadline) 1010 { 1011 int rc; 1012 1013 DPRINTK("ENTER\n"); 1014 1015 ahci_stop_engine(ap); 1016 1017 rc = sata_port_hardreset(ap, sata_ehc_deb_timing(&ap->eh_context), 1018 deadline); 1019 1020 /* vt8251 needs SError cleared for the port to operate */ 1021 ahci_scr_write(ap, SCR_ERROR, ahci_scr_read(ap, SCR_ERROR)); 1022 1023 ahci_start_engine(ap); 1024 1025 DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class); 1026 1027 /* vt8251 doesn't clear BSY on signature FIS reception, 1028 * request follow-up softreset. 1029 */ 1030 return rc ?: -EAGAIN; 1031 } 1032 1033 static void ahci_postreset(struct ata_port *ap, unsigned int *class) 1034 { 1035 void __iomem *port_mmio = ahci_port_base(ap); 1036 u32 new_tmp, tmp; 1037 1038 ata_std_postreset(ap, class); 1039 1040 /* Make sure port's ATAPI bit is set appropriately */ 1041 new_tmp = tmp = readl(port_mmio + PORT_CMD); 1042 if (*class == ATA_DEV_ATAPI) 1043 new_tmp |= PORT_CMD_ATAPI; 1044 else 1045 new_tmp &= ~PORT_CMD_ATAPI; 1046 if (new_tmp != tmp) { 1047 writel(new_tmp, port_mmio + PORT_CMD); 1048 readl(port_mmio + PORT_CMD); /* flush */ 1049 } 1050 } 1051 1052 static u8 ahci_check_status(struct ata_port *ap) 1053 { 1054 void __iomem *mmio = ap->ioaddr.cmd_addr; 1055 1056 return readl(mmio + PORT_TFDATA) & 0xFF; 1057 } 1058 1059 static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf) 1060 { 1061 struct ahci_port_priv *pp = ap->private_data; 1062 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG; 1063 1064 ata_tf_from_fis(d2h_fis, tf); 1065 } 1066 1067 static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl) 1068 { 1069 struct scatterlist *sg; 1070 struct ahci_sg *ahci_sg; 1071 unsigned int n_sg = 0; 1072 1073 VPRINTK("ENTER\n"); 1074 1075 /* 1076 * Next, the S/G list. 1077 */ 1078 ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ; 1079 ata_for_each_sg(sg, qc) { 1080 dma_addr_t addr = sg_dma_address(sg); 1081 u32 sg_len = sg_dma_len(sg); 1082 1083 ahci_sg->addr = cpu_to_le32(addr & 0xffffffff); 1084 ahci_sg->addr_hi = cpu_to_le32((addr >> 16) >> 16); 1085 ahci_sg->flags_size = cpu_to_le32(sg_len - 1); 1086 1087 ahci_sg++; 1088 n_sg++; 1089 } 1090 1091 return n_sg; 1092 } 1093 1094 static void ahci_qc_prep(struct ata_queued_cmd *qc) 1095 { 1096 struct ata_port *ap = qc->ap; 1097 struct ahci_port_priv *pp = ap->private_data; 1098 int is_atapi = is_atapi_taskfile(&qc->tf); 1099 void *cmd_tbl; 1100 u32 opts; 1101 const u32 cmd_fis_len = 5; /* five dwords */ 1102 unsigned int n_elem; 1103 1104 /* 1105 * Fill in command table information. First, the header, 1106 * a SATA Register - Host to Device command FIS. 1107 */ 1108 cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ; 1109 1110 ata_tf_to_fis(&qc->tf, cmd_tbl, 0); 1111 if (is_atapi) { 1112 memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32); 1113 memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len); 1114 } 1115 1116 n_elem = 0; 1117 if (qc->flags & ATA_QCFLAG_DMAMAP) 1118 n_elem = ahci_fill_sg(qc, cmd_tbl); 1119 1120 /* 1121 * Fill in command slot information. 1122 */ 1123 opts = cmd_fis_len | n_elem << 16; 1124 if (qc->tf.flags & ATA_TFLAG_WRITE) 1125 opts |= AHCI_CMD_WRITE; 1126 if (is_atapi) 1127 opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH; 1128 1129 ahci_fill_cmd_slot(pp, qc->tag, opts); 1130 } 1131 1132 static void ahci_error_intr(struct ata_port *ap, u32 irq_stat) 1133 { 1134 struct ahci_port_priv *pp = ap->private_data; 1135 struct ata_eh_info *ehi = &ap->eh_info; 1136 unsigned int err_mask = 0, action = 0; 1137 struct ata_queued_cmd *qc; 1138 u32 serror; 1139 1140 ata_ehi_clear_desc(ehi); 1141 1142 /* AHCI needs SError cleared; otherwise, it might lock up */ 1143 serror = ahci_scr_read(ap, SCR_ERROR); 1144 ahci_scr_write(ap, SCR_ERROR, serror); 1145 1146 /* analyze @irq_stat */ 1147 ata_ehi_push_desc(ehi, "irq_stat 0x%08x", irq_stat); 1148 1149 /* some controllers set IRQ_IF_ERR on device errors, ignore it */ 1150 if (ap->flags & AHCI_FLAG_IGN_IRQ_IF_ERR) 1151 irq_stat &= ~PORT_IRQ_IF_ERR; 1152 1153 if (irq_stat & PORT_IRQ_TF_ERR) { 1154 err_mask |= AC_ERR_DEV; 1155 if (ap->flags & AHCI_FLAG_IGN_SERR_INTERNAL) 1156 serror &= ~SERR_INTERNAL; 1157 } 1158 1159 if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) { 1160 err_mask |= AC_ERR_HOST_BUS; 1161 action |= ATA_EH_SOFTRESET; 1162 } 1163 1164 if (irq_stat & PORT_IRQ_IF_ERR) { 1165 err_mask |= AC_ERR_ATA_BUS; 1166 action |= ATA_EH_SOFTRESET; 1167 ata_ehi_push_desc(ehi, ", interface fatal error"); 1168 } 1169 1170 if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) { 1171 ata_ehi_hotplugged(ehi); 1172 ata_ehi_push_desc(ehi, ", %s", irq_stat & PORT_IRQ_CONNECT ? 1173 "connection status changed" : "PHY RDY changed"); 1174 } 1175 1176 if (irq_stat & PORT_IRQ_UNK_FIS) { 1177 u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK); 1178 1179 err_mask |= AC_ERR_HSM; 1180 action |= ATA_EH_SOFTRESET; 1181 ata_ehi_push_desc(ehi, ", unknown FIS %08x %08x %08x %08x", 1182 unk[0], unk[1], unk[2], unk[3]); 1183 } 1184 1185 /* okay, let's hand over to EH */ 1186 ehi->serror |= serror; 1187 ehi->action |= action; 1188 1189 qc = ata_qc_from_tag(ap, ap->active_tag); 1190 if (qc) 1191 qc->err_mask |= err_mask; 1192 else 1193 ehi->err_mask |= err_mask; 1194 1195 if (irq_stat & PORT_IRQ_FREEZE) 1196 ata_port_freeze(ap); 1197 else 1198 ata_port_abort(ap); 1199 } 1200 1201 static void ahci_host_intr(struct ata_port *ap) 1202 { 1203 void __iomem *port_mmio = ap->ioaddr.cmd_addr; 1204 struct ata_eh_info *ehi = &ap->eh_info; 1205 struct ahci_port_priv *pp = ap->private_data; 1206 u32 status, qc_active; 1207 int rc, known_irq = 0; 1208 1209 status = readl(port_mmio + PORT_IRQ_STAT); 1210 writel(status, port_mmio + PORT_IRQ_STAT); 1211 1212 if (unlikely(status & PORT_IRQ_ERROR)) { 1213 ahci_error_intr(ap, status); 1214 return; 1215 } 1216 1217 if (ap->sactive) 1218 qc_active = readl(port_mmio + PORT_SCR_ACT); 1219 else 1220 qc_active = readl(port_mmio + PORT_CMD_ISSUE); 1221 1222 rc = ata_qc_complete_multiple(ap, qc_active, NULL); 1223 if (rc > 0) 1224 return; 1225 if (rc < 0) { 1226 ehi->err_mask |= AC_ERR_HSM; 1227 ehi->action |= ATA_EH_SOFTRESET; 1228 ata_port_freeze(ap); 1229 return; 1230 } 1231 1232 /* hmmm... a spurious interupt */ 1233 1234 /* if !NCQ, ignore. No modern ATA device has broken HSM 1235 * implementation for non-NCQ commands. 1236 */ 1237 if (!ap->sactive) 1238 return; 1239 1240 if (status & PORT_IRQ_D2H_REG_FIS) { 1241 if (!pp->ncq_saw_d2h) 1242 ata_port_printk(ap, KERN_INFO, 1243 "D2H reg with I during NCQ, " 1244 "this message won't be printed again\n"); 1245 pp->ncq_saw_d2h = 1; 1246 known_irq = 1; 1247 } 1248 1249 if (status & PORT_IRQ_DMAS_FIS) { 1250 if (!pp->ncq_saw_dmas) 1251 ata_port_printk(ap, KERN_INFO, 1252 "DMAS FIS during NCQ, " 1253 "this message won't be printed again\n"); 1254 pp->ncq_saw_dmas = 1; 1255 known_irq = 1; 1256 } 1257 1258 if (status & PORT_IRQ_SDB_FIS) { 1259 const __le32 *f = pp->rx_fis + RX_FIS_SDB; 1260 1261 if (le32_to_cpu(f[1])) { 1262 /* SDB FIS containing spurious completions 1263 * might be dangerous, whine and fail commands 1264 * with HSM violation. EH will turn off NCQ 1265 * after several such failures. 1266 */ 1267 ata_ehi_push_desc(ehi, 1268 "spurious completions during NCQ " 1269 "issue=0x%x SAct=0x%x FIS=%08x:%08x", 1270 readl(port_mmio + PORT_CMD_ISSUE), 1271 readl(port_mmio + PORT_SCR_ACT), 1272 le32_to_cpu(f[0]), le32_to_cpu(f[1])); 1273 ehi->err_mask |= AC_ERR_HSM; 1274 ehi->action |= ATA_EH_SOFTRESET; 1275 ata_port_freeze(ap); 1276 } else { 1277 if (!pp->ncq_saw_sdb) 1278 ata_port_printk(ap, KERN_INFO, 1279 "spurious SDB FIS %08x:%08x during NCQ, " 1280 "this message won't be printed again\n", 1281 le32_to_cpu(f[0]), le32_to_cpu(f[1])); 1282 pp->ncq_saw_sdb = 1; 1283 } 1284 known_irq = 1; 1285 } 1286 1287 if (!known_irq) 1288 ata_port_printk(ap, KERN_INFO, "spurious interrupt " 1289 "(irq_stat 0x%x active_tag 0x%x sactive 0x%x)\n", 1290 status, ap->active_tag, ap->sactive); 1291 } 1292 1293 static void ahci_irq_clear(struct ata_port *ap) 1294 { 1295 /* TODO */ 1296 } 1297 1298 static irqreturn_t ahci_interrupt(int irq, void *dev_instance) 1299 { 1300 struct ata_host *host = dev_instance; 1301 struct ahci_host_priv *hpriv; 1302 unsigned int i, handled = 0; 1303 void __iomem *mmio; 1304 u32 irq_stat, irq_ack = 0; 1305 1306 VPRINTK("ENTER\n"); 1307 1308 hpriv = host->private_data; 1309 mmio = host->iomap[AHCI_PCI_BAR]; 1310 1311 /* sigh. 0xffffffff is a valid return from h/w */ 1312 irq_stat = readl(mmio + HOST_IRQ_STAT); 1313 irq_stat &= hpriv->port_map; 1314 if (!irq_stat) 1315 return IRQ_NONE; 1316 1317 spin_lock(&host->lock); 1318 1319 for (i = 0; i < host->n_ports; i++) { 1320 struct ata_port *ap; 1321 1322 if (!(irq_stat & (1 << i))) 1323 continue; 1324 1325 ap = host->ports[i]; 1326 if (ap) { 1327 ahci_host_intr(ap); 1328 VPRINTK("port %u\n", i); 1329 } else { 1330 VPRINTK("port %u (no irq)\n", i); 1331 if (ata_ratelimit()) 1332 dev_printk(KERN_WARNING, host->dev, 1333 "interrupt on disabled port %u\n", i); 1334 } 1335 1336 irq_ack |= (1 << i); 1337 } 1338 1339 if (irq_ack) { 1340 writel(irq_ack, mmio + HOST_IRQ_STAT); 1341 handled = 1; 1342 } 1343 1344 spin_unlock(&host->lock); 1345 1346 VPRINTK("EXIT\n"); 1347 1348 return IRQ_RETVAL(handled); 1349 } 1350 1351 static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc) 1352 { 1353 struct ata_port *ap = qc->ap; 1354 void __iomem *port_mmio = ahci_port_base(ap); 1355 1356 if (qc->tf.protocol == ATA_PROT_NCQ) 1357 writel(1 << qc->tag, port_mmio + PORT_SCR_ACT); 1358 writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE); 1359 readl(port_mmio + PORT_CMD_ISSUE); /* flush */ 1360 1361 return 0; 1362 } 1363 1364 static void ahci_freeze(struct ata_port *ap) 1365 { 1366 void __iomem *port_mmio = ahci_port_base(ap); 1367 1368 /* turn IRQ off */ 1369 writel(0, port_mmio + PORT_IRQ_MASK); 1370 } 1371 1372 static void ahci_thaw(struct ata_port *ap) 1373 { 1374 void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR]; 1375 void __iomem *port_mmio = ahci_port_base(ap); 1376 u32 tmp; 1377 1378 /* clear IRQ */ 1379 tmp = readl(port_mmio + PORT_IRQ_STAT); 1380 writel(tmp, port_mmio + PORT_IRQ_STAT); 1381 writel(1 << ap->port_no, mmio + HOST_IRQ_STAT); 1382 1383 /* turn IRQ back on */ 1384 writel(DEF_PORT_IRQ, port_mmio + PORT_IRQ_MASK); 1385 } 1386 1387 static void ahci_error_handler(struct ata_port *ap) 1388 { 1389 if (!(ap->pflags & ATA_PFLAG_FROZEN)) { 1390 /* restart engine */ 1391 ahci_stop_engine(ap); 1392 ahci_start_engine(ap); 1393 } 1394 1395 /* perform recovery */ 1396 ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_hardreset, 1397 ahci_postreset); 1398 } 1399 1400 static void ahci_vt8251_error_handler(struct ata_port *ap) 1401 { 1402 if (!(ap->pflags & ATA_PFLAG_FROZEN)) { 1403 /* restart engine */ 1404 ahci_stop_engine(ap); 1405 ahci_start_engine(ap); 1406 } 1407 1408 /* perform recovery */ 1409 ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_vt8251_hardreset, 1410 ahci_postreset); 1411 } 1412 1413 static void ahci_post_internal_cmd(struct ata_queued_cmd *qc) 1414 { 1415 struct ata_port *ap = qc->ap; 1416 1417 if (qc->flags & ATA_QCFLAG_FAILED) { 1418 /* make DMA engine forget about the failed command */ 1419 ahci_stop_engine(ap); 1420 ahci_start_engine(ap); 1421 } 1422 } 1423 1424 #ifdef CONFIG_PM 1425 static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg) 1426 { 1427 const char *emsg = NULL; 1428 int rc; 1429 1430 rc = ahci_deinit_port(ap, &emsg); 1431 if (rc == 0) 1432 ahci_power_down(ap); 1433 else { 1434 ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc); 1435 ahci_init_port(ap); 1436 } 1437 1438 return rc; 1439 } 1440 1441 static int ahci_port_resume(struct ata_port *ap) 1442 { 1443 ahci_power_up(ap); 1444 ahci_init_port(ap); 1445 1446 return 0; 1447 } 1448 1449 static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg) 1450 { 1451 struct ata_host *host = dev_get_drvdata(&pdev->dev); 1452 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 1453 u32 ctl; 1454 1455 if (mesg.event == PM_EVENT_SUSPEND) { 1456 /* AHCI spec rev1.1 section 8.3.3: 1457 * Software must disable interrupts prior to requesting a 1458 * transition of the HBA to D3 state. 1459 */ 1460 ctl = readl(mmio + HOST_CTL); 1461 ctl &= ~HOST_IRQ_EN; 1462 writel(ctl, mmio + HOST_CTL); 1463 readl(mmio + HOST_CTL); /* flush */ 1464 } 1465 1466 return ata_pci_device_suspend(pdev, mesg); 1467 } 1468 1469 static int ahci_pci_device_resume(struct pci_dev *pdev) 1470 { 1471 struct ata_host *host = dev_get_drvdata(&pdev->dev); 1472 int rc; 1473 1474 rc = ata_pci_device_do_resume(pdev); 1475 if (rc) 1476 return rc; 1477 1478 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) { 1479 rc = ahci_reset_controller(host); 1480 if (rc) 1481 return rc; 1482 1483 ahci_init_controller(host); 1484 } 1485 1486 ata_host_resume(host); 1487 1488 return 0; 1489 } 1490 #endif 1491 1492 static int ahci_port_start(struct ata_port *ap) 1493 { 1494 struct device *dev = ap->host->dev; 1495 struct ahci_port_priv *pp; 1496 void *mem; 1497 dma_addr_t mem_dma; 1498 int rc; 1499 1500 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL); 1501 if (!pp) 1502 return -ENOMEM; 1503 1504 rc = ata_pad_alloc(ap, dev); 1505 if (rc) 1506 return rc; 1507 1508 mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma, 1509 GFP_KERNEL); 1510 if (!mem) 1511 return -ENOMEM; 1512 memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ); 1513 1514 /* 1515 * First item in chunk of DMA memory: 32-slot command table, 1516 * 32 bytes each in size 1517 */ 1518 pp->cmd_slot = mem; 1519 pp->cmd_slot_dma = mem_dma; 1520 1521 mem += AHCI_CMD_SLOT_SZ; 1522 mem_dma += AHCI_CMD_SLOT_SZ; 1523 1524 /* 1525 * Second item: Received-FIS area 1526 */ 1527 pp->rx_fis = mem; 1528 pp->rx_fis_dma = mem_dma; 1529 1530 mem += AHCI_RX_FIS_SZ; 1531 mem_dma += AHCI_RX_FIS_SZ; 1532 1533 /* 1534 * Third item: data area for storing a single command 1535 * and its scatter-gather table 1536 */ 1537 pp->cmd_tbl = mem; 1538 pp->cmd_tbl_dma = mem_dma; 1539 1540 ap->private_data = pp; 1541 1542 /* power up port */ 1543 ahci_power_up(ap); 1544 1545 /* initialize port */ 1546 ahci_init_port(ap); 1547 1548 return 0; 1549 } 1550 1551 static void ahci_port_stop(struct ata_port *ap) 1552 { 1553 const char *emsg = NULL; 1554 int rc; 1555 1556 /* de-initialize port */ 1557 rc = ahci_deinit_port(ap, &emsg); 1558 if (rc) 1559 ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc); 1560 } 1561 1562 static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac) 1563 { 1564 int rc; 1565 1566 if (using_dac && 1567 !pci_set_dma_mask(pdev, DMA_64BIT_MASK)) { 1568 rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK); 1569 if (rc) { 1570 rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK); 1571 if (rc) { 1572 dev_printk(KERN_ERR, &pdev->dev, 1573 "64-bit DMA enable failed\n"); 1574 return rc; 1575 } 1576 } 1577 } else { 1578 rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK); 1579 if (rc) { 1580 dev_printk(KERN_ERR, &pdev->dev, 1581 "32-bit DMA enable failed\n"); 1582 return rc; 1583 } 1584 rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK); 1585 if (rc) { 1586 dev_printk(KERN_ERR, &pdev->dev, 1587 "32-bit consistent DMA enable failed\n"); 1588 return rc; 1589 } 1590 } 1591 return 0; 1592 } 1593 1594 static void ahci_print_info(struct ata_host *host) 1595 { 1596 struct ahci_host_priv *hpriv = host->private_data; 1597 struct pci_dev *pdev = to_pci_dev(host->dev); 1598 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 1599 u32 vers, cap, impl, speed; 1600 const char *speed_s; 1601 u16 cc; 1602 const char *scc_s; 1603 1604 vers = readl(mmio + HOST_VERSION); 1605 cap = hpriv->cap; 1606 impl = hpriv->port_map; 1607 1608 speed = (cap >> 20) & 0xf; 1609 if (speed == 1) 1610 speed_s = "1.5"; 1611 else if (speed == 2) 1612 speed_s = "3"; 1613 else 1614 speed_s = "?"; 1615 1616 pci_read_config_word(pdev, 0x0a, &cc); 1617 if (cc == PCI_CLASS_STORAGE_IDE) 1618 scc_s = "IDE"; 1619 else if (cc == PCI_CLASS_STORAGE_SATA) 1620 scc_s = "SATA"; 1621 else if (cc == PCI_CLASS_STORAGE_RAID) 1622 scc_s = "RAID"; 1623 else 1624 scc_s = "unknown"; 1625 1626 dev_printk(KERN_INFO, &pdev->dev, 1627 "AHCI %02x%02x.%02x%02x " 1628 "%u slots %u ports %s Gbps 0x%x impl %s mode\n" 1629 , 1630 1631 (vers >> 24) & 0xff, 1632 (vers >> 16) & 0xff, 1633 (vers >> 8) & 0xff, 1634 vers & 0xff, 1635 1636 ((cap >> 8) & 0x1f) + 1, 1637 (cap & 0x1f) + 1, 1638 speed_s, 1639 impl, 1640 scc_s); 1641 1642 dev_printk(KERN_INFO, &pdev->dev, 1643 "flags: " 1644 "%s%s%s%s%s%s" 1645 "%s%s%s%s%s%s%s\n" 1646 , 1647 1648 cap & (1 << 31) ? "64bit " : "", 1649 cap & (1 << 30) ? "ncq " : "", 1650 cap & (1 << 28) ? "ilck " : "", 1651 cap & (1 << 27) ? "stag " : "", 1652 cap & (1 << 26) ? "pm " : "", 1653 cap & (1 << 25) ? "led " : "", 1654 1655 cap & (1 << 24) ? "clo " : "", 1656 cap & (1 << 19) ? "nz " : "", 1657 cap & (1 << 18) ? "only " : "", 1658 cap & (1 << 17) ? "pmp " : "", 1659 cap & (1 << 15) ? "pio " : "", 1660 cap & (1 << 14) ? "slum " : "", 1661 cap & (1 << 13) ? "part " : "" 1662 ); 1663 } 1664 1665 static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 1666 { 1667 static int printed_version; 1668 struct ata_port_info pi = ahci_port_info[ent->driver_data]; 1669 const struct ata_port_info *ppi[] = { &pi, NULL }; 1670 struct device *dev = &pdev->dev; 1671 struct ahci_host_priv *hpriv; 1672 struct ata_host *host; 1673 int i, rc; 1674 1675 VPRINTK("ENTER\n"); 1676 1677 WARN_ON(ATA_MAX_QUEUE > AHCI_MAX_CMDS); 1678 1679 if (!printed_version++) 1680 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n"); 1681 1682 /* acquire resources */ 1683 rc = pcim_enable_device(pdev); 1684 if (rc) 1685 return rc; 1686 1687 rc = pcim_iomap_regions(pdev, 1 << AHCI_PCI_BAR, DRV_NAME); 1688 if (rc == -EBUSY) 1689 pcim_pin_device(pdev); 1690 if (rc) 1691 return rc; 1692 1693 if (pci_enable_msi(pdev)) 1694 pci_intx(pdev, 1); 1695 1696 hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL); 1697 if (!hpriv) 1698 return -ENOMEM; 1699 1700 /* save initial config */ 1701 ahci_save_initial_config(pdev, &pi, hpriv); 1702 1703 /* prepare host */ 1704 if (!(pi.flags & AHCI_FLAG_NO_NCQ) && (hpriv->cap & HOST_CAP_NCQ)) 1705 pi.flags |= ATA_FLAG_NCQ; 1706 1707 host = ata_host_alloc_pinfo(&pdev->dev, ppi, fls(hpriv->port_map)); 1708 if (!host) 1709 return -ENOMEM; 1710 host->iomap = pcim_iomap_table(pdev); 1711 host->private_data = hpriv; 1712 1713 for (i = 0; i < host->n_ports; i++) { 1714 if (hpriv->port_map & (1 << i)) { 1715 struct ata_port *ap = host->ports[i]; 1716 void __iomem *port_mmio = ahci_port_base(ap); 1717 1718 ap->ioaddr.cmd_addr = port_mmio; 1719 ap->ioaddr.scr_addr = port_mmio + PORT_SCR; 1720 } else 1721 host->ports[i]->ops = &ata_dummy_port_ops; 1722 } 1723 1724 /* initialize adapter */ 1725 rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64); 1726 if (rc) 1727 return rc; 1728 1729 rc = ahci_reset_controller(host); 1730 if (rc) 1731 return rc; 1732 1733 ahci_init_controller(host); 1734 ahci_print_info(host); 1735 1736 pci_set_master(pdev); 1737 return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED, 1738 &ahci_sht); 1739 } 1740 1741 static int __init ahci_init(void) 1742 { 1743 return pci_register_driver(&ahci_pci_driver); 1744 } 1745 1746 static void __exit ahci_exit(void) 1747 { 1748 pci_unregister_driver(&ahci_pci_driver); 1749 } 1750 1751 1752 MODULE_AUTHOR("Jeff Garzik"); 1753 MODULE_DESCRIPTION("AHCI SATA low-level driver"); 1754 MODULE_LICENSE("GPL"); 1755 MODULE_DEVICE_TABLE(pci, ahci_pci_tbl); 1756 MODULE_VERSION(DRV_VERSION); 1757 1758 module_init(ahci_init); 1759 module_exit(ahci_exit); 1760