1 /* 2 * sata_nv.c - NVIDIA nForce SATA 3 * 4 * Copyright 2004 NVIDIA Corp. All rights reserved. 5 * Copyright 2004 Andrew Chew 6 * 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2, or (at your option) 11 * any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License 19 * along with this program; see the file COPYING. If not, write to 20 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 21 * 22 * 23 * libata documentation is available via 'make {ps|pdf}docs', 24 * as Documentation/DocBook/libata.* 25 * 26 * No hardware documentation available outside of NVIDIA. 27 * This driver programs the NVIDIA SATA controller in a similar 28 * fashion as with other PCI IDE BMDMA controllers, with a few 29 * NV-specific details such as register offsets, SATA phy location, 30 * hotplug info, etc. 31 * 32 * CK804/MCP04 controllers support an alternate programming interface 33 * similar to the ADMA specification (with some modifications). 34 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still 35 * sent through the legacy interface. 36 * 37 */ 38 39 #include <linux/kernel.h> 40 #include <linux/module.h> 41 #include <linux/gfp.h> 42 #include <linux/pci.h> 43 #include <linux/init.h> 44 #include <linux/blkdev.h> 45 #include <linux/delay.h> 46 #include <linux/interrupt.h> 47 #include <linux/device.h> 48 #include <scsi/scsi_host.h> 49 #include <scsi/scsi_device.h> 50 #include <linux/libata.h> 51 52 #define DRV_NAME "sata_nv" 53 #define DRV_VERSION "3.5" 54 55 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL 56 57 enum { 58 NV_MMIO_BAR = 5, 59 60 NV_PORTS = 2, 61 NV_PIO_MASK = ATA_PIO4, 62 NV_MWDMA_MASK = ATA_MWDMA2, 63 NV_UDMA_MASK = ATA_UDMA6, 64 NV_PORT0_SCR_REG_OFFSET = 0x00, 65 NV_PORT1_SCR_REG_OFFSET = 0x40, 66 67 /* INT_STATUS/ENABLE */ 68 NV_INT_STATUS = 0x10, 69 NV_INT_ENABLE = 0x11, 70 NV_INT_STATUS_CK804 = 0x440, 71 NV_INT_ENABLE_CK804 = 0x441, 72 73 /* INT_STATUS/ENABLE bits */ 74 NV_INT_DEV = 0x01, 75 NV_INT_PM = 0x02, 76 NV_INT_ADDED = 0x04, 77 NV_INT_REMOVED = 0x08, 78 79 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */ 80 81 NV_INT_ALL = 0x0f, 82 NV_INT_MASK = NV_INT_DEV | 83 NV_INT_ADDED | NV_INT_REMOVED, 84 85 /* INT_CONFIG */ 86 NV_INT_CONFIG = 0x12, 87 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI 88 89 // For PCI config register 20 90 NV_MCP_SATA_CFG_20 = 0x50, 91 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04, 92 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17), 93 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16), 94 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14), 95 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12), 96 97 NV_ADMA_MAX_CPBS = 32, 98 NV_ADMA_CPB_SZ = 128, 99 NV_ADMA_APRD_SZ = 16, 100 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) / 101 NV_ADMA_APRD_SZ, 102 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5, 103 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ, 104 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS * 105 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ), 106 107 /* BAR5 offset to ADMA general registers */ 108 NV_ADMA_GEN = 0x400, 109 NV_ADMA_GEN_CTL = 0x00, 110 NV_ADMA_NOTIFIER_CLEAR = 0x30, 111 112 /* BAR5 offset to ADMA ports */ 113 NV_ADMA_PORT = 0x480, 114 115 /* size of ADMA port register space */ 116 NV_ADMA_PORT_SIZE = 0x100, 117 118 /* ADMA port registers */ 119 NV_ADMA_CTL = 0x40, 120 NV_ADMA_CPB_COUNT = 0x42, 121 NV_ADMA_NEXT_CPB_IDX = 0x43, 122 NV_ADMA_STAT = 0x44, 123 NV_ADMA_CPB_BASE_LOW = 0x48, 124 NV_ADMA_CPB_BASE_HIGH = 0x4C, 125 NV_ADMA_APPEND = 0x50, 126 NV_ADMA_NOTIFIER = 0x68, 127 NV_ADMA_NOTIFIER_ERROR = 0x6C, 128 129 /* NV_ADMA_CTL register bits */ 130 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0), 131 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5), 132 NV_ADMA_CTL_GO = (1 << 7), 133 NV_ADMA_CTL_AIEN = (1 << 8), 134 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11), 135 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12), 136 137 /* CPB response flag bits */ 138 NV_CPB_RESP_DONE = (1 << 0), 139 NV_CPB_RESP_ATA_ERR = (1 << 3), 140 NV_CPB_RESP_CMD_ERR = (1 << 4), 141 NV_CPB_RESP_CPB_ERR = (1 << 7), 142 143 /* CPB control flag bits */ 144 NV_CPB_CTL_CPB_VALID = (1 << 0), 145 NV_CPB_CTL_QUEUE = (1 << 1), 146 NV_CPB_CTL_APRD_VALID = (1 << 2), 147 NV_CPB_CTL_IEN = (1 << 3), 148 NV_CPB_CTL_FPDMA = (1 << 4), 149 150 /* APRD flags */ 151 NV_APRD_WRITE = (1 << 1), 152 NV_APRD_END = (1 << 2), 153 NV_APRD_CONT = (1 << 3), 154 155 /* NV_ADMA_STAT flags */ 156 NV_ADMA_STAT_TIMEOUT = (1 << 0), 157 NV_ADMA_STAT_HOTUNPLUG = (1 << 1), 158 NV_ADMA_STAT_HOTPLUG = (1 << 2), 159 NV_ADMA_STAT_CPBERR = (1 << 4), 160 NV_ADMA_STAT_SERROR = (1 << 5), 161 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6), 162 NV_ADMA_STAT_IDLE = (1 << 8), 163 NV_ADMA_STAT_LEGACY = (1 << 9), 164 NV_ADMA_STAT_STOPPED = (1 << 10), 165 NV_ADMA_STAT_DONE = (1 << 12), 166 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR | 167 NV_ADMA_STAT_TIMEOUT, 168 169 /* port flags */ 170 NV_ADMA_PORT_REGISTER_MODE = (1 << 0), 171 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1), 172 173 /* MCP55 reg offset */ 174 NV_CTL_MCP55 = 0x400, 175 NV_INT_STATUS_MCP55 = 0x440, 176 NV_INT_ENABLE_MCP55 = 0x444, 177 NV_NCQ_REG_MCP55 = 0x448, 178 179 /* MCP55 */ 180 NV_INT_ALL_MCP55 = 0xffff, 181 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */ 182 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd, 183 184 /* SWNCQ ENABLE BITS*/ 185 NV_CTL_PRI_SWNCQ = 0x02, 186 NV_CTL_SEC_SWNCQ = 0x04, 187 188 /* SW NCQ status bits*/ 189 NV_SWNCQ_IRQ_DEV = (1 << 0), 190 NV_SWNCQ_IRQ_PM = (1 << 1), 191 NV_SWNCQ_IRQ_ADDED = (1 << 2), 192 NV_SWNCQ_IRQ_REMOVED = (1 << 3), 193 194 NV_SWNCQ_IRQ_BACKOUT = (1 << 4), 195 NV_SWNCQ_IRQ_SDBFIS = (1 << 5), 196 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6), 197 NV_SWNCQ_IRQ_DMASETUP = (1 << 7), 198 199 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED | 200 NV_SWNCQ_IRQ_REMOVED, 201 202 }; 203 204 /* ADMA Physical Region Descriptor - one SG segment */ 205 struct nv_adma_prd { 206 __le64 addr; 207 __le32 len; 208 u8 flags; 209 u8 packet_len; 210 __le16 reserved; 211 }; 212 213 enum nv_adma_regbits { 214 CMDEND = (1 << 15), /* end of command list */ 215 WNB = (1 << 14), /* wait-not-BSY */ 216 IGN = (1 << 13), /* ignore this entry */ 217 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */ 218 DA2 = (1 << (2 + 8)), 219 DA1 = (1 << (1 + 8)), 220 DA0 = (1 << (0 + 8)), 221 }; 222 223 /* ADMA Command Parameter Block 224 The first 5 SG segments are stored inside the Command Parameter Block itself. 225 If there are more than 5 segments the remainder are stored in a separate 226 memory area indicated by next_aprd. */ 227 struct nv_adma_cpb { 228 u8 resp_flags; /* 0 */ 229 u8 reserved1; /* 1 */ 230 u8 ctl_flags; /* 2 */ 231 /* len is length of taskfile in 64 bit words */ 232 u8 len; /* 3 */ 233 u8 tag; /* 4 */ 234 u8 next_cpb_idx; /* 5 */ 235 __le16 reserved2; /* 6-7 */ 236 __le16 tf[12]; /* 8-31 */ 237 struct nv_adma_prd aprd[5]; /* 32-111 */ 238 __le64 next_aprd; /* 112-119 */ 239 __le64 reserved3; /* 120-127 */ 240 }; 241 242 243 struct nv_adma_port_priv { 244 struct nv_adma_cpb *cpb; 245 dma_addr_t cpb_dma; 246 struct nv_adma_prd *aprd; 247 dma_addr_t aprd_dma; 248 void __iomem *ctl_block; 249 void __iomem *gen_block; 250 void __iomem *notifier_clear_block; 251 u64 adma_dma_mask; 252 u8 flags; 253 int last_issue_ncq; 254 }; 255 256 struct nv_host_priv { 257 unsigned long type; 258 }; 259 260 struct defer_queue { 261 u32 defer_bits; 262 unsigned int head; 263 unsigned int tail; 264 unsigned int tag[ATA_MAX_QUEUE]; 265 }; 266 267 enum ncq_saw_flag_list { 268 ncq_saw_d2h = (1U << 0), 269 ncq_saw_dmas = (1U << 1), 270 ncq_saw_sdb = (1U << 2), 271 ncq_saw_backout = (1U << 3), 272 }; 273 274 struct nv_swncq_port_priv { 275 struct ata_bmdma_prd *prd; /* our SG list */ 276 dma_addr_t prd_dma; /* and its DMA mapping */ 277 void __iomem *sactive_block; 278 void __iomem *irq_block; 279 void __iomem *tag_block; 280 u32 qc_active; 281 282 unsigned int last_issue_tag; 283 284 /* fifo circular queue to store deferral command */ 285 struct defer_queue defer_queue; 286 287 /* for NCQ interrupt analysis */ 288 u32 dhfis_bits; 289 u32 dmafis_bits; 290 u32 sdbfis_bits; 291 292 unsigned int ncq_flags; 293 }; 294 295 296 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT))))) 297 298 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent); 299 #ifdef CONFIG_PM 300 static int nv_pci_device_resume(struct pci_dev *pdev); 301 #endif 302 static void nv_ck804_host_stop(struct ata_host *host); 303 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance); 304 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance); 305 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance); 306 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val); 307 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val); 308 309 static int nv_hardreset(struct ata_link *link, unsigned int *class, 310 unsigned long deadline); 311 static void nv_nf2_freeze(struct ata_port *ap); 312 static void nv_nf2_thaw(struct ata_port *ap); 313 static void nv_ck804_freeze(struct ata_port *ap); 314 static void nv_ck804_thaw(struct ata_port *ap); 315 static int nv_adma_slave_config(struct scsi_device *sdev); 316 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc); 317 static void nv_adma_qc_prep(struct ata_queued_cmd *qc); 318 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc); 319 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance); 320 static void nv_adma_irq_clear(struct ata_port *ap); 321 static int nv_adma_port_start(struct ata_port *ap); 322 static void nv_adma_port_stop(struct ata_port *ap); 323 #ifdef CONFIG_PM 324 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg); 325 static int nv_adma_port_resume(struct ata_port *ap); 326 #endif 327 static void nv_adma_freeze(struct ata_port *ap); 328 static void nv_adma_thaw(struct ata_port *ap); 329 static void nv_adma_error_handler(struct ata_port *ap); 330 static void nv_adma_host_stop(struct ata_host *host); 331 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc); 332 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf); 333 334 static void nv_mcp55_thaw(struct ata_port *ap); 335 static void nv_mcp55_freeze(struct ata_port *ap); 336 static void nv_swncq_error_handler(struct ata_port *ap); 337 static int nv_swncq_slave_config(struct scsi_device *sdev); 338 static int nv_swncq_port_start(struct ata_port *ap); 339 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc); 340 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc); 341 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc); 342 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis); 343 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance); 344 #ifdef CONFIG_PM 345 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg); 346 static int nv_swncq_port_resume(struct ata_port *ap); 347 #endif 348 349 enum nv_host_type 350 { 351 GENERIC, 352 NFORCE2, 353 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */ 354 CK804, 355 ADMA, 356 MCP5x, 357 SWNCQ, 358 }; 359 360 static const struct pci_device_id nv_pci_tbl[] = { 361 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), NFORCE2 }, 362 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), NFORCE3 }, 363 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), NFORCE3 }, 364 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), CK804 }, 365 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), CK804 }, 366 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), CK804 }, 367 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), CK804 }, 368 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), MCP5x }, 369 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), MCP5x }, 370 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), MCP5x }, 371 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), MCP5x }, 372 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), GENERIC }, 373 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), GENERIC }, 374 { PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), GENERIC }, 375 376 { } /* terminate list */ 377 }; 378 379 static struct pci_driver nv_pci_driver = { 380 .name = DRV_NAME, 381 .id_table = nv_pci_tbl, 382 .probe = nv_init_one, 383 #ifdef CONFIG_PM 384 .suspend = ata_pci_device_suspend, 385 .resume = nv_pci_device_resume, 386 #endif 387 .remove = ata_pci_remove_one, 388 }; 389 390 static struct scsi_host_template nv_sht = { 391 ATA_BMDMA_SHT(DRV_NAME), 392 }; 393 394 static struct scsi_host_template nv_adma_sht = { 395 ATA_NCQ_SHT(DRV_NAME), 396 .can_queue = NV_ADMA_MAX_CPBS, 397 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN, 398 .dma_boundary = NV_ADMA_DMA_BOUNDARY, 399 .slave_configure = nv_adma_slave_config, 400 }; 401 402 static struct scsi_host_template nv_swncq_sht = { 403 ATA_NCQ_SHT(DRV_NAME), 404 .can_queue = ATA_MAX_QUEUE, 405 .sg_tablesize = LIBATA_MAX_PRD, 406 .dma_boundary = ATA_DMA_BOUNDARY, 407 .slave_configure = nv_swncq_slave_config, 408 }; 409 410 /* 411 * NV SATA controllers have various different problems with hardreset 412 * protocol depending on the specific controller and device. 413 * 414 * GENERIC: 415 * 416 * bko11195 reports that link doesn't come online after hardreset on 417 * generic nv's and there have been several other similar reports on 418 * linux-ide. 419 * 420 * bko12351#c23 reports that warmplug on MCP61 doesn't work with 421 * softreset. 422 * 423 * NF2/3: 424 * 425 * bko3352 reports nf2/3 controllers can't determine device signature 426 * reliably after hardreset. The following thread reports detection 427 * failure on cold boot with the standard debouncing timing. 428 * 429 * http://thread.gmane.org/gmane.linux.ide/34098 430 * 431 * bko12176 reports that hardreset fails to bring up the link during 432 * boot on nf2. 433 * 434 * CK804: 435 * 436 * For initial probing after boot and hot plugging, hardreset mostly 437 * works fine on CK804 but curiously, reprobing on the initial port 438 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg 439 * FIS in somewhat undeterministic way. 440 * 441 * SWNCQ: 442 * 443 * bko12351 reports that when SWNCQ is enabled, for hotplug to work, 444 * hardreset should be used and hardreset can't report proper 445 * signature, which suggests that mcp5x is closer to nf2 as long as 446 * reset quirkiness is concerned. 447 * 448 * bko12703 reports that boot probing fails for intel SSD with 449 * hardreset. Link fails to come online. Softreset works fine. 450 * 451 * The failures are varied but the following patterns seem true for 452 * all flavors. 453 * 454 * - Softreset during boot always works. 455 * 456 * - Hardreset during boot sometimes fails to bring up the link on 457 * certain comibnations and device signature acquisition is 458 * unreliable. 459 * 460 * - Hardreset is often necessary after hotplug. 461 * 462 * So, preferring softreset for boot probing and error handling (as 463 * hardreset might bring down the link) but using hardreset for 464 * post-boot probing should work around the above issues in most 465 * cases. Define nv_hardreset() which only kicks in for post-boot 466 * probing and use it for all variants. 467 */ 468 static struct ata_port_operations nv_generic_ops = { 469 .inherits = &ata_bmdma_port_ops, 470 .lost_interrupt = ATA_OP_NULL, 471 .scr_read = nv_scr_read, 472 .scr_write = nv_scr_write, 473 .hardreset = nv_hardreset, 474 }; 475 476 static struct ata_port_operations nv_nf2_ops = { 477 .inherits = &nv_generic_ops, 478 .freeze = nv_nf2_freeze, 479 .thaw = nv_nf2_thaw, 480 }; 481 482 static struct ata_port_operations nv_ck804_ops = { 483 .inherits = &nv_generic_ops, 484 .freeze = nv_ck804_freeze, 485 .thaw = nv_ck804_thaw, 486 .host_stop = nv_ck804_host_stop, 487 }; 488 489 static struct ata_port_operations nv_adma_ops = { 490 .inherits = &nv_ck804_ops, 491 492 .check_atapi_dma = nv_adma_check_atapi_dma, 493 .sff_tf_read = nv_adma_tf_read, 494 .qc_defer = ata_std_qc_defer, 495 .qc_prep = nv_adma_qc_prep, 496 .qc_issue = nv_adma_qc_issue, 497 .sff_irq_clear = nv_adma_irq_clear, 498 499 .freeze = nv_adma_freeze, 500 .thaw = nv_adma_thaw, 501 .error_handler = nv_adma_error_handler, 502 .post_internal_cmd = nv_adma_post_internal_cmd, 503 504 .port_start = nv_adma_port_start, 505 .port_stop = nv_adma_port_stop, 506 #ifdef CONFIG_PM 507 .port_suspend = nv_adma_port_suspend, 508 .port_resume = nv_adma_port_resume, 509 #endif 510 .host_stop = nv_adma_host_stop, 511 }; 512 513 static struct ata_port_operations nv_swncq_ops = { 514 .inherits = &nv_generic_ops, 515 516 .qc_defer = ata_std_qc_defer, 517 .qc_prep = nv_swncq_qc_prep, 518 .qc_issue = nv_swncq_qc_issue, 519 520 .freeze = nv_mcp55_freeze, 521 .thaw = nv_mcp55_thaw, 522 .error_handler = nv_swncq_error_handler, 523 524 #ifdef CONFIG_PM 525 .port_suspend = nv_swncq_port_suspend, 526 .port_resume = nv_swncq_port_resume, 527 #endif 528 .port_start = nv_swncq_port_start, 529 }; 530 531 struct nv_pi_priv { 532 irq_handler_t irq_handler; 533 struct scsi_host_template *sht; 534 }; 535 536 #define NV_PI_PRIV(_irq_handler, _sht) \ 537 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht } 538 539 static const struct ata_port_info nv_port_info[] = { 540 /* generic */ 541 { 542 .flags = ATA_FLAG_SATA, 543 .pio_mask = NV_PIO_MASK, 544 .mwdma_mask = NV_MWDMA_MASK, 545 .udma_mask = NV_UDMA_MASK, 546 .port_ops = &nv_generic_ops, 547 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht), 548 }, 549 /* nforce2/3 */ 550 { 551 .flags = ATA_FLAG_SATA, 552 .pio_mask = NV_PIO_MASK, 553 .mwdma_mask = NV_MWDMA_MASK, 554 .udma_mask = NV_UDMA_MASK, 555 .port_ops = &nv_nf2_ops, 556 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht), 557 }, 558 /* ck804 */ 559 { 560 .flags = ATA_FLAG_SATA, 561 .pio_mask = NV_PIO_MASK, 562 .mwdma_mask = NV_MWDMA_MASK, 563 .udma_mask = NV_UDMA_MASK, 564 .port_ops = &nv_ck804_ops, 565 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht), 566 }, 567 /* ADMA */ 568 { 569 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ, 570 .pio_mask = NV_PIO_MASK, 571 .mwdma_mask = NV_MWDMA_MASK, 572 .udma_mask = NV_UDMA_MASK, 573 .port_ops = &nv_adma_ops, 574 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht), 575 }, 576 /* MCP5x */ 577 { 578 .flags = ATA_FLAG_SATA, 579 .pio_mask = NV_PIO_MASK, 580 .mwdma_mask = NV_MWDMA_MASK, 581 .udma_mask = NV_UDMA_MASK, 582 .port_ops = &nv_generic_ops, 583 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht), 584 }, 585 /* SWNCQ */ 586 { 587 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ, 588 .pio_mask = NV_PIO_MASK, 589 .mwdma_mask = NV_MWDMA_MASK, 590 .udma_mask = NV_UDMA_MASK, 591 .port_ops = &nv_swncq_ops, 592 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht), 593 }, 594 }; 595 596 MODULE_AUTHOR("NVIDIA"); 597 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller"); 598 MODULE_LICENSE("GPL"); 599 MODULE_DEVICE_TABLE(pci, nv_pci_tbl); 600 MODULE_VERSION(DRV_VERSION); 601 602 static bool adma_enabled; 603 static bool swncq_enabled = 1; 604 static bool msi_enabled; 605 606 static void nv_adma_register_mode(struct ata_port *ap) 607 { 608 struct nv_adma_port_priv *pp = ap->private_data; 609 void __iomem *mmio = pp->ctl_block; 610 u16 tmp, status; 611 int count = 0; 612 613 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) 614 return; 615 616 status = readw(mmio + NV_ADMA_STAT); 617 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) { 618 ndelay(50); 619 status = readw(mmio + NV_ADMA_STAT); 620 count++; 621 } 622 if (count == 20) 623 ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n", 624 status); 625 626 tmp = readw(mmio + NV_ADMA_CTL); 627 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL); 628 629 count = 0; 630 status = readw(mmio + NV_ADMA_STAT); 631 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) { 632 ndelay(50); 633 status = readw(mmio + NV_ADMA_STAT); 634 count++; 635 } 636 if (count == 20) 637 ata_port_warn(ap, 638 "timeout waiting for ADMA LEGACY, stat=0x%hx\n", 639 status); 640 641 pp->flags |= NV_ADMA_PORT_REGISTER_MODE; 642 } 643 644 static void nv_adma_mode(struct ata_port *ap) 645 { 646 struct nv_adma_port_priv *pp = ap->private_data; 647 void __iomem *mmio = pp->ctl_block; 648 u16 tmp, status; 649 int count = 0; 650 651 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) 652 return; 653 654 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE); 655 656 tmp = readw(mmio + NV_ADMA_CTL); 657 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL); 658 659 status = readw(mmio + NV_ADMA_STAT); 660 while (((status & NV_ADMA_STAT_LEGACY) || 661 !(status & NV_ADMA_STAT_IDLE)) && count < 20) { 662 ndelay(50); 663 status = readw(mmio + NV_ADMA_STAT); 664 count++; 665 } 666 if (count == 20) 667 ata_port_warn(ap, 668 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n", 669 status); 670 671 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE; 672 } 673 674 static int nv_adma_slave_config(struct scsi_device *sdev) 675 { 676 struct ata_port *ap = ata_shost_to_port(sdev->host); 677 struct nv_adma_port_priv *pp = ap->private_data; 678 struct nv_adma_port_priv *port0, *port1; 679 struct scsi_device *sdev0, *sdev1; 680 struct pci_dev *pdev = to_pci_dev(ap->host->dev); 681 unsigned long segment_boundary, flags; 682 unsigned short sg_tablesize; 683 int rc; 684 int adma_enable; 685 u32 current_reg, new_reg, config_mask; 686 687 rc = ata_scsi_slave_config(sdev); 688 689 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun) 690 /* Not a proper libata device, ignore */ 691 return rc; 692 693 spin_lock_irqsave(ap->lock, flags); 694 695 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) { 696 /* 697 * NVIDIA reports that ADMA mode does not support ATAPI commands. 698 * Therefore ATAPI commands are sent through the legacy interface. 699 * However, the legacy interface only supports 32-bit DMA. 700 * Restrict DMA parameters as required by the legacy interface 701 * when an ATAPI device is connected. 702 */ 703 segment_boundary = ATA_DMA_BOUNDARY; 704 /* Subtract 1 since an extra entry may be needed for padding, see 705 libata-scsi.c */ 706 sg_tablesize = LIBATA_MAX_PRD - 1; 707 708 /* Since the legacy DMA engine is in use, we need to disable ADMA 709 on the port. */ 710 adma_enable = 0; 711 nv_adma_register_mode(ap); 712 } else { 713 segment_boundary = NV_ADMA_DMA_BOUNDARY; 714 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN; 715 adma_enable = 1; 716 } 717 718 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, ¤t_reg); 719 720 if (ap->port_no == 1) 721 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN | 722 NV_MCP_SATA_CFG_20_PORT1_PWB_EN; 723 else 724 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN | 725 NV_MCP_SATA_CFG_20_PORT0_PWB_EN; 726 727 if (adma_enable) { 728 new_reg = current_reg | config_mask; 729 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE; 730 } else { 731 new_reg = current_reg & ~config_mask; 732 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE; 733 } 734 735 if (current_reg != new_reg) 736 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg); 737 738 port0 = ap->host->ports[0]->private_data; 739 port1 = ap->host->ports[1]->private_data; 740 sdev0 = ap->host->ports[0]->link.device[0].sdev; 741 sdev1 = ap->host->ports[1]->link.device[0].sdev; 742 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) || 743 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) { 744 /** We have to set the DMA mask to 32-bit if either port is in 745 ATAPI mode, since they are on the same PCI device which is 746 used for DMA mapping. If we set the mask we also need to set 747 the bounce limit on both ports to ensure that the block 748 layer doesn't feed addresses that cause DMA mapping to 749 choke. If either SCSI device is not allocated yet, it's OK 750 since that port will discover its correct setting when it 751 does get allocated. 752 Note: Setting 32-bit mask should not fail. */ 753 if (sdev0) 754 blk_queue_bounce_limit(sdev0->request_queue, 755 ATA_DMA_MASK); 756 if (sdev1) 757 blk_queue_bounce_limit(sdev1->request_queue, 758 ATA_DMA_MASK); 759 760 pci_set_dma_mask(pdev, ATA_DMA_MASK); 761 } else { 762 /** This shouldn't fail as it was set to this value before */ 763 pci_set_dma_mask(pdev, pp->adma_dma_mask); 764 if (sdev0) 765 blk_queue_bounce_limit(sdev0->request_queue, 766 pp->adma_dma_mask); 767 if (sdev1) 768 blk_queue_bounce_limit(sdev1->request_queue, 769 pp->adma_dma_mask); 770 } 771 772 blk_queue_segment_boundary(sdev->request_queue, segment_boundary); 773 blk_queue_max_segments(sdev->request_queue, sg_tablesize); 774 ata_port_info(ap, 775 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n", 776 (unsigned long long)*ap->host->dev->dma_mask, 777 segment_boundary, sg_tablesize); 778 779 spin_unlock_irqrestore(ap->lock, flags); 780 781 return rc; 782 } 783 784 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc) 785 { 786 struct nv_adma_port_priv *pp = qc->ap->private_data; 787 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE); 788 } 789 790 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf) 791 { 792 /* Other than when internal or pass-through commands are executed, 793 the only time this function will be called in ADMA mode will be 794 if a command fails. In the failure case we don't care about going 795 into register mode with ADMA commands pending, as the commands will 796 all shortly be aborted anyway. We assume that NCQ commands are not 797 issued via passthrough, which is the only way that switching into 798 ADMA mode could abort outstanding commands. */ 799 nv_adma_register_mode(ap); 800 801 ata_sff_tf_read(ap, tf); 802 } 803 804 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb) 805 { 806 unsigned int idx = 0; 807 808 if (tf->flags & ATA_TFLAG_ISADDR) { 809 if (tf->flags & ATA_TFLAG_LBA48) { 810 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB); 811 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect); 812 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal); 813 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam); 814 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah); 815 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature); 816 } else 817 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB); 818 819 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect); 820 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal); 821 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam); 822 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah); 823 } 824 825 if (tf->flags & ATA_TFLAG_DEVICE) 826 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device); 827 828 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND); 829 830 while (idx < 12) 831 cpb[idx++] = cpu_to_le16(IGN); 832 833 return idx; 834 } 835 836 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err) 837 { 838 struct nv_adma_port_priv *pp = ap->private_data; 839 u8 flags = pp->cpb[cpb_num].resp_flags; 840 841 VPRINTK("CPB %d, flags=0x%x\n", cpb_num, flags); 842 843 if (unlikely((force_err || 844 flags & (NV_CPB_RESP_ATA_ERR | 845 NV_CPB_RESP_CMD_ERR | 846 NV_CPB_RESP_CPB_ERR)))) { 847 struct ata_eh_info *ehi = &ap->link.eh_info; 848 int freeze = 0; 849 850 ata_ehi_clear_desc(ehi); 851 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags); 852 if (flags & NV_CPB_RESP_ATA_ERR) { 853 ata_ehi_push_desc(ehi, "ATA error"); 854 ehi->err_mask |= AC_ERR_DEV; 855 } else if (flags & NV_CPB_RESP_CMD_ERR) { 856 ata_ehi_push_desc(ehi, "CMD error"); 857 ehi->err_mask |= AC_ERR_DEV; 858 } else if (flags & NV_CPB_RESP_CPB_ERR) { 859 ata_ehi_push_desc(ehi, "CPB error"); 860 ehi->err_mask |= AC_ERR_SYSTEM; 861 freeze = 1; 862 } else { 863 /* notifier error, but no error in CPB flags? */ 864 ata_ehi_push_desc(ehi, "unknown"); 865 ehi->err_mask |= AC_ERR_OTHER; 866 freeze = 1; 867 } 868 /* Kill all commands. EH will determine what actually failed. */ 869 if (freeze) 870 ata_port_freeze(ap); 871 else 872 ata_port_abort(ap); 873 return -1; 874 } 875 876 if (likely(flags & NV_CPB_RESP_DONE)) 877 return 1; 878 return 0; 879 } 880 881 static int nv_host_intr(struct ata_port *ap, u8 irq_stat) 882 { 883 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag); 884 885 /* freeze if hotplugged */ 886 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) { 887 ata_port_freeze(ap); 888 return 1; 889 } 890 891 /* bail out if not our interrupt */ 892 if (!(irq_stat & NV_INT_DEV)) 893 return 0; 894 895 /* DEV interrupt w/ no active qc? */ 896 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) { 897 ata_sff_check_status(ap); 898 return 1; 899 } 900 901 /* handle interrupt */ 902 return ata_bmdma_port_intr(ap, qc); 903 } 904 905 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance) 906 { 907 struct ata_host *host = dev_instance; 908 int i, handled = 0; 909 u32 notifier_clears[2]; 910 911 spin_lock(&host->lock); 912 913 for (i = 0; i < host->n_ports; i++) { 914 struct ata_port *ap = host->ports[i]; 915 struct nv_adma_port_priv *pp = ap->private_data; 916 void __iomem *mmio = pp->ctl_block; 917 u16 status; 918 u32 gen_ctl; 919 u32 notifier, notifier_error; 920 921 notifier_clears[i] = 0; 922 923 /* if ADMA is disabled, use standard ata interrupt handler */ 924 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) { 925 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804) 926 >> (NV_INT_PORT_SHIFT * i); 927 handled += nv_host_intr(ap, irq_stat); 928 continue; 929 } 930 931 /* if in ATA register mode, check for standard interrupts */ 932 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) { 933 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804) 934 >> (NV_INT_PORT_SHIFT * i); 935 if (ata_tag_valid(ap->link.active_tag)) 936 /** NV_INT_DEV indication seems unreliable 937 at times at least in ADMA mode. Force it 938 on always when a command is active, to 939 prevent losing interrupts. */ 940 irq_stat |= NV_INT_DEV; 941 handled += nv_host_intr(ap, irq_stat); 942 } 943 944 notifier = readl(mmio + NV_ADMA_NOTIFIER); 945 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR); 946 notifier_clears[i] = notifier | notifier_error; 947 948 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL); 949 950 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier && 951 !notifier_error) 952 /* Nothing to do */ 953 continue; 954 955 status = readw(mmio + NV_ADMA_STAT); 956 957 /* 958 * Clear status. Ensure the controller sees the 959 * clearing before we start looking at any of the CPB 960 * statuses, so that any CPB completions after this 961 * point in the handler will raise another interrupt. 962 */ 963 writew(status, mmio + NV_ADMA_STAT); 964 readw(mmio + NV_ADMA_STAT); /* flush posted write */ 965 rmb(); 966 967 handled++; /* irq handled if we got here */ 968 969 /* freeze if hotplugged or controller error */ 970 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG | 971 NV_ADMA_STAT_HOTUNPLUG | 972 NV_ADMA_STAT_TIMEOUT | 973 NV_ADMA_STAT_SERROR))) { 974 struct ata_eh_info *ehi = &ap->link.eh_info; 975 976 ata_ehi_clear_desc(ehi); 977 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status); 978 if (status & NV_ADMA_STAT_TIMEOUT) { 979 ehi->err_mask |= AC_ERR_SYSTEM; 980 ata_ehi_push_desc(ehi, "timeout"); 981 } else if (status & NV_ADMA_STAT_HOTPLUG) { 982 ata_ehi_hotplugged(ehi); 983 ata_ehi_push_desc(ehi, "hotplug"); 984 } else if (status & NV_ADMA_STAT_HOTUNPLUG) { 985 ata_ehi_hotplugged(ehi); 986 ata_ehi_push_desc(ehi, "hot unplug"); 987 } else if (status & NV_ADMA_STAT_SERROR) { 988 /* let EH analyze SError and figure out cause */ 989 ata_ehi_push_desc(ehi, "SError"); 990 } else 991 ata_ehi_push_desc(ehi, "unknown"); 992 ata_port_freeze(ap); 993 continue; 994 } 995 996 if (status & (NV_ADMA_STAT_DONE | 997 NV_ADMA_STAT_CPBERR | 998 NV_ADMA_STAT_CMD_COMPLETE)) { 999 u32 check_commands = notifier_clears[i]; 1000 u32 done_mask = 0; 1001 int pos, rc; 1002 1003 if (status & NV_ADMA_STAT_CPBERR) { 1004 /* check all active commands */ 1005 if (ata_tag_valid(ap->link.active_tag)) 1006 check_commands = 1 << 1007 ap->link.active_tag; 1008 else 1009 check_commands = ap->link.sactive; 1010 } 1011 1012 /* check CPBs for completed commands */ 1013 while ((pos = ffs(check_commands))) { 1014 pos--; 1015 rc = nv_adma_check_cpb(ap, pos, 1016 notifier_error & (1 << pos)); 1017 if (rc > 0) 1018 done_mask |= 1 << pos; 1019 else if (unlikely(rc < 0)) 1020 check_commands = 0; 1021 check_commands &= ~(1 << pos); 1022 } 1023 ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask); 1024 } 1025 } 1026 1027 if (notifier_clears[0] || notifier_clears[1]) { 1028 /* Note: Both notifier clear registers must be written 1029 if either is set, even if one is zero, according to NVIDIA. */ 1030 struct nv_adma_port_priv *pp = host->ports[0]->private_data; 1031 writel(notifier_clears[0], pp->notifier_clear_block); 1032 pp = host->ports[1]->private_data; 1033 writel(notifier_clears[1], pp->notifier_clear_block); 1034 } 1035 1036 spin_unlock(&host->lock); 1037 1038 return IRQ_RETVAL(handled); 1039 } 1040 1041 static void nv_adma_freeze(struct ata_port *ap) 1042 { 1043 struct nv_adma_port_priv *pp = ap->private_data; 1044 void __iomem *mmio = pp->ctl_block; 1045 u16 tmp; 1046 1047 nv_ck804_freeze(ap); 1048 1049 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) 1050 return; 1051 1052 /* clear any outstanding CK804 notifications */ 1053 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT), 1054 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804); 1055 1056 /* Disable interrupt */ 1057 tmp = readw(mmio + NV_ADMA_CTL); 1058 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN), 1059 mmio + NV_ADMA_CTL); 1060 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1061 } 1062 1063 static void nv_adma_thaw(struct ata_port *ap) 1064 { 1065 struct nv_adma_port_priv *pp = ap->private_data; 1066 void __iomem *mmio = pp->ctl_block; 1067 u16 tmp; 1068 1069 nv_ck804_thaw(ap); 1070 1071 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) 1072 return; 1073 1074 /* Enable interrupt */ 1075 tmp = readw(mmio + NV_ADMA_CTL); 1076 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN), 1077 mmio + NV_ADMA_CTL); 1078 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1079 } 1080 1081 static void nv_adma_irq_clear(struct ata_port *ap) 1082 { 1083 struct nv_adma_port_priv *pp = ap->private_data; 1084 void __iomem *mmio = pp->ctl_block; 1085 u32 notifier_clears[2]; 1086 1087 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) { 1088 ata_bmdma_irq_clear(ap); 1089 return; 1090 } 1091 1092 /* clear any outstanding CK804 notifications */ 1093 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT), 1094 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804); 1095 1096 /* clear ADMA status */ 1097 writew(0xffff, mmio + NV_ADMA_STAT); 1098 1099 /* clear notifiers - note both ports need to be written with 1100 something even though we are only clearing on one */ 1101 if (ap->port_no == 0) { 1102 notifier_clears[0] = 0xFFFFFFFF; 1103 notifier_clears[1] = 0; 1104 } else { 1105 notifier_clears[0] = 0; 1106 notifier_clears[1] = 0xFFFFFFFF; 1107 } 1108 pp = ap->host->ports[0]->private_data; 1109 writel(notifier_clears[0], pp->notifier_clear_block); 1110 pp = ap->host->ports[1]->private_data; 1111 writel(notifier_clears[1], pp->notifier_clear_block); 1112 } 1113 1114 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc) 1115 { 1116 struct nv_adma_port_priv *pp = qc->ap->private_data; 1117 1118 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) 1119 ata_bmdma_post_internal_cmd(qc); 1120 } 1121 1122 static int nv_adma_port_start(struct ata_port *ap) 1123 { 1124 struct device *dev = ap->host->dev; 1125 struct nv_adma_port_priv *pp; 1126 int rc; 1127 void *mem; 1128 dma_addr_t mem_dma; 1129 void __iomem *mmio; 1130 struct pci_dev *pdev = to_pci_dev(dev); 1131 u16 tmp; 1132 1133 VPRINTK("ENTER\n"); 1134 1135 /* Ensure DMA mask is set to 32-bit before allocating legacy PRD and 1136 pad buffers */ 1137 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 1138 if (rc) 1139 return rc; 1140 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 1141 if (rc) 1142 return rc; 1143 1144 /* we might fallback to bmdma, allocate bmdma resources */ 1145 rc = ata_bmdma_port_start(ap); 1146 if (rc) 1147 return rc; 1148 1149 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL); 1150 if (!pp) 1151 return -ENOMEM; 1152 1153 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT + 1154 ap->port_no * NV_ADMA_PORT_SIZE; 1155 pp->ctl_block = mmio; 1156 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN; 1157 pp->notifier_clear_block = pp->gen_block + 1158 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no); 1159 1160 /* Now that the legacy PRD and padding buffer are allocated we can 1161 safely raise the DMA mask to allocate the CPB/APRD table. 1162 These are allowed to fail since we store the value that ends up 1163 being used to set as the bounce limit in slave_config later if 1164 needed. */ 1165 pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 1166 pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 1167 pp->adma_dma_mask = *dev->dma_mask; 1168 1169 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ, 1170 &mem_dma, GFP_KERNEL); 1171 if (!mem) 1172 return -ENOMEM; 1173 memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ); 1174 1175 /* 1176 * First item in chunk of DMA memory: 1177 * 128-byte command parameter block (CPB) 1178 * one for each command tag 1179 */ 1180 pp->cpb = mem; 1181 pp->cpb_dma = mem_dma; 1182 1183 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW); 1184 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH); 1185 1186 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ; 1187 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ; 1188 1189 /* 1190 * Second item: block of ADMA_SGTBL_LEN s/g entries 1191 */ 1192 pp->aprd = mem; 1193 pp->aprd_dma = mem_dma; 1194 1195 ap->private_data = pp; 1196 1197 /* clear any outstanding interrupt conditions */ 1198 writew(0xffff, mmio + NV_ADMA_STAT); 1199 1200 /* initialize port variables */ 1201 pp->flags = NV_ADMA_PORT_REGISTER_MODE; 1202 1203 /* clear CPB fetch count */ 1204 writew(0, mmio + NV_ADMA_CPB_COUNT); 1205 1206 /* clear GO for register mode, enable interrupt */ 1207 tmp = readw(mmio + NV_ADMA_CTL); 1208 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN | 1209 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL); 1210 1211 tmp = readw(mmio + NV_ADMA_CTL); 1212 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1213 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1214 udelay(1); 1215 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1216 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1217 1218 return 0; 1219 } 1220 1221 static void nv_adma_port_stop(struct ata_port *ap) 1222 { 1223 struct nv_adma_port_priv *pp = ap->private_data; 1224 void __iomem *mmio = pp->ctl_block; 1225 1226 VPRINTK("ENTER\n"); 1227 writew(0, mmio + NV_ADMA_CTL); 1228 } 1229 1230 #ifdef CONFIG_PM 1231 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg) 1232 { 1233 struct nv_adma_port_priv *pp = ap->private_data; 1234 void __iomem *mmio = pp->ctl_block; 1235 1236 /* Go to register mode - clears GO */ 1237 nv_adma_register_mode(ap); 1238 1239 /* clear CPB fetch count */ 1240 writew(0, mmio + NV_ADMA_CPB_COUNT); 1241 1242 /* disable interrupt, shut down port */ 1243 writew(0, mmio + NV_ADMA_CTL); 1244 1245 return 0; 1246 } 1247 1248 static int nv_adma_port_resume(struct ata_port *ap) 1249 { 1250 struct nv_adma_port_priv *pp = ap->private_data; 1251 void __iomem *mmio = pp->ctl_block; 1252 u16 tmp; 1253 1254 /* set CPB block location */ 1255 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW); 1256 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH); 1257 1258 /* clear any outstanding interrupt conditions */ 1259 writew(0xffff, mmio + NV_ADMA_STAT); 1260 1261 /* initialize port variables */ 1262 pp->flags |= NV_ADMA_PORT_REGISTER_MODE; 1263 1264 /* clear CPB fetch count */ 1265 writew(0, mmio + NV_ADMA_CPB_COUNT); 1266 1267 /* clear GO for register mode, enable interrupt */ 1268 tmp = readw(mmio + NV_ADMA_CTL); 1269 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN | 1270 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL); 1271 1272 tmp = readw(mmio + NV_ADMA_CTL); 1273 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1274 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1275 udelay(1); 1276 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1277 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1278 1279 return 0; 1280 } 1281 #endif 1282 1283 static void nv_adma_setup_port(struct ata_port *ap) 1284 { 1285 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR]; 1286 struct ata_ioports *ioport = &ap->ioaddr; 1287 1288 VPRINTK("ENTER\n"); 1289 1290 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE; 1291 1292 ioport->cmd_addr = mmio; 1293 ioport->data_addr = mmio + (ATA_REG_DATA * 4); 1294 ioport->error_addr = 1295 ioport->feature_addr = mmio + (ATA_REG_ERR * 4); 1296 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4); 1297 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4); 1298 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4); 1299 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4); 1300 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4); 1301 ioport->status_addr = 1302 ioport->command_addr = mmio + (ATA_REG_STATUS * 4); 1303 ioport->altstatus_addr = 1304 ioport->ctl_addr = mmio + 0x20; 1305 } 1306 1307 static int nv_adma_host_init(struct ata_host *host) 1308 { 1309 struct pci_dev *pdev = to_pci_dev(host->dev); 1310 unsigned int i; 1311 u32 tmp32; 1312 1313 VPRINTK("ENTER\n"); 1314 1315 /* enable ADMA on the ports */ 1316 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32); 1317 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN | 1318 NV_MCP_SATA_CFG_20_PORT0_PWB_EN | 1319 NV_MCP_SATA_CFG_20_PORT1_EN | 1320 NV_MCP_SATA_CFG_20_PORT1_PWB_EN; 1321 1322 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32); 1323 1324 for (i = 0; i < host->n_ports; i++) 1325 nv_adma_setup_port(host->ports[i]); 1326 1327 return 0; 1328 } 1329 1330 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc, 1331 struct scatterlist *sg, 1332 int idx, 1333 struct nv_adma_prd *aprd) 1334 { 1335 u8 flags = 0; 1336 if (qc->tf.flags & ATA_TFLAG_WRITE) 1337 flags |= NV_APRD_WRITE; 1338 if (idx == qc->n_elem - 1) 1339 flags |= NV_APRD_END; 1340 else if (idx != 4) 1341 flags |= NV_APRD_CONT; 1342 1343 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg))); 1344 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */ 1345 aprd->flags = flags; 1346 aprd->packet_len = 0; 1347 } 1348 1349 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb) 1350 { 1351 struct nv_adma_port_priv *pp = qc->ap->private_data; 1352 struct nv_adma_prd *aprd; 1353 struct scatterlist *sg; 1354 unsigned int si; 1355 1356 VPRINTK("ENTER\n"); 1357 1358 for_each_sg(qc->sg, sg, qc->n_elem, si) { 1359 aprd = (si < 5) ? &cpb->aprd[si] : 1360 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->tag + (si-5)]; 1361 nv_adma_fill_aprd(qc, sg, si, aprd); 1362 } 1363 if (si > 5) 1364 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->tag))); 1365 else 1366 cpb->next_aprd = cpu_to_le64(0); 1367 } 1368 1369 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc) 1370 { 1371 struct nv_adma_port_priv *pp = qc->ap->private_data; 1372 1373 /* ADMA engine can only be used for non-ATAPI DMA commands, 1374 or interrupt-driven no-data commands. */ 1375 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) || 1376 (qc->tf.flags & ATA_TFLAG_POLLING)) 1377 return 1; 1378 1379 if ((qc->flags & ATA_QCFLAG_DMAMAP) || 1380 (qc->tf.protocol == ATA_PROT_NODATA)) 1381 return 0; 1382 1383 return 1; 1384 } 1385 1386 static void nv_adma_qc_prep(struct ata_queued_cmd *qc) 1387 { 1388 struct nv_adma_port_priv *pp = qc->ap->private_data; 1389 struct nv_adma_cpb *cpb = &pp->cpb[qc->tag]; 1390 u8 ctl_flags = NV_CPB_CTL_CPB_VALID | 1391 NV_CPB_CTL_IEN; 1392 1393 if (nv_adma_use_reg_mode(qc)) { 1394 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) && 1395 (qc->flags & ATA_QCFLAG_DMAMAP)); 1396 nv_adma_register_mode(qc->ap); 1397 ata_bmdma_qc_prep(qc); 1398 return; 1399 } 1400 1401 cpb->resp_flags = NV_CPB_RESP_DONE; 1402 wmb(); 1403 cpb->ctl_flags = 0; 1404 wmb(); 1405 1406 cpb->len = 3; 1407 cpb->tag = qc->tag; 1408 cpb->next_cpb_idx = 0; 1409 1410 /* turn on NCQ flags for NCQ commands */ 1411 if (qc->tf.protocol == ATA_PROT_NCQ) 1412 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA; 1413 1414 VPRINTK("qc->flags = 0x%lx\n", qc->flags); 1415 1416 nv_adma_tf_to_cpb(&qc->tf, cpb->tf); 1417 1418 if (qc->flags & ATA_QCFLAG_DMAMAP) { 1419 nv_adma_fill_sg(qc, cpb); 1420 ctl_flags |= NV_CPB_CTL_APRD_VALID; 1421 } else 1422 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5); 1423 1424 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID 1425 until we are finished filling in all of the contents */ 1426 wmb(); 1427 cpb->ctl_flags = ctl_flags; 1428 wmb(); 1429 cpb->resp_flags = 0; 1430 } 1431 1432 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc) 1433 { 1434 struct nv_adma_port_priv *pp = qc->ap->private_data; 1435 void __iomem *mmio = pp->ctl_block; 1436 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ); 1437 1438 VPRINTK("ENTER\n"); 1439 1440 /* We can't handle result taskfile with NCQ commands, since 1441 retrieving the taskfile switches us out of ADMA mode and would abort 1442 existing commands. */ 1443 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ && 1444 (qc->flags & ATA_QCFLAG_RESULT_TF))) { 1445 ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n"); 1446 return AC_ERR_SYSTEM; 1447 } 1448 1449 if (nv_adma_use_reg_mode(qc)) { 1450 /* use ATA register mode */ 1451 VPRINTK("using ATA register mode: 0x%lx\n", qc->flags); 1452 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) && 1453 (qc->flags & ATA_QCFLAG_DMAMAP)); 1454 nv_adma_register_mode(qc->ap); 1455 return ata_bmdma_qc_issue(qc); 1456 } else 1457 nv_adma_mode(qc->ap); 1458 1459 /* write append register, command tag in lower 8 bits 1460 and (number of cpbs to append -1) in top 8 bits */ 1461 wmb(); 1462 1463 if (curr_ncq != pp->last_issue_ncq) { 1464 /* Seems to need some delay before switching between NCQ and 1465 non-NCQ commands, else we get command timeouts and such. */ 1466 udelay(20); 1467 pp->last_issue_ncq = curr_ncq; 1468 } 1469 1470 writew(qc->tag, mmio + NV_ADMA_APPEND); 1471 1472 DPRINTK("Issued tag %u\n", qc->tag); 1473 1474 return 0; 1475 } 1476 1477 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance) 1478 { 1479 struct ata_host *host = dev_instance; 1480 unsigned int i; 1481 unsigned int handled = 0; 1482 unsigned long flags; 1483 1484 spin_lock_irqsave(&host->lock, flags); 1485 1486 for (i = 0; i < host->n_ports; i++) { 1487 struct ata_port *ap = host->ports[i]; 1488 struct ata_queued_cmd *qc; 1489 1490 qc = ata_qc_from_tag(ap, ap->link.active_tag); 1491 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) { 1492 handled += ata_bmdma_port_intr(ap, qc); 1493 } else { 1494 /* 1495 * No request pending? Clear interrupt status 1496 * anyway, in case there's one pending. 1497 */ 1498 ap->ops->sff_check_status(ap); 1499 } 1500 } 1501 1502 spin_unlock_irqrestore(&host->lock, flags); 1503 1504 return IRQ_RETVAL(handled); 1505 } 1506 1507 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat) 1508 { 1509 int i, handled = 0; 1510 1511 for (i = 0; i < host->n_ports; i++) { 1512 handled += nv_host_intr(host->ports[i], irq_stat); 1513 irq_stat >>= NV_INT_PORT_SHIFT; 1514 } 1515 1516 return IRQ_RETVAL(handled); 1517 } 1518 1519 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance) 1520 { 1521 struct ata_host *host = dev_instance; 1522 u8 irq_stat; 1523 irqreturn_t ret; 1524 1525 spin_lock(&host->lock); 1526 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS); 1527 ret = nv_do_interrupt(host, irq_stat); 1528 spin_unlock(&host->lock); 1529 1530 return ret; 1531 } 1532 1533 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance) 1534 { 1535 struct ata_host *host = dev_instance; 1536 u8 irq_stat; 1537 irqreturn_t ret; 1538 1539 spin_lock(&host->lock); 1540 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804); 1541 ret = nv_do_interrupt(host, irq_stat); 1542 spin_unlock(&host->lock); 1543 1544 return ret; 1545 } 1546 1547 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val) 1548 { 1549 if (sc_reg > SCR_CONTROL) 1550 return -EINVAL; 1551 1552 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4)); 1553 return 0; 1554 } 1555 1556 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val) 1557 { 1558 if (sc_reg > SCR_CONTROL) 1559 return -EINVAL; 1560 1561 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4)); 1562 return 0; 1563 } 1564 1565 static int nv_hardreset(struct ata_link *link, unsigned int *class, 1566 unsigned long deadline) 1567 { 1568 struct ata_eh_context *ehc = &link->eh_context; 1569 1570 /* Do hardreset iff it's post-boot probing, please read the 1571 * comment above port ops for details. 1572 */ 1573 if (!(link->ap->pflags & ATA_PFLAG_LOADING) && 1574 !ata_dev_enabled(link->device)) 1575 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline, 1576 NULL, NULL); 1577 else { 1578 const unsigned long *timing = sata_ehc_deb_timing(ehc); 1579 int rc; 1580 1581 if (!(ehc->i.flags & ATA_EHI_QUIET)) 1582 ata_link_info(link, 1583 "nv: skipping hardreset on occupied port\n"); 1584 1585 /* make sure the link is online */ 1586 rc = sata_link_resume(link, timing, deadline); 1587 /* whine about phy resume failure but proceed */ 1588 if (rc && rc != -EOPNOTSUPP) 1589 ata_link_warn(link, "failed to resume link (errno=%d)\n", 1590 rc); 1591 } 1592 1593 /* device signature acquisition is unreliable */ 1594 return -EAGAIN; 1595 } 1596 1597 static void nv_nf2_freeze(struct ata_port *ap) 1598 { 1599 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr; 1600 int shift = ap->port_no * NV_INT_PORT_SHIFT; 1601 u8 mask; 1602 1603 mask = ioread8(scr_addr + NV_INT_ENABLE); 1604 mask &= ~(NV_INT_ALL << shift); 1605 iowrite8(mask, scr_addr + NV_INT_ENABLE); 1606 } 1607 1608 static void nv_nf2_thaw(struct ata_port *ap) 1609 { 1610 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr; 1611 int shift = ap->port_no * NV_INT_PORT_SHIFT; 1612 u8 mask; 1613 1614 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS); 1615 1616 mask = ioread8(scr_addr + NV_INT_ENABLE); 1617 mask |= (NV_INT_MASK << shift); 1618 iowrite8(mask, scr_addr + NV_INT_ENABLE); 1619 } 1620 1621 static void nv_ck804_freeze(struct ata_port *ap) 1622 { 1623 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR]; 1624 int shift = ap->port_no * NV_INT_PORT_SHIFT; 1625 u8 mask; 1626 1627 mask = readb(mmio_base + NV_INT_ENABLE_CK804); 1628 mask &= ~(NV_INT_ALL << shift); 1629 writeb(mask, mmio_base + NV_INT_ENABLE_CK804); 1630 } 1631 1632 static void nv_ck804_thaw(struct ata_port *ap) 1633 { 1634 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR]; 1635 int shift = ap->port_no * NV_INT_PORT_SHIFT; 1636 u8 mask; 1637 1638 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804); 1639 1640 mask = readb(mmio_base + NV_INT_ENABLE_CK804); 1641 mask |= (NV_INT_MASK << shift); 1642 writeb(mask, mmio_base + NV_INT_ENABLE_CK804); 1643 } 1644 1645 static void nv_mcp55_freeze(struct ata_port *ap) 1646 { 1647 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR]; 1648 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55; 1649 u32 mask; 1650 1651 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55); 1652 1653 mask = readl(mmio_base + NV_INT_ENABLE_MCP55); 1654 mask &= ~(NV_INT_ALL_MCP55 << shift); 1655 writel(mask, mmio_base + NV_INT_ENABLE_MCP55); 1656 } 1657 1658 static void nv_mcp55_thaw(struct ata_port *ap) 1659 { 1660 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR]; 1661 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55; 1662 u32 mask; 1663 1664 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55); 1665 1666 mask = readl(mmio_base + NV_INT_ENABLE_MCP55); 1667 mask |= (NV_INT_MASK_MCP55 << shift); 1668 writel(mask, mmio_base + NV_INT_ENABLE_MCP55); 1669 } 1670 1671 static void nv_adma_error_handler(struct ata_port *ap) 1672 { 1673 struct nv_adma_port_priv *pp = ap->private_data; 1674 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) { 1675 void __iomem *mmio = pp->ctl_block; 1676 int i; 1677 u16 tmp; 1678 1679 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) { 1680 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER); 1681 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR); 1682 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL); 1683 u32 status = readw(mmio + NV_ADMA_STAT); 1684 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT); 1685 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX); 1686 1687 ata_port_err(ap, 1688 "EH in ADMA mode, notifier 0x%X " 1689 "notifier_error 0x%X gen_ctl 0x%X status 0x%X " 1690 "next cpb count 0x%X next cpb idx 0x%x\n", 1691 notifier, notifier_error, gen_ctl, status, 1692 cpb_count, next_cpb_idx); 1693 1694 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) { 1695 struct nv_adma_cpb *cpb = &pp->cpb[i]; 1696 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) || 1697 ap->link.sactive & (1 << i)) 1698 ata_port_err(ap, 1699 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n", 1700 i, cpb->ctl_flags, cpb->resp_flags); 1701 } 1702 } 1703 1704 /* Push us back into port register mode for error handling. */ 1705 nv_adma_register_mode(ap); 1706 1707 /* Mark all of the CPBs as invalid to prevent them from 1708 being executed */ 1709 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) 1710 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID; 1711 1712 /* clear CPB fetch count */ 1713 writew(0, mmio + NV_ADMA_CPB_COUNT); 1714 1715 /* Reset channel */ 1716 tmp = readw(mmio + NV_ADMA_CTL); 1717 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1718 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1719 udelay(1); 1720 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1721 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1722 } 1723 1724 ata_bmdma_error_handler(ap); 1725 } 1726 1727 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc) 1728 { 1729 struct nv_swncq_port_priv *pp = ap->private_data; 1730 struct defer_queue *dq = &pp->defer_queue; 1731 1732 /* queue is full */ 1733 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE); 1734 dq->defer_bits |= (1 << qc->tag); 1735 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->tag; 1736 } 1737 1738 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap) 1739 { 1740 struct nv_swncq_port_priv *pp = ap->private_data; 1741 struct defer_queue *dq = &pp->defer_queue; 1742 unsigned int tag; 1743 1744 if (dq->head == dq->tail) /* null queue */ 1745 return NULL; 1746 1747 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)]; 1748 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON; 1749 WARN_ON(!(dq->defer_bits & (1 << tag))); 1750 dq->defer_bits &= ~(1 << tag); 1751 1752 return ata_qc_from_tag(ap, tag); 1753 } 1754 1755 static void nv_swncq_fis_reinit(struct ata_port *ap) 1756 { 1757 struct nv_swncq_port_priv *pp = ap->private_data; 1758 1759 pp->dhfis_bits = 0; 1760 pp->dmafis_bits = 0; 1761 pp->sdbfis_bits = 0; 1762 pp->ncq_flags = 0; 1763 } 1764 1765 static void nv_swncq_pp_reinit(struct ata_port *ap) 1766 { 1767 struct nv_swncq_port_priv *pp = ap->private_data; 1768 struct defer_queue *dq = &pp->defer_queue; 1769 1770 dq->head = 0; 1771 dq->tail = 0; 1772 dq->defer_bits = 0; 1773 pp->qc_active = 0; 1774 pp->last_issue_tag = ATA_TAG_POISON; 1775 nv_swncq_fis_reinit(ap); 1776 } 1777 1778 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis) 1779 { 1780 struct nv_swncq_port_priv *pp = ap->private_data; 1781 1782 writew(fis, pp->irq_block); 1783 } 1784 1785 static void __ata_bmdma_stop(struct ata_port *ap) 1786 { 1787 struct ata_queued_cmd qc; 1788 1789 qc.ap = ap; 1790 ata_bmdma_stop(&qc); 1791 } 1792 1793 static void nv_swncq_ncq_stop(struct ata_port *ap) 1794 { 1795 struct nv_swncq_port_priv *pp = ap->private_data; 1796 unsigned int i; 1797 u32 sactive; 1798 u32 done_mask; 1799 1800 ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%X sactive 0x%X\n", 1801 ap->qc_active, ap->link.sactive); 1802 ata_port_err(ap, 1803 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n " 1804 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n", 1805 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag, 1806 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits); 1807 1808 ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n", 1809 ap->ops->sff_check_status(ap), 1810 ioread8(ap->ioaddr.error_addr)); 1811 1812 sactive = readl(pp->sactive_block); 1813 done_mask = pp->qc_active ^ sactive; 1814 1815 ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n"); 1816 for (i = 0; i < ATA_MAX_QUEUE; i++) { 1817 u8 err = 0; 1818 if (pp->qc_active & (1 << i)) 1819 err = 0; 1820 else if (done_mask & (1 << i)) 1821 err = 1; 1822 else 1823 continue; 1824 1825 ata_port_err(ap, 1826 "tag 0x%x: %01x %01x %01x %01x %s\n", i, 1827 (pp->dhfis_bits >> i) & 0x1, 1828 (pp->dmafis_bits >> i) & 0x1, 1829 (pp->sdbfis_bits >> i) & 0x1, 1830 (sactive >> i) & 0x1, 1831 (err ? "error! tag doesn't exit" : " ")); 1832 } 1833 1834 nv_swncq_pp_reinit(ap); 1835 ap->ops->sff_irq_clear(ap); 1836 __ata_bmdma_stop(ap); 1837 nv_swncq_irq_clear(ap, 0xffff); 1838 } 1839 1840 static void nv_swncq_error_handler(struct ata_port *ap) 1841 { 1842 struct ata_eh_context *ehc = &ap->link.eh_context; 1843 1844 if (ap->link.sactive) { 1845 nv_swncq_ncq_stop(ap); 1846 ehc->i.action |= ATA_EH_RESET; 1847 } 1848 1849 ata_bmdma_error_handler(ap); 1850 } 1851 1852 #ifdef CONFIG_PM 1853 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg) 1854 { 1855 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR]; 1856 u32 tmp; 1857 1858 /* clear irq */ 1859 writel(~0, mmio + NV_INT_STATUS_MCP55); 1860 1861 /* disable irq */ 1862 writel(0, mmio + NV_INT_ENABLE_MCP55); 1863 1864 /* disable swncq */ 1865 tmp = readl(mmio + NV_CTL_MCP55); 1866 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ); 1867 writel(tmp, mmio + NV_CTL_MCP55); 1868 1869 return 0; 1870 } 1871 1872 static int nv_swncq_port_resume(struct ata_port *ap) 1873 { 1874 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR]; 1875 u32 tmp; 1876 1877 /* clear irq */ 1878 writel(~0, mmio + NV_INT_STATUS_MCP55); 1879 1880 /* enable irq */ 1881 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55); 1882 1883 /* enable swncq */ 1884 tmp = readl(mmio + NV_CTL_MCP55); 1885 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55); 1886 1887 return 0; 1888 } 1889 #endif 1890 1891 static void nv_swncq_host_init(struct ata_host *host) 1892 { 1893 u32 tmp; 1894 void __iomem *mmio = host->iomap[NV_MMIO_BAR]; 1895 struct pci_dev *pdev = to_pci_dev(host->dev); 1896 u8 regval; 1897 1898 /* disable ECO 398 */ 1899 pci_read_config_byte(pdev, 0x7f, ®val); 1900 regval &= ~(1 << 7); 1901 pci_write_config_byte(pdev, 0x7f, regval); 1902 1903 /* enable swncq */ 1904 tmp = readl(mmio + NV_CTL_MCP55); 1905 VPRINTK("HOST_CTL:0x%X\n", tmp); 1906 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55); 1907 1908 /* enable irq intr */ 1909 tmp = readl(mmio + NV_INT_ENABLE_MCP55); 1910 VPRINTK("HOST_ENABLE:0x%X\n", tmp); 1911 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55); 1912 1913 /* clear port irq */ 1914 writel(~0x0, mmio + NV_INT_STATUS_MCP55); 1915 } 1916 1917 static int nv_swncq_slave_config(struct scsi_device *sdev) 1918 { 1919 struct ata_port *ap = ata_shost_to_port(sdev->host); 1920 struct pci_dev *pdev = to_pci_dev(ap->host->dev); 1921 struct ata_device *dev; 1922 int rc; 1923 u8 rev; 1924 u8 check_maxtor = 0; 1925 unsigned char model_num[ATA_ID_PROD_LEN + 1]; 1926 1927 rc = ata_scsi_slave_config(sdev); 1928 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun) 1929 /* Not a proper libata device, ignore */ 1930 return rc; 1931 1932 dev = &ap->link.device[sdev->id]; 1933 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI) 1934 return rc; 1935 1936 /* if MCP51 and Maxtor, then disable ncq */ 1937 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA || 1938 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2) 1939 check_maxtor = 1; 1940 1941 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */ 1942 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA || 1943 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) { 1944 pci_read_config_byte(pdev, 0x8, &rev); 1945 if (rev <= 0xa2) 1946 check_maxtor = 1; 1947 } 1948 1949 if (!check_maxtor) 1950 return rc; 1951 1952 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num)); 1953 1954 if (strncmp(model_num, "Maxtor", 6) == 0) { 1955 ata_scsi_change_queue_depth(sdev, 1, SCSI_QDEPTH_DEFAULT); 1956 ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n", 1957 sdev->queue_depth); 1958 } 1959 1960 return rc; 1961 } 1962 1963 static int nv_swncq_port_start(struct ata_port *ap) 1964 { 1965 struct device *dev = ap->host->dev; 1966 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR]; 1967 struct nv_swncq_port_priv *pp; 1968 int rc; 1969 1970 /* we might fallback to bmdma, allocate bmdma resources */ 1971 rc = ata_bmdma_port_start(ap); 1972 if (rc) 1973 return rc; 1974 1975 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL); 1976 if (!pp) 1977 return -ENOMEM; 1978 1979 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE, 1980 &pp->prd_dma, GFP_KERNEL); 1981 if (!pp->prd) 1982 return -ENOMEM; 1983 memset(pp->prd, 0, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE); 1984 1985 ap->private_data = pp; 1986 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE; 1987 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2; 1988 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2; 1989 1990 return 0; 1991 } 1992 1993 static void nv_swncq_qc_prep(struct ata_queued_cmd *qc) 1994 { 1995 if (qc->tf.protocol != ATA_PROT_NCQ) { 1996 ata_bmdma_qc_prep(qc); 1997 return; 1998 } 1999 2000 if (!(qc->flags & ATA_QCFLAG_DMAMAP)) 2001 return; 2002 2003 nv_swncq_fill_sg(qc); 2004 } 2005 2006 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc) 2007 { 2008 struct ata_port *ap = qc->ap; 2009 struct scatterlist *sg; 2010 struct nv_swncq_port_priv *pp = ap->private_data; 2011 struct ata_bmdma_prd *prd; 2012 unsigned int si, idx; 2013 2014 prd = pp->prd + ATA_MAX_PRD * qc->tag; 2015 2016 idx = 0; 2017 for_each_sg(qc->sg, sg, qc->n_elem, si) { 2018 u32 addr, offset; 2019 u32 sg_len, len; 2020 2021 addr = (u32)sg_dma_address(sg); 2022 sg_len = sg_dma_len(sg); 2023 2024 while (sg_len) { 2025 offset = addr & 0xffff; 2026 len = sg_len; 2027 if ((offset + sg_len) > 0x10000) 2028 len = 0x10000 - offset; 2029 2030 prd[idx].addr = cpu_to_le32(addr); 2031 prd[idx].flags_len = cpu_to_le32(len & 0xffff); 2032 2033 idx++; 2034 sg_len -= len; 2035 addr += len; 2036 } 2037 } 2038 2039 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT); 2040 } 2041 2042 static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap, 2043 struct ata_queued_cmd *qc) 2044 { 2045 struct nv_swncq_port_priv *pp = ap->private_data; 2046 2047 if (qc == NULL) 2048 return 0; 2049 2050 DPRINTK("Enter\n"); 2051 2052 writel((1 << qc->tag), pp->sactive_block); 2053 pp->last_issue_tag = qc->tag; 2054 pp->dhfis_bits &= ~(1 << qc->tag); 2055 pp->dmafis_bits &= ~(1 << qc->tag); 2056 pp->qc_active |= (0x1 << qc->tag); 2057 2058 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */ 2059 ap->ops->sff_exec_command(ap, &qc->tf); 2060 2061 DPRINTK("Issued tag %u\n", qc->tag); 2062 2063 return 0; 2064 } 2065 2066 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc) 2067 { 2068 struct ata_port *ap = qc->ap; 2069 struct nv_swncq_port_priv *pp = ap->private_data; 2070 2071 if (qc->tf.protocol != ATA_PROT_NCQ) 2072 return ata_bmdma_qc_issue(qc); 2073 2074 DPRINTK("Enter\n"); 2075 2076 if (!pp->qc_active) 2077 nv_swncq_issue_atacmd(ap, qc); 2078 else 2079 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */ 2080 2081 return 0; 2082 } 2083 2084 static void nv_swncq_hotplug(struct ata_port *ap, u32 fis) 2085 { 2086 u32 serror; 2087 struct ata_eh_info *ehi = &ap->link.eh_info; 2088 2089 ata_ehi_clear_desc(ehi); 2090 2091 /* AHCI needs SError cleared; otherwise, it might lock up */ 2092 sata_scr_read(&ap->link, SCR_ERROR, &serror); 2093 sata_scr_write(&ap->link, SCR_ERROR, serror); 2094 2095 /* analyze @irq_stat */ 2096 if (fis & NV_SWNCQ_IRQ_ADDED) 2097 ata_ehi_push_desc(ehi, "hot plug"); 2098 else if (fis & NV_SWNCQ_IRQ_REMOVED) 2099 ata_ehi_push_desc(ehi, "hot unplug"); 2100 2101 ata_ehi_hotplugged(ehi); 2102 2103 /* okay, let's hand over to EH */ 2104 ehi->serror |= serror; 2105 2106 ata_port_freeze(ap); 2107 } 2108 2109 static int nv_swncq_sdbfis(struct ata_port *ap) 2110 { 2111 struct ata_queued_cmd *qc; 2112 struct nv_swncq_port_priv *pp = ap->private_data; 2113 struct ata_eh_info *ehi = &ap->link.eh_info; 2114 u32 sactive; 2115 u32 done_mask; 2116 u8 host_stat; 2117 u8 lack_dhfis = 0; 2118 2119 host_stat = ap->ops->bmdma_status(ap); 2120 if (unlikely(host_stat & ATA_DMA_ERR)) { 2121 /* error when transferring data to/from memory */ 2122 ata_ehi_clear_desc(ehi); 2123 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat); 2124 ehi->err_mask |= AC_ERR_HOST_BUS; 2125 ehi->action |= ATA_EH_RESET; 2126 return -EINVAL; 2127 } 2128 2129 ap->ops->sff_irq_clear(ap); 2130 __ata_bmdma_stop(ap); 2131 2132 sactive = readl(pp->sactive_block); 2133 done_mask = pp->qc_active ^ sactive; 2134 2135 pp->qc_active &= ~done_mask; 2136 pp->dhfis_bits &= ~done_mask; 2137 pp->dmafis_bits &= ~done_mask; 2138 pp->sdbfis_bits |= done_mask; 2139 ata_qc_complete_multiple(ap, ap->qc_active ^ done_mask); 2140 2141 if (!ap->qc_active) { 2142 DPRINTK("over\n"); 2143 nv_swncq_pp_reinit(ap); 2144 return 0; 2145 } 2146 2147 if (pp->qc_active & pp->dhfis_bits) 2148 return 0; 2149 2150 if ((pp->ncq_flags & ncq_saw_backout) || 2151 (pp->qc_active ^ pp->dhfis_bits)) 2152 /* if the controller can't get a device to host register FIS, 2153 * The driver needs to reissue the new command. 2154 */ 2155 lack_dhfis = 1; 2156 2157 DPRINTK("id 0x%x QC: qc_active 0x%x," 2158 "SWNCQ:qc_active 0x%X defer_bits %X " 2159 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n", 2160 ap->print_id, ap->qc_active, pp->qc_active, 2161 pp->defer_queue.defer_bits, pp->dhfis_bits, 2162 pp->dmafis_bits, pp->last_issue_tag); 2163 2164 nv_swncq_fis_reinit(ap); 2165 2166 if (lack_dhfis) { 2167 qc = ata_qc_from_tag(ap, pp->last_issue_tag); 2168 nv_swncq_issue_atacmd(ap, qc); 2169 return 0; 2170 } 2171 2172 if (pp->defer_queue.defer_bits) { 2173 /* send deferral queue command */ 2174 qc = nv_swncq_qc_from_dq(ap); 2175 WARN_ON(qc == NULL); 2176 nv_swncq_issue_atacmd(ap, qc); 2177 } 2178 2179 return 0; 2180 } 2181 2182 static inline u32 nv_swncq_tag(struct ata_port *ap) 2183 { 2184 struct nv_swncq_port_priv *pp = ap->private_data; 2185 u32 tag; 2186 2187 tag = readb(pp->tag_block) >> 2; 2188 return (tag & 0x1f); 2189 } 2190 2191 static void nv_swncq_dmafis(struct ata_port *ap) 2192 { 2193 struct ata_queued_cmd *qc; 2194 unsigned int rw; 2195 u8 dmactl; 2196 u32 tag; 2197 struct nv_swncq_port_priv *pp = ap->private_data; 2198 2199 __ata_bmdma_stop(ap); 2200 tag = nv_swncq_tag(ap); 2201 2202 DPRINTK("dma setup tag 0x%x\n", tag); 2203 qc = ata_qc_from_tag(ap, tag); 2204 2205 if (unlikely(!qc)) 2206 return; 2207 2208 rw = qc->tf.flags & ATA_TFLAG_WRITE; 2209 2210 /* load PRD table addr. */ 2211 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->tag, 2212 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS); 2213 2214 /* specify data direction, triple-check start bit is clear */ 2215 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD); 2216 dmactl &= ~ATA_DMA_WR; 2217 if (!rw) 2218 dmactl |= ATA_DMA_WR; 2219 2220 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD); 2221 } 2222 2223 static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis) 2224 { 2225 struct nv_swncq_port_priv *pp = ap->private_data; 2226 struct ata_queued_cmd *qc; 2227 struct ata_eh_info *ehi = &ap->link.eh_info; 2228 u32 serror; 2229 u8 ata_stat; 2230 2231 ata_stat = ap->ops->sff_check_status(ap); 2232 nv_swncq_irq_clear(ap, fis); 2233 if (!fis) 2234 return; 2235 2236 if (ap->pflags & ATA_PFLAG_FROZEN) 2237 return; 2238 2239 if (fis & NV_SWNCQ_IRQ_HOTPLUG) { 2240 nv_swncq_hotplug(ap, fis); 2241 return; 2242 } 2243 2244 if (!pp->qc_active) 2245 return; 2246 2247 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror)) 2248 return; 2249 ap->ops->scr_write(&ap->link, SCR_ERROR, serror); 2250 2251 if (ata_stat & ATA_ERR) { 2252 ata_ehi_clear_desc(ehi); 2253 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis); 2254 ehi->err_mask |= AC_ERR_DEV; 2255 ehi->serror |= serror; 2256 ehi->action |= ATA_EH_RESET; 2257 ata_port_freeze(ap); 2258 return; 2259 } 2260 2261 if (fis & NV_SWNCQ_IRQ_BACKOUT) { 2262 /* If the IRQ is backout, driver must issue 2263 * the new command again some time later. 2264 */ 2265 pp->ncq_flags |= ncq_saw_backout; 2266 } 2267 2268 if (fis & NV_SWNCQ_IRQ_SDBFIS) { 2269 pp->ncq_flags |= ncq_saw_sdb; 2270 DPRINTK("id 0x%x SWNCQ: qc_active 0x%X " 2271 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n", 2272 ap->print_id, pp->qc_active, pp->dhfis_bits, 2273 pp->dmafis_bits, readl(pp->sactive_block)); 2274 if (nv_swncq_sdbfis(ap) < 0) 2275 goto irq_error; 2276 } 2277 2278 if (fis & NV_SWNCQ_IRQ_DHREGFIS) { 2279 /* The interrupt indicates the new command 2280 * was transmitted correctly to the drive. 2281 */ 2282 pp->dhfis_bits |= (0x1 << pp->last_issue_tag); 2283 pp->ncq_flags |= ncq_saw_d2h; 2284 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) { 2285 ata_ehi_push_desc(ehi, "illegal fis transaction"); 2286 ehi->err_mask |= AC_ERR_HSM; 2287 ehi->action |= ATA_EH_RESET; 2288 goto irq_error; 2289 } 2290 2291 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) && 2292 !(pp->ncq_flags & ncq_saw_dmas)) { 2293 ata_stat = ap->ops->sff_check_status(ap); 2294 if (ata_stat & ATA_BUSY) 2295 goto irq_exit; 2296 2297 if (pp->defer_queue.defer_bits) { 2298 DPRINTK("send next command\n"); 2299 qc = nv_swncq_qc_from_dq(ap); 2300 nv_swncq_issue_atacmd(ap, qc); 2301 } 2302 } 2303 } 2304 2305 if (fis & NV_SWNCQ_IRQ_DMASETUP) { 2306 /* program the dma controller with appropriate PRD buffers 2307 * and start the DMA transfer for requested command. 2308 */ 2309 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap)); 2310 pp->ncq_flags |= ncq_saw_dmas; 2311 nv_swncq_dmafis(ap); 2312 } 2313 2314 irq_exit: 2315 return; 2316 irq_error: 2317 ata_ehi_push_desc(ehi, "fis:0x%x", fis); 2318 ata_port_freeze(ap); 2319 return; 2320 } 2321 2322 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance) 2323 { 2324 struct ata_host *host = dev_instance; 2325 unsigned int i; 2326 unsigned int handled = 0; 2327 unsigned long flags; 2328 u32 irq_stat; 2329 2330 spin_lock_irqsave(&host->lock, flags); 2331 2332 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55); 2333 2334 for (i = 0; i < host->n_ports; i++) { 2335 struct ata_port *ap = host->ports[i]; 2336 2337 if (ap->link.sactive) { 2338 nv_swncq_host_interrupt(ap, (u16)irq_stat); 2339 handled = 1; 2340 } else { 2341 if (irq_stat) /* reserve Hotplug */ 2342 nv_swncq_irq_clear(ap, 0xfff0); 2343 2344 handled += nv_host_intr(ap, (u8)irq_stat); 2345 } 2346 irq_stat >>= NV_INT_PORT_SHIFT_MCP55; 2347 } 2348 2349 spin_unlock_irqrestore(&host->lock, flags); 2350 2351 return IRQ_RETVAL(handled); 2352 } 2353 2354 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 2355 { 2356 const struct ata_port_info *ppi[] = { NULL, NULL }; 2357 struct nv_pi_priv *ipriv; 2358 struct ata_host *host; 2359 struct nv_host_priv *hpriv; 2360 int rc; 2361 u32 bar; 2362 void __iomem *base; 2363 unsigned long type = ent->driver_data; 2364 2365 // Make sure this is a SATA controller by counting the number of bars 2366 // (NVIDIA SATA controllers will always have six bars). Otherwise, 2367 // it's an IDE controller and we ignore it. 2368 for (bar = 0; bar < 6; bar++) 2369 if (pci_resource_start(pdev, bar) == 0) 2370 return -ENODEV; 2371 2372 ata_print_version_once(&pdev->dev, DRV_VERSION); 2373 2374 rc = pcim_enable_device(pdev); 2375 if (rc) 2376 return rc; 2377 2378 /* determine type and allocate host */ 2379 if (type == CK804 && adma_enabled) { 2380 dev_notice(&pdev->dev, "Using ADMA mode\n"); 2381 type = ADMA; 2382 } else if (type == MCP5x && swncq_enabled) { 2383 dev_notice(&pdev->dev, "Using SWNCQ mode\n"); 2384 type = SWNCQ; 2385 } 2386 2387 ppi[0] = &nv_port_info[type]; 2388 ipriv = ppi[0]->private_data; 2389 rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host); 2390 if (rc) 2391 return rc; 2392 2393 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL); 2394 if (!hpriv) 2395 return -ENOMEM; 2396 hpriv->type = type; 2397 host->private_data = hpriv; 2398 2399 /* request and iomap NV_MMIO_BAR */ 2400 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME); 2401 if (rc) 2402 return rc; 2403 2404 /* configure SCR access */ 2405 base = host->iomap[NV_MMIO_BAR]; 2406 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET; 2407 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET; 2408 2409 /* enable SATA space for CK804 */ 2410 if (type >= CK804) { 2411 u8 regval; 2412 2413 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val); 2414 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN; 2415 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval); 2416 } 2417 2418 /* init ADMA */ 2419 if (type == ADMA) { 2420 rc = nv_adma_host_init(host); 2421 if (rc) 2422 return rc; 2423 } else if (type == SWNCQ) 2424 nv_swncq_host_init(host); 2425 2426 if (msi_enabled) { 2427 dev_notice(&pdev->dev, "Using MSI\n"); 2428 pci_enable_msi(pdev); 2429 } 2430 2431 pci_set_master(pdev); 2432 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht); 2433 } 2434 2435 #ifdef CONFIG_PM 2436 static int nv_pci_device_resume(struct pci_dev *pdev) 2437 { 2438 struct ata_host *host = pci_get_drvdata(pdev); 2439 struct nv_host_priv *hpriv = host->private_data; 2440 int rc; 2441 2442 rc = ata_pci_device_do_resume(pdev); 2443 if (rc) 2444 return rc; 2445 2446 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) { 2447 if (hpriv->type >= CK804) { 2448 u8 regval; 2449 2450 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val); 2451 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN; 2452 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval); 2453 } 2454 if (hpriv->type == ADMA) { 2455 u32 tmp32; 2456 struct nv_adma_port_priv *pp; 2457 /* enable/disable ADMA on the ports appropriately */ 2458 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32); 2459 2460 pp = host->ports[0]->private_data; 2461 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) 2462 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN | 2463 NV_MCP_SATA_CFG_20_PORT0_PWB_EN); 2464 else 2465 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN | 2466 NV_MCP_SATA_CFG_20_PORT0_PWB_EN); 2467 pp = host->ports[1]->private_data; 2468 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) 2469 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN | 2470 NV_MCP_SATA_CFG_20_PORT1_PWB_EN); 2471 else 2472 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN | 2473 NV_MCP_SATA_CFG_20_PORT1_PWB_EN); 2474 2475 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32); 2476 } 2477 } 2478 2479 ata_host_resume(host); 2480 2481 return 0; 2482 } 2483 #endif 2484 2485 static void nv_ck804_host_stop(struct ata_host *host) 2486 { 2487 struct pci_dev *pdev = to_pci_dev(host->dev); 2488 u8 regval; 2489 2490 /* disable SATA space for CK804 */ 2491 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val); 2492 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN; 2493 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval); 2494 } 2495 2496 static void nv_adma_host_stop(struct ata_host *host) 2497 { 2498 struct pci_dev *pdev = to_pci_dev(host->dev); 2499 u32 tmp32; 2500 2501 /* disable ADMA on the ports */ 2502 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32); 2503 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN | 2504 NV_MCP_SATA_CFG_20_PORT0_PWB_EN | 2505 NV_MCP_SATA_CFG_20_PORT1_EN | 2506 NV_MCP_SATA_CFG_20_PORT1_PWB_EN); 2507 2508 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32); 2509 2510 nv_ck804_host_stop(host); 2511 } 2512 2513 module_pci_driver(nv_pci_driver); 2514 2515 module_param_named(adma, adma_enabled, bool, 0444); 2516 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)"); 2517 module_param_named(swncq, swncq_enabled, bool, 0444); 2518 MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)"); 2519 module_param_named(msi, msi_enabled, bool, 0444); 2520 MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)"); 2521