xref: /openbmc/linux/drivers/ata/ahci.c (revision e868d61272caa648214046a096e5a6bfc068dc8c)
1 /*
2  *  ahci.c - AHCI SATA support
3  *
4  *  Maintained by:  Jeff Garzik <jgarzik@pobox.com>
5  *    		    Please ALWAYS copy linux-ide@vger.kernel.org
6  *		    on emails.
7  *
8  *  Copyright 2004-2005 Red Hat, Inc.
9  *
10  *
11  *  This program is free software; you can redistribute it and/or modify
12  *  it under the terms of the GNU General Public License as published by
13  *  the Free Software Foundation; either version 2, or (at your option)
14  *  any later version.
15  *
16  *  This program is distributed in the hope that it will be useful,
17  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
18  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19  *  GNU General Public License for more details.
20  *
21  *  You should have received a copy of the GNU General Public License
22  *  along with this program; see the file COPYING.  If not, write to
23  *  the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
24  *
25  *
26  * libata documentation is available via 'make {ps|pdf}docs',
27  * as Documentation/DocBook/libata.*
28  *
29  * AHCI hardware documentation:
30  * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
31  * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
32  *
33  */
34 
35 #include <linux/kernel.h>
36 #include <linux/module.h>
37 #include <linux/pci.h>
38 #include <linux/init.h>
39 #include <linux/blkdev.h>
40 #include <linux/delay.h>
41 #include <linux/interrupt.h>
42 #include <linux/dma-mapping.h>
43 #include <linux/device.h>
44 #include <scsi/scsi_host.h>
45 #include <scsi/scsi_cmnd.h>
46 #include <linux/libata.h>
47 
48 #define DRV_NAME	"ahci"
49 #define DRV_VERSION	"2.1"
50 
51 
52 enum {
53 	AHCI_PCI_BAR		= 5,
54 	AHCI_MAX_PORTS		= 32,
55 	AHCI_MAX_SG		= 168, /* hardware max is 64K */
56 	AHCI_DMA_BOUNDARY	= 0xffffffff,
57 	AHCI_USE_CLUSTERING	= 0,
58 	AHCI_MAX_CMDS		= 32,
59 	AHCI_CMD_SZ		= 32,
60 	AHCI_CMD_SLOT_SZ	= AHCI_MAX_CMDS * AHCI_CMD_SZ,
61 	AHCI_RX_FIS_SZ		= 256,
62 	AHCI_CMD_TBL_CDB	= 0x40,
63 	AHCI_CMD_TBL_HDR_SZ	= 0x80,
64 	AHCI_CMD_TBL_SZ		= AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16),
65 	AHCI_CMD_TBL_AR_SZ	= AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS,
66 	AHCI_PORT_PRIV_DMA_SZ	= AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ +
67 				  AHCI_RX_FIS_SZ,
68 	AHCI_IRQ_ON_SG		= (1 << 31),
69 	AHCI_CMD_ATAPI		= (1 << 5),
70 	AHCI_CMD_WRITE		= (1 << 6),
71 	AHCI_CMD_PREFETCH	= (1 << 7),
72 	AHCI_CMD_RESET		= (1 << 8),
73 	AHCI_CMD_CLR_BUSY	= (1 << 10),
74 
75 	RX_FIS_D2H_REG		= 0x40,	/* offset of D2H Register FIS data */
76 	RX_FIS_SDB		= 0x58, /* offset of SDB FIS data */
77 	RX_FIS_UNK		= 0x60, /* offset of Unknown FIS data */
78 
79 	board_ahci		= 0,
80 	board_ahci_pi		= 1,
81 	board_ahci_vt8251	= 2,
82 	board_ahci_ign_iferr	= 3,
83 	board_ahci_sb600	= 4,
84 
85 	/* global controller registers */
86 	HOST_CAP		= 0x00, /* host capabilities */
87 	HOST_CTL		= 0x04, /* global host control */
88 	HOST_IRQ_STAT		= 0x08, /* interrupt status */
89 	HOST_PORTS_IMPL		= 0x0c, /* bitmap of implemented ports */
90 	HOST_VERSION		= 0x10, /* AHCI spec. version compliancy */
91 
92 	/* HOST_CTL bits */
93 	HOST_RESET		= (1 << 0),  /* reset controller; self-clear */
94 	HOST_IRQ_EN		= (1 << 1),  /* global IRQ enable */
95 	HOST_AHCI_EN		= (1 << 31), /* AHCI enabled */
96 
97 	/* HOST_CAP bits */
98 	HOST_CAP_SSC		= (1 << 14), /* Slumber capable */
99 	HOST_CAP_CLO		= (1 << 24), /* Command List Override support */
100 	HOST_CAP_SSS		= (1 << 27), /* Staggered Spin-up */
101 	HOST_CAP_NCQ		= (1 << 30), /* Native Command Queueing */
102 	HOST_CAP_64		= (1 << 31), /* PCI DAC (64-bit DMA) support */
103 
104 	/* registers for each SATA port */
105 	PORT_LST_ADDR		= 0x00, /* command list DMA addr */
106 	PORT_LST_ADDR_HI	= 0x04, /* command list DMA addr hi */
107 	PORT_FIS_ADDR		= 0x08, /* FIS rx buf addr */
108 	PORT_FIS_ADDR_HI	= 0x0c, /* FIS rx buf addr hi */
109 	PORT_IRQ_STAT		= 0x10, /* interrupt status */
110 	PORT_IRQ_MASK		= 0x14, /* interrupt enable/disable mask */
111 	PORT_CMD		= 0x18, /* port command */
112 	PORT_TFDATA		= 0x20,	/* taskfile data */
113 	PORT_SIG		= 0x24,	/* device TF signature */
114 	PORT_CMD_ISSUE		= 0x38, /* command issue */
115 	PORT_SCR		= 0x28, /* SATA phy register block */
116 	PORT_SCR_STAT		= 0x28, /* SATA phy register: SStatus */
117 	PORT_SCR_CTL		= 0x2c, /* SATA phy register: SControl */
118 	PORT_SCR_ERR		= 0x30, /* SATA phy register: SError */
119 	PORT_SCR_ACT		= 0x34, /* SATA phy register: SActive */
120 
121 	/* PORT_IRQ_{STAT,MASK} bits */
122 	PORT_IRQ_COLD_PRES	= (1 << 31), /* cold presence detect */
123 	PORT_IRQ_TF_ERR		= (1 << 30), /* task file error */
124 	PORT_IRQ_HBUS_ERR	= (1 << 29), /* host bus fatal error */
125 	PORT_IRQ_HBUS_DATA_ERR	= (1 << 28), /* host bus data error */
126 	PORT_IRQ_IF_ERR		= (1 << 27), /* interface fatal error */
127 	PORT_IRQ_IF_NONFATAL	= (1 << 26), /* interface non-fatal error */
128 	PORT_IRQ_OVERFLOW	= (1 << 24), /* xfer exhausted available S/G */
129 	PORT_IRQ_BAD_PMP	= (1 << 23), /* incorrect port multiplier */
130 
131 	PORT_IRQ_PHYRDY		= (1 << 22), /* PhyRdy changed */
132 	PORT_IRQ_DEV_ILCK	= (1 << 7), /* device interlock */
133 	PORT_IRQ_CONNECT	= (1 << 6), /* port connect change status */
134 	PORT_IRQ_SG_DONE	= (1 << 5), /* descriptor processed */
135 	PORT_IRQ_UNK_FIS	= (1 << 4), /* unknown FIS rx'd */
136 	PORT_IRQ_SDB_FIS	= (1 << 3), /* Set Device Bits FIS rx'd */
137 	PORT_IRQ_DMAS_FIS	= (1 << 2), /* DMA Setup FIS rx'd */
138 	PORT_IRQ_PIOS_FIS	= (1 << 1), /* PIO Setup FIS rx'd */
139 	PORT_IRQ_D2H_REG_FIS	= (1 << 0), /* D2H Register FIS rx'd */
140 
141 	PORT_IRQ_FREEZE		= PORT_IRQ_HBUS_ERR |
142 				  PORT_IRQ_IF_ERR |
143 				  PORT_IRQ_CONNECT |
144 				  PORT_IRQ_PHYRDY |
145 				  PORT_IRQ_UNK_FIS,
146 	PORT_IRQ_ERROR		= PORT_IRQ_FREEZE |
147 				  PORT_IRQ_TF_ERR |
148 				  PORT_IRQ_HBUS_DATA_ERR,
149 	DEF_PORT_IRQ		= PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |
150 				  PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |
151 				  PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS,
152 
153 	/* PORT_CMD bits */
154 	PORT_CMD_ATAPI		= (1 << 24), /* Device is ATAPI */
155 	PORT_CMD_LIST_ON	= (1 << 15), /* cmd list DMA engine running */
156 	PORT_CMD_FIS_ON		= (1 << 14), /* FIS DMA engine running */
157 	PORT_CMD_FIS_RX		= (1 << 4), /* Enable FIS receive DMA engine */
158 	PORT_CMD_CLO		= (1 << 3), /* Command list override */
159 	PORT_CMD_POWER_ON	= (1 << 2), /* Power up device */
160 	PORT_CMD_SPIN_UP	= (1 << 1), /* Spin up device */
161 	PORT_CMD_START		= (1 << 0), /* Enable port DMA engine */
162 
163 	PORT_CMD_ICC_MASK	= (0xf << 28), /* i/f ICC state mask */
164 	PORT_CMD_ICC_ACTIVE	= (0x1 << 28), /* Put i/f in active state */
165 	PORT_CMD_ICC_PARTIAL	= (0x2 << 28), /* Put i/f in partial state */
166 	PORT_CMD_ICC_SLUMBER	= (0x6 << 28), /* Put i/f in slumber state */
167 
168 	/* ap->flags bits */
169 	AHCI_FLAG_NO_NCQ		= (1 << 24),
170 	AHCI_FLAG_IGN_IRQ_IF_ERR	= (1 << 25), /* ignore IRQ_IF_ERR */
171 	AHCI_FLAG_HONOR_PI		= (1 << 26), /* honor PORTS_IMPL */
172 	AHCI_FLAG_IGN_SERR_INTERNAL	= (1 << 27), /* ignore SERR_INTERNAL */
173 
174 	AHCI_FLAG_COMMON		= ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
175 					  ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
176 					  ATA_FLAG_SKIP_D2H_BSY,
177 };
178 
179 struct ahci_cmd_hdr {
180 	u32			opts;
181 	u32			status;
182 	u32			tbl_addr;
183 	u32			tbl_addr_hi;
184 	u32			reserved[4];
185 };
186 
187 struct ahci_sg {
188 	u32			addr;
189 	u32			addr_hi;
190 	u32			reserved;
191 	u32			flags_size;
192 };
193 
194 struct ahci_host_priv {
195 	u32			cap;		/* cap to use */
196 	u32			port_map;	/* port map to use */
197 	u32			saved_cap;	/* saved initial cap */
198 	u32			saved_port_map;	/* saved initial port_map */
199 };
200 
201 struct ahci_port_priv {
202 	struct ahci_cmd_hdr	*cmd_slot;
203 	dma_addr_t		cmd_slot_dma;
204 	void			*cmd_tbl;
205 	dma_addr_t		cmd_tbl_dma;
206 	void			*rx_fis;
207 	dma_addr_t		rx_fis_dma;
208 	/* for NCQ spurious interrupt analysis */
209 	unsigned int		ncq_saw_d2h:1;
210 	unsigned int		ncq_saw_dmas:1;
211 	unsigned int		ncq_saw_sdb:1;
212 };
213 
214 static u32 ahci_scr_read (struct ata_port *ap, unsigned int sc_reg);
215 static void ahci_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val);
216 static int ahci_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
217 static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
218 static void ahci_irq_clear(struct ata_port *ap);
219 static int ahci_port_start(struct ata_port *ap);
220 static void ahci_port_stop(struct ata_port *ap);
221 static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf);
222 static void ahci_qc_prep(struct ata_queued_cmd *qc);
223 static u8 ahci_check_status(struct ata_port *ap);
224 static void ahci_freeze(struct ata_port *ap);
225 static void ahci_thaw(struct ata_port *ap);
226 static void ahci_error_handler(struct ata_port *ap);
227 static void ahci_vt8251_error_handler(struct ata_port *ap);
228 static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
229 #ifdef CONFIG_PM
230 static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
231 static int ahci_port_resume(struct ata_port *ap);
232 static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
233 static int ahci_pci_device_resume(struct pci_dev *pdev);
234 #endif
235 
236 static struct scsi_host_template ahci_sht = {
237 	.module			= THIS_MODULE,
238 	.name			= DRV_NAME,
239 	.ioctl			= ata_scsi_ioctl,
240 	.queuecommand		= ata_scsi_queuecmd,
241 	.change_queue_depth	= ata_scsi_change_queue_depth,
242 	.can_queue		= AHCI_MAX_CMDS - 1,
243 	.this_id		= ATA_SHT_THIS_ID,
244 	.sg_tablesize		= AHCI_MAX_SG,
245 	.cmd_per_lun		= ATA_SHT_CMD_PER_LUN,
246 	.emulated		= ATA_SHT_EMULATED,
247 	.use_clustering		= AHCI_USE_CLUSTERING,
248 	.proc_name		= DRV_NAME,
249 	.dma_boundary		= AHCI_DMA_BOUNDARY,
250 	.slave_configure	= ata_scsi_slave_config,
251 	.slave_destroy		= ata_scsi_slave_destroy,
252 	.bios_param		= ata_std_bios_param,
253 };
254 
255 static const struct ata_port_operations ahci_ops = {
256 	.port_disable		= ata_port_disable,
257 
258 	.check_status		= ahci_check_status,
259 	.check_altstatus	= ahci_check_status,
260 	.dev_select		= ata_noop_dev_select,
261 
262 	.tf_read		= ahci_tf_read,
263 
264 	.qc_prep		= ahci_qc_prep,
265 	.qc_issue		= ahci_qc_issue,
266 
267 	.irq_clear		= ahci_irq_clear,
268 	.irq_on			= ata_dummy_irq_on,
269 	.irq_ack		= ata_dummy_irq_ack,
270 
271 	.scr_read		= ahci_scr_read,
272 	.scr_write		= ahci_scr_write,
273 
274 	.freeze			= ahci_freeze,
275 	.thaw			= ahci_thaw,
276 
277 	.error_handler		= ahci_error_handler,
278 	.post_internal_cmd	= ahci_post_internal_cmd,
279 
280 #ifdef CONFIG_PM
281 	.port_suspend		= ahci_port_suspend,
282 	.port_resume		= ahci_port_resume,
283 #endif
284 
285 	.port_start		= ahci_port_start,
286 	.port_stop		= ahci_port_stop,
287 };
288 
289 static const struct ata_port_operations ahci_vt8251_ops = {
290 	.port_disable		= ata_port_disable,
291 
292 	.check_status		= ahci_check_status,
293 	.check_altstatus	= ahci_check_status,
294 	.dev_select		= ata_noop_dev_select,
295 
296 	.tf_read		= ahci_tf_read,
297 
298 	.qc_prep		= ahci_qc_prep,
299 	.qc_issue		= ahci_qc_issue,
300 
301 	.irq_clear		= ahci_irq_clear,
302 	.irq_on			= ata_dummy_irq_on,
303 	.irq_ack		= ata_dummy_irq_ack,
304 
305 	.scr_read		= ahci_scr_read,
306 	.scr_write		= ahci_scr_write,
307 
308 	.freeze			= ahci_freeze,
309 	.thaw			= ahci_thaw,
310 
311 	.error_handler		= ahci_vt8251_error_handler,
312 	.post_internal_cmd	= ahci_post_internal_cmd,
313 
314 #ifdef CONFIG_PM
315 	.port_suspend		= ahci_port_suspend,
316 	.port_resume		= ahci_port_resume,
317 #endif
318 
319 	.port_start		= ahci_port_start,
320 	.port_stop		= ahci_port_stop,
321 };
322 
323 static const struct ata_port_info ahci_port_info[] = {
324 	/* board_ahci */
325 	{
326 		.flags		= AHCI_FLAG_COMMON,
327 		.pio_mask	= 0x1f, /* pio0-4 */
328 		.udma_mask	= 0x7f, /* udma0-6 ; FIXME */
329 		.port_ops	= &ahci_ops,
330 	},
331 	/* board_ahci_pi */
332 	{
333 		.flags		= AHCI_FLAG_COMMON | AHCI_FLAG_HONOR_PI,
334 		.pio_mask	= 0x1f, /* pio0-4 */
335 		.udma_mask	= 0x7f, /* udma0-6 ; FIXME */
336 		.port_ops	= &ahci_ops,
337 	},
338 	/* board_ahci_vt8251 */
339 	{
340 		.flags		= AHCI_FLAG_COMMON | ATA_FLAG_HRST_TO_RESUME |
341 				  AHCI_FLAG_NO_NCQ,
342 		.pio_mask	= 0x1f, /* pio0-4 */
343 		.udma_mask	= 0x7f, /* udma0-6 ; FIXME */
344 		.port_ops	= &ahci_vt8251_ops,
345 	},
346 	/* board_ahci_ign_iferr */
347 	{
348 		.flags		= AHCI_FLAG_COMMON | AHCI_FLAG_IGN_IRQ_IF_ERR,
349 		.pio_mask	= 0x1f, /* pio0-4 */
350 		.udma_mask	= 0x7f, /* udma0-6 ; FIXME */
351 		.port_ops	= &ahci_ops,
352 	},
353 	/* board_ahci_sb600 */
354 	{
355 		.flags		= AHCI_FLAG_COMMON |
356 				  AHCI_FLAG_IGN_SERR_INTERNAL,
357 		.pio_mask	= 0x1f, /* pio0-4 */
358 		.udma_mask	= 0x7f, /* udma0-6 ; FIXME */
359 		.port_ops	= &ahci_ops,
360 	},
361 };
362 
363 static const struct pci_device_id ahci_pci_tbl[] = {
364 	/* Intel */
365 	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
366 	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
367 	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
368 	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
369 	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
370 	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
371 	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
372 	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
373 	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
374 	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
375 	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci_pi }, /* ICH8 */
376 	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci_pi }, /* ICH8 */
377 	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci_pi }, /* ICH8 */
378 	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci_pi }, /* ICH8M */
379 	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci_pi }, /* ICH8M */
380 	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci_pi }, /* ICH9 */
381 	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci_pi }, /* ICH9 */
382 	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci_pi }, /* ICH9 */
383 	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci_pi }, /* ICH9 */
384 	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci_pi }, /* ICH9 */
385 	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci_pi }, /* ICH9M */
386 	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci_pi }, /* ICH9M */
387 	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci_pi }, /* ICH9M */
388 	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci_pi }, /* ICH9M */
389 	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci_pi }, /* ICH9M */
390 	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci_pi }, /* ICH9 */
391 	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci_pi }, /* ICH9M */
392 
393 	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
394 	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
395 	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
396 
397 	/* ATI */
398 	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
399 	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb600 }, /* ATI SB700 */
400 
401 	/* VIA */
402 	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
403 	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
404 
405 	/* NVIDIA */
406 	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci },		/* MCP65 */
407 	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci },		/* MCP65 */
408 	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci },		/* MCP65 */
409 	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci },		/* MCP65 */
410 	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci },		/* MCP65 */
411 	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci },		/* MCP65 */
412 	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci },		/* MCP65 */
413 	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci },		/* MCP65 */
414 	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci },		/* MCP67 */
415 	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci },		/* MCP67 */
416 	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci },		/* MCP67 */
417 	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci },		/* MCP67 */
418 	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci },		/* MCP67 */
419 	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci },		/* MCP67 */
420 	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci },		/* MCP67 */
421 	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci },		/* MCP67 */
422 	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci },		/* MCP67 */
423 	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci },		/* MCP67 */
424 	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci },		/* MCP67 */
425 	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci },		/* MCP67 */
426 
427 	/* SiS */
428 	{ PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
429 	{ PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
430 	{ PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
431 
432 	/* Generic, PCI class code for AHCI */
433 	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
434 	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
435 
436 	{ }	/* terminate list */
437 };
438 
439 
440 static struct pci_driver ahci_pci_driver = {
441 	.name			= DRV_NAME,
442 	.id_table		= ahci_pci_tbl,
443 	.probe			= ahci_init_one,
444 	.remove			= ata_pci_remove_one,
445 #ifdef CONFIG_PM
446 	.suspend		= ahci_pci_device_suspend,
447 	.resume			= ahci_pci_device_resume,
448 #endif
449 };
450 
451 
452 static inline int ahci_nr_ports(u32 cap)
453 {
454 	return (cap & 0x1f) + 1;
455 }
456 
457 static inline void __iomem *ahci_port_base(struct ata_port *ap)
458 {
459 	void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
460 
461 	return mmio + 0x100 + (ap->port_no * 0x80);
462 }
463 
464 /**
465  *	ahci_save_initial_config - Save and fixup initial config values
466  *	@pdev: target PCI device
467  *	@pi: associated ATA port info
468  *	@hpriv: host private area to store config values
469  *
470  *	Some registers containing configuration info might be setup by
471  *	BIOS and might be cleared on reset.  This function saves the
472  *	initial values of those registers into @hpriv such that they
473  *	can be restored after controller reset.
474  *
475  *	If inconsistent, config values are fixed up by this function.
476  *
477  *	LOCKING:
478  *	None.
479  */
480 static void ahci_save_initial_config(struct pci_dev *pdev,
481 				     const struct ata_port_info *pi,
482 				     struct ahci_host_priv *hpriv)
483 {
484 	void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
485 	u32 cap, port_map;
486 	int i;
487 
488 	/* Values prefixed with saved_ are written back to host after
489 	 * reset.  Values without are used for driver operation.
490 	 */
491 	hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
492 	hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
493 
494 	/* fixup zero port_map */
495 	if (!port_map) {
496 		port_map = (1 << ahci_nr_ports(hpriv->cap)) - 1;
497 		dev_printk(KERN_WARNING, &pdev->dev,
498 			   "PORTS_IMPL is zero, forcing 0x%x\n", port_map);
499 
500 		/* write the fixed up value to the PI register */
501 		hpriv->saved_port_map = port_map;
502 	}
503 
504 	/* cross check port_map and cap.n_ports */
505 	if (pi->flags & AHCI_FLAG_HONOR_PI) {
506 		u32 tmp_port_map = port_map;
507 		int n_ports = ahci_nr_ports(cap);
508 
509 		for (i = 0; i < AHCI_MAX_PORTS && n_ports; i++) {
510 			if (tmp_port_map & (1 << i)) {
511 				n_ports--;
512 				tmp_port_map &= ~(1 << i);
513 			}
514 		}
515 
516 		/* Whine if inconsistent.  No need to update cap.
517 		 * port_map is used to determine number of ports.
518 		 */
519 		if (n_ports || tmp_port_map)
520 			dev_printk(KERN_WARNING, &pdev->dev,
521 				   "nr_ports (%u) and implemented port map "
522 				   "(0x%x) don't match\n",
523 				   ahci_nr_ports(cap), port_map);
524 	} else {
525 		/* fabricate port_map from cap.nr_ports */
526 		port_map = (1 << ahci_nr_ports(cap)) - 1;
527 	}
528 
529 	/* record values to use during operation */
530 	hpriv->cap = cap;
531 	hpriv->port_map = port_map;
532 }
533 
534 /**
535  *	ahci_restore_initial_config - Restore initial config
536  *	@host: target ATA host
537  *
538  *	Restore initial config stored by ahci_save_initial_config().
539  *
540  *	LOCKING:
541  *	None.
542  */
543 static void ahci_restore_initial_config(struct ata_host *host)
544 {
545 	struct ahci_host_priv *hpriv = host->private_data;
546 	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
547 
548 	writel(hpriv->saved_cap, mmio + HOST_CAP);
549 	writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
550 	(void) readl(mmio + HOST_PORTS_IMPL);	/* flush */
551 }
552 
553 static u32 ahci_scr_read (struct ata_port *ap, unsigned int sc_reg_in)
554 {
555 	unsigned int sc_reg;
556 
557 	switch (sc_reg_in) {
558 	case SCR_STATUS:	sc_reg = 0; break;
559 	case SCR_CONTROL:	sc_reg = 1; break;
560 	case SCR_ERROR:		sc_reg = 2; break;
561 	case SCR_ACTIVE:	sc_reg = 3; break;
562 	default:
563 		return 0xffffffffU;
564 	}
565 
566 	return readl(ap->ioaddr.scr_addr + (sc_reg * 4));
567 }
568 
569 
570 static void ahci_scr_write (struct ata_port *ap, unsigned int sc_reg_in,
571 			       u32 val)
572 {
573 	unsigned int sc_reg;
574 
575 	switch (sc_reg_in) {
576 	case SCR_STATUS:	sc_reg = 0; break;
577 	case SCR_CONTROL:	sc_reg = 1; break;
578 	case SCR_ERROR:		sc_reg = 2; break;
579 	case SCR_ACTIVE:	sc_reg = 3; break;
580 	default:
581 		return;
582 	}
583 
584 	writel(val, ap->ioaddr.scr_addr + (sc_reg * 4));
585 }
586 
587 static void ahci_start_engine(struct ata_port *ap)
588 {
589 	void __iomem *port_mmio = ahci_port_base(ap);
590 	u32 tmp;
591 
592 	/* start DMA */
593 	tmp = readl(port_mmio + PORT_CMD);
594 	tmp |= PORT_CMD_START;
595 	writel(tmp, port_mmio + PORT_CMD);
596 	readl(port_mmio + PORT_CMD); /* flush */
597 }
598 
599 static int ahci_stop_engine(struct ata_port *ap)
600 {
601 	void __iomem *port_mmio = ahci_port_base(ap);
602 	u32 tmp;
603 
604 	tmp = readl(port_mmio + PORT_CMD);
605 
606 	/* check if the HBA is idle */
607 	if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0)
608 		return 0;
609 
610 	/* setting HBA to idle */
611 	tmp &= ~PORT_CMD_START;
612 	writel(tmp, port_mmio + PORT_CMD);
613 
614 	/* wait for engine to stop. This could be as long as 500 msec */
615 	tmp = ata_wait_register(port_mmio + PORT_CMD,
616 			        PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500);
617 	if (tmp & PORT_CMD_LIST_ON)
618 		return -EIO;
619 
620 	return 0;
621 }
622 
623 static void ahci_start_fis_rx(struct ata_port *ap)
624 {
625 	void __iomem *port_mmio = ahci_port_base(ap);
626 	struct ahci_host_priv *hpriv = ap->host->private_data;
627 	struct ahci_port_priv *pp = ap->private_data;
628 	u32 tmp;
629 
630 	/* set FIS registers */
631 	if (hpriv->cap & HOST_CAP_64)
632 		writel((pp->cmd_slot_dma >> 16) >> 16,
633 		       port_mmio + PORT_LST_ADDR_HI);
634 	writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
635 
636 	if (hpriv->cap & HOST_CAP_64)
637 		writel((pp->rx_fis_dma >> 16) >> 16,
638 		       port_mmio + PORT_FIS_ADDR_HI);
639 	writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
640 
641 	/* enable FIS reception */
642 	tmp = readl(port_mmio + PORT_CMD);
643 	tmp |= PORT_CMD_FIS_RX;
644 	writel(tmp, port_mmio + PORT_CMD);
645 
646 	/* flush */
647 	readl(port_mmio + PORT_CMD);
648 }
649 
650 static int ahci_stop_fis_rx(struct ata_port *ap)
651 {
652 	void __iomem *port_mmio = ahci_port_base(ap);
653 	u32 tmp;
654 
655 	/* disable FIS reception */
656 	tmp = readl(port_mmio + PORT_CMD);
657 	tmp &= ~PORT_CMD_FIS_RX;
658 	writel(tmp, port_mmio + PORT_CMD);
659 
660 	/* wait for completion, spec says 500ms, give it 1000 */
661 	tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON,
662 				PORT_CMD_FIS_ON, 10, 1000);
663 	if (tmp & PORT_CMD_FIS_ON)
664 		return -EBUSY;
665 
666 	return 0;
667 }
668 
669 static void ahci_power_up(struct ata_port *ap)
670 {
671 	struct ahci_host_priv *hpriv = ap->host->private_data;
672 	void __iomem *port_mmio = ahci_port_base(ap);
673 	u32 cmd;
674 
675 	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
676 
677 	/* spin up device */
678 	if (hpriv->cap & HOST_CAP_SSS) {
679 		cmd |= PORT_CMD_SPIN_UP;
680 		writel(cmd, port_mmio + PORT_CMD);
681 	}
682 
683 	/* wake up link */
684 	writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
685 }
686 
687 #ifdef CONFIG_PM
688 static void ahci_power_down(struct ata_port *ap)
689 {
690 	struct ahci_host_priv *hpriv = ap->host->private_data;
691 	void __iomem *port_mmio = ahci_port_base(ap);
692 	u32 cmd, scontrol;
693 
694 	if (!(hpriv->cap & HOST_CAP_SSS))
695 		return;
696 
697 	/* put device into listen mode, first set PxSCTL.DET to 0 */
698 	scontrol = readl(port_mmio + PORT_SCR_CTL);
699 	scontrol &= ~0xf;
700 	writel(scontrol, port_mmio + PORT_SCR_CTL);
701 
702 	/* then set PxCMD.SUD to 0 */
703 	cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
704 	cmd &= ~PORT_CMD_SPIN_UP;
705 	writel(cmd, port_mmio + PORT_CMD);
706 }
707 #endif
708 
709 static void ahci_init_port(struct ata_port *ap)
710 {
711 	/* enable FIS reception */
712 	ahci_start_fis_rx(ap);
713 
714 	/* enable DMA */
715 	ahci_start_engine(ap);
716 }
717 
718 static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
719 {
720 	int rc;
721 
722 	/* disable DMA */
723 	rc = ahci_stop_engine(ap);
724 	if (rc) {
725 		*emsg = "failed to stop engine";
726 		return rc;
727 	}
728 
729 	/* disable FIS reception */
730 	rc = ahci_stop_fis_rx(ap);
731 	if (rc) {
732 		*emsg = "failed stop FIS RX";
733 		return rc;
734 	}
735 
736 	return 0;
737 }
738 
739 static int ahci_reset_controller(struct ata_host *host)
740 {
741 	struct pci_dev *pdev = to_pci_dev(host->dev);
742 	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
743 	u32 tmp;
744 
745 	/* global controller reset */
746 	tmp = readl(mmio + HOST_CTL);
747 	if ((tmp & HOST_RESET) == 0) {
748 		writel(tmp | HOST_RESET, mmio + HOST_CTL);
749 		readl(mmio + HOST_CTL); /* flush */
750 	}
751 
752 	/* reset must complete within 1 second, or
753 	 * the hardware should be considered fried.
754 	 */
755 	ssleep(1);
756 
757 	tmp = readl(mmio + HOST_CTL);
758 	if (tmp & HOST_RESET) {
759 		dev_printk(KERN_ERR, host->dev,
760 			   "controller reset failed (0x%x)\n", tmp);
761 		return -EIO;
762 	}
763 
764 	/* turn on AHCI mode */
765 	writel(HOST_AHCI_EN, mmio + HOST_CTL);
766 	(void) readl(mmio + HOST_CTL);	/* flush */
767 
768 	/* some registers might be cleared on reset.  restore initial values */
769 	ahci_restore_initial_config(host);
770 
771 	if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
772 		u16 tmp16;
773 
774 		/* configure PCS */
775 		pci_read_config_word(pdev, 0x92, &tmp16);
776 		tmp16 |= 0xf;
777 		pci_write_config_word(pdev, 0x92, tmp16);
778 	}
779 
780 	return 0;
781 }
782 
783 static void ahci_init_controller(struct ata_host *host)
784 {
785 	struct pci_dev *pdev = to_pci_dev(host->dev);
786 	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
787 	int i, rc;
788 	u32 tmp;
789 
790 	for (i = 0; i < host->n_ports; i++) {
791 		struct ata_port *ap = host->ports[i];
792 		void __iomem *port_mmio = ahci_port_base(ap);
793 		const char *emsg = NULL;
794 
795 		if (ata_port_is_dummy(ap))
796 			continue;
797 
798 		/* make sure port is not active */
799 		rc = ahci_deinit_port(ap, &emsg);
800 		if (rc)
801 			dev_printk(KERN_WARNING, &pdev->dev,
802 				   "%s (%d)\n", emsg, rc);
803 
804 		/* clear SError */
805 		tmp = readl(port_mmio + PORT_SCR_ERR);
806 		VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
807 		writel(tmp, port_mmio + PORT_SCR_ERR);
808 
809 		/* clear port IRQ */
810 		tmp = readl(port_mmio + PORT_IRQ_STAT);
811 		VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
812 		if (tmp)
813 			writel(tmp, port_mmio + PORT_IRQ_STAT);
814 
815 		writel(1 << i, mmio + HOST_IRQ_STAT);
816 	}
817 
818 	tmp = readl(mmio + HOST_CTL);
819 	VPRINTK("HOST_CTL 0x%x\n", tmp);
820 	writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL);
821 	tmp = readl(mmio + HOST_CTL);
822 	VPRINTK("HOST_CTL 0x%x\n", tmp);
823 }
824 
825 static unsigned int ahci_dev_classify(struct ata_port *ap)
826 {
827 	void __iomem *port_mmio = ahci_port_base(ap);
828 	struct ata_taskfile tf;
829 	u32 tmp;
830 
831 	tmp = readl(port_mmio + PORT_SIG);
832 	tf.lbah		= (tmp >> 24)	& 0xff;
833 	tf.lbam		= (tmp >> 16)	& 0xff;
834 	tf.lbal		= (tmp >> 8)	& 0xff;
835 	tf.nsect	= (tmp)		& 0xff;
836 
837 	return ata_dev_classify(&tf);
838 }
839 
840 static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
841 			       u32 opts)
842 {
843 	dma_addr_t cmd_tbl_dma;
844 
845 	cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ;
846 
847 	pp->cmd_slot[tag].opts = cpu_to_le32(opts);
848 	pp->cmd_slot[tag].status = 0;
849 	pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff);
850 	pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
851 }
852 
853 static int ahci_clo(struct ata_port *ap)
854 {
855 	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
856 	struct ahci_host_priv *hpriv = ap->host->private_data;
857 	u32 tmp;
858 
859 	if (!(hpriv->cap & HOST_CAP_CLO))
860 		return -EOPNOTSUPP;
861 
862 	tmp = readl(port_mmio + PORT_CMD);
863 	tmp |= PORT_CMD_CLO;
864 	writel(tmp, port_mmio + PORT_CMD);
865 
866 	tmp = ata_wait_register(port_mmio + PORT_CMD,
867 				PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
868 	if (tmp & PORT_CMD_CLO)
869 		return -EIO;
870 
871 	return 0;
872 }
873 
874 static int ahci_softreset(struct ata_port *ap, unsigned int *class,
875 			  unsigned long deadline)
876 {
877 	struct ahci_port_priv *pp = ap->private_data;
878 	void __iomem *port_mmio = ahci_port_base(ap);
879 	const u32 cmd_fis_len = 5; /* five dwords */
880 	const char *reason = NULL;
881 	struct ata_taskfile tf;
882 	u32 tmp;
883 	u8 *fis;
884 	int rc;
885 
886 	DPRINTK("ENTER\n");
887 
888 	if (ata_port_offline(ap)) {
889 		DPRINTK("PHY reports no device\n");
890 		*class = ATA_DEV_NONE;
891 		return 0;
892 	}
893 
894 	/* prepare for SRST (AHCI-1.1 10.4.1) */
895 	rc = ahci_stop_engine(ap);
896 	if (rc) {
897 		reason = "failed to stop engine";
898 		goto fail_restart;
899 	}
900 
901 	/* check BUSY/DRQ, perform Command List Override if necessary */
902 	if (ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ)) {
903 		rc = ahci_clo(ap);
904 
905 		if (rc == -EOPNOTSUPP) {
906 			reason = "port busy but CLO unavailable";
907 			goto fail_restart;
908 		} else if (rc) {
909 			reason = "port busy but CLO failed";
910 			goto fail_restart;
911 		}
912 	}
913 
914 	/* restart engine */
915 	ahci_start_engine(ap);
916 
917 	ata_tf_init(ap->device, &tf);
918 	fis = pp->cmd_tbl;
919 
920 	/* issue the first D2H Register FIS */
921 	ahci_fill_cmd_slot(pp, 0,
922 			   cmd_fis_len | AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY);
923 
924 	tf.ctl |= ATA_SRST;
925 	ata_tf_to_fis(&tf, fis, 0);
926 	fis[1] &= ~(1 << 7);	/* turn off Command FIS bit */
927 
928 	writel(1, port_mmio + PORT_CMD_ISSUE);
929 
930 	tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1, 1, 500);
931 	if (tmp & 0x1) {
932 		rc = -EIO;
933 		reason = "1st FIS failed";
934 		goto fail;
935 	}
936 
937 	/* spec says at least 5us, but be generous and sleep for 1ms */
938 	msleep(1);
939 
940 	/* issue the second D2H Register FIS */
941 	ahci_fill_cmd_slot(pp, 0, cmd_fis_len);
942 
943 	tf.ctl &= ~ATA_SRST;
944 	ata_tf_to_fis(&tf, fis, 0);
945 	fis[1] &= ~(1 << 7);	/* turn off Command FIS bit */
946 
947 	writel(1, port_mmio + PORT_CMD_ISSUE);
948 	readl(port_mmio + PORT_CMD_ISSUE);	/* flush */
949 
950 	/* spec mandates ">= 2ms" before checking status.
951 	 * We wait 150ms, because that was the magic delay used for
952 	 * ATAPI devices in Hale Landis's ATADRVR, for the period of time
953 	 * between when the ATA command register is written, and then
954 	 * status is checked.  Because waiting for "a while" before
955 	 * checking status is fine, post SRST, we perform this magic
956 	 * delay here as well.
957 	 */
958 	msleep(150);
959 
960 	rc = ata_wait_ready(ap, deadline);
961 	/* link occupied, -ENODEV too is an error */
962 	if (rc) {
963 		reason = "device not ready";
964 		goto fail;
965 	}
966 	*class = ahci_dev_classify(ap);
967 
968 	DPRINTK("EXIT, class=%u\n", *class);
969 	return 0;
970 
971  fail_restart:
972 	ahci_start_engine(ap);
973  fail:
974 	ata_port_printk(ap, KERN_ERR, "softreset failed (%s)\n", reason);
975 	return rc;
976 }
977 
978 static int ahci_hardreset(struct ata_port *ap, unsigned int *class,
979 			  unsigned long deadline)
980 {
981 	struct ahci_port_priv *pp = ap->private_data;
982 	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
983 	struct ata_taskfile tf;
984 	int rc;
985 
986 	DPRINTK("ENTER\n");
987 
988 	ahci_stop_engine(ap);
989 
990 	/* clear D2H reception area to properly wait for D2H FIS */
991 	ata_tf_init(ap->device, &tf);
992 	tf.command = 0x80;
993 	ata_tf_to_fis(&tf, d2h_fis, 0);
994 
995 	rc = sata_std_hardreset(ap, class, deadline);
996 
997 	ahci_start_engine(ap);
998 
999 	if (rc == 0 && ata_port_online(ap))
1000 		*class = ahci_dev_classify(ap);
1001 	if (*class == ATA_DEV_UNKNOWN)
1002 		*class = ATA_DEV_NONE;
1003 
1004 	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1005 	return rc;
1006 }
1007 
1008 static int ahci_vt8251_hardreset(struct ata_port *ap, unsigned int *class,
1009 				 unsigned long deadline)
1010 {
1011 	int rc;
1012 
1013 	DPRINTK("ENTER\n");
1014 
1015 	ahci_stop_engine(ap);
1016 
1017 	rc = sata_port_hardreset(ap, sata_ehc_deb_timing(&ap->eh_context),
1018 				 deadline);
1019 
1020 	/* vt8251 needs SError cleared for the port to operate */
1021 	ahci_scr_write(ap, SCR_ERROR, ahci_scr_read(ap, SCR_ERROR));
1022 
1023 	ahci_start_engine(ap);
1024 
1025 	DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
1026 
1027 	/* vt8251 doesn't clear BSY on signature FIS reception,
1028 	 * request follow-up softreset.
1029 	 */
1030 	return rc ?: -EAGAIN;
1031 }
1032 
1033 static void ahci_postreset(struct ata_port *ap, unsigned int *class)
1034 {
1035 	void __iomem *port_mmio = ahci_port_base(ap);
1036 	u32 new_tmp, tmp;
1037 
1038 	ata_std_postreset(ap, class);
1039 
1040 	/* Make sure port's ATAPI bit is set appropriately */
1041 	new_tmp = tmp = readl(port_mmio + PORT_CMD);
1042 	if (*class == ATA_DEV_ATAPI)
1043 		new_tmp |= PORT_CMD_ATAPI;
1044 	else
1045 		new_tmp &= ~PORT_CMD_ATAPI;
1046 	if (new_tmp != tmp) {
1047 		writel(new_tmp, port_mmio + PORT_CMD);
1048 		readl(port_mmio + PORT_CMD); /* flush */
1049 	}
1050 }
1051 
1052 static u8 ahci_check_status(struct ata_port *ap)
1053 {
1054 	void __iomem *mmio = ap->ioaddr.cmd_addr;
1055 
1056 	return readl(mmio + PORT_TFDATA) & 0xFF;
1057 }
1058 
1059 static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf)
1060 {
1061 	struct ahci_port_priv *pp = ap->private_data;
1062 	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
1063 
1064 	ata_tf_from_fis(d2h_fis, tf);
1065 }
1066 
1067 static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl)
1068 {
1069 	struct scatterlist *sg;
1070 	struct ahci_sg *ahci_sg;
1071 	unsigned int n_sg = 0;
1072 
1073 	VPRINTK("ENTER\n");
1074 
1075 	/*
1076 	 * Next, the S/G list.
1077 	 */
1078 	ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ;
1079 	ata_for_each_sg(sg, qc) {
1080 		dma_addr_t addr = sg_dma_address(sg);
1081 		u32 sg_len = sg_dma_len(sg);
1082 
1083 		ahci_sg->addr = cpu_to_le32(addr & 0xffffffff);
1084 		ahci_sg->addr_hi = cpu_to_le32((addr >> 16) >> 16);
1085 		ahci_sg->flags_size = cpu_to_le32(sg_len - 1);
1086 
1087 		ahci_sg++;
1088 		n_sg++;
1089 	}
1090 
1091 	return n_sg;
1092 }
1093 
1094 static void ahci_qc_prep(struct ata_queued_cmd *qc)
1095 {
1096 	struct ata_port *ap = qc->ap;
1097 	struct ahci_port_priv *pp = ap->private_data;
1098 	int is_atapi = is_atapi_taskfile(&qc->tf);
1099 	void *cmd_tbl;
1100 	u32 opts;
1101 	const u32 cmd_fis_len = 5; /* five dwords */
1102 	unsigned int n_elem;
1103 
1104 	/*
1105 	 * Fill in command table information.  First, the header,
1106 	 * a SATA Register - Host to Device command FIS.
1107 	 */
1108 	cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
1109 
1110 	ata_tf_to_fis(&qc->tf, cmd_tbl, 0);
1111 	if (is_atapi) {
1112 		memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
1113 		memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
1114 	}
1115 
1116 	n_elem = 0;
1117 	if (qc->flags & ATA_QCFLAG_DMAMAP)
1118 		n_elem = ahci_fill_sg(qc, cmd_tbl);
1119 
1120 	/*
1121 	 * Fill in command slot information.
1122 	 */
1123 	opts = cmd_fis_len | n_elem << 16;
1124 	if (qc->tf.flags & ATA_TFLAG_WRITE)
1125 		opts |= AHCI_CMD_WRITE;
1126 	if (is_atapi)
1127 		opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH;
1128 
1129 	ahci_fill_cmd_slot(pp, qc->tag, opts);
1130 }
1131 
1132 static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
1133 {
1134 	struct ahci_port_priv *pp = ap->private_data;
1135 	struct ata_eh_info *ehi = &ap->eh_info;
1136 	unsigned int err_mask = 0, action = 0;
1137 	struct ata_queued_cmd *qc;
1138 	u32 serror;
1139 
1140 	ata_ehi_clear_desc(ehi);
1141 
1142 	/* AHCI needs SError cleared; otherwise, it might lock up */
1143 	serror = ahci_scr_read(ap, SCR_ERROR);
1144 	ahci_scr_write(ap, SCR_ERROR, serror);
1145 
1146 	/* analyze @irq_stat */
1147 	ata_ehi_push_desc(ehi, "irq_stat 0x%08x", irq_stat);
1148 
1149 	/* some controllers set IRQ_IF_ERR on device errors, ignore it */
1150 	if (ap->flags & AHCI_FLAG_IGN_IRQ_IF_ERR)
1151 		irq_stat &= ~PORT_IRQ_IF_ERR;
1152 
1153 	if (irq_stat & PORT_IRQ_TF_ERR) {
1154 		err_mask |= AC_ERR_DEV;
1155 		if (ap->flags & AHCI_FLAG_IGN_SERR_INTERNAL)
1156 			serror &= ~SERR_INTERNAL;
1157 	}
1158 
1159 	if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
1160 		err_mask |= AC_ERR_HOST_BUS;
1161 		action |= ATA_EH_SOFTRESET;
1162 	}
1163 
1164 	if (irq_stat & PORT_IRQ_IF_ERR) {
1165 		err_mask |= AC_ERR_ATA_BUS;
1166 		action |= ATA_EH_SOFTRESET;
1167 		ata_ehi_push_desc(ehi, ", interface fatal error");
1168 	}
1169 
1170 	if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
1171 		ata_ehi_hotplugged(ehi);
1172 		ata_ehi_push_desc(ehi, ", %s", irq_stat & PORT_IRQ_CONNECT ?
1173 			"connection status changed" : "PHY RDY changed");
1174 	}
1175 
1176 	if (irq_stat & PORT_IRQ_UNK_FIS) {
1177 		u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);
1178 
1179 		err_mask |= AC_ERR_HSM;
1180 		action |= ATA_EH_SOFTRESET;
1181 		ata_ehi_push_desc(ehi, ", unknown FIS %08x %08x %08x %08x",
1182 				  unk[0], unk[1], unk[2], unk[3]);
1183 	}
1184 
1185 	/* okay, let's hand over to EH */
1186 	ehi->serror |= serror;
1187 	ehi->action |= action;
1188 
1189 	qc = ata_qc_from_tag(ap, ap->active_tag);
1190 	if (qc)
1191 		qc->err_mask |= err_mask;
1192 	else
1193 		ehi->err_mask |= err_mask;
1194 
1195 	if (irq_stat & PORT_IRQ_FREEZE)
1196 		ata_port_freeze(ap);
1197 	else
1198 		ata_port_abort(ap);
1199 }
1200 
1201 static void ahci_host_intr(struct ata_port *ap)
1202 {
1203 	void __iomem *port_mmio = ap->ioaddr.cmd_addr;
1204 	struct ata_eh_info *ehi = &ap->eh_info;
1205 	struct ahci_port_priv *pp = ap->private_data;
1206 	u32 status, qc_active;
1207 	int rc, known_irq = 0;
1208 
1209 	status = readl(port_mmio + PORT_IRQ_STAT);
1210 	writel(status, port_mmio + PORT_IRQ_STAT);
1211 
1212 	if (unlikely(status & PORT_IRQ_ERROR)) {
1213 		ahci_error_intr(ap, status);
1214 		return;
1215 	}
1216 
1217 	if (ap->sactive)
1218 		qc_active = readl(port_mmio + PORT_SCR_ACT);
1219 	else
1220 		qc_active = readl(port_mmio + PORT_CMD_ISSUE);
1221 
1222 	rc = ata_qc_complete_multiple(ap, qc_active, NULL);
1223 	if (rc > 0)
1224 		return;
1225 	if (rc < 0) {
1226 		ehi->err_mask |= AC_ERR_HSM;
1227 		ehi->action |= ATA_EH_SOFTRESET;
1228 		ata_port_freeze(ap);
1229 		return;
1230 	}
1231 
1232 	/* hmmm... a spurious interupt */
1233 
1234 	/* if !NCQ, ignore.  No modern ATA device has broken HSM
1235 	 * implementation for non-NCQ commands.
1236 	 */
1237 	if (!ap->sactive)
1238 		return;
1239 
1240 	if (status & PORT_IRQ_D2H_REG_FIS) {
1241 		if (!pp->ncq_saw_d2h)
1242 			ata_port_printk(ap, KERN_INFO,
1243 				"D2H reg with I during NCQ, "
1244 				"this message won't be printed again\n");
1245 		pp->ncq_saw_d2h = 1;
1246 		known_irq = 1;
1247 	}
1248 
1249 	if (status & PORT_IRQ_DMAS_FIS) {
1250 		if (!pp->ncq_saw_dmas)
1251 			ata_port_printk(ap, KERN_INFO,
1252 				"DMAS FIS during NCQ, "
1253 				"this message won't be printed again\n");
1254 		pp->ncq_saw_dmas = 1;
1255 		known_irq = 1;
1256 	}
1257 
1258 	if (status & PORT_IRQ_SDB_FIS) {
1259 		const __le32 *f = pp->rx_fis + RX_FIS_SDB;
1260 
1261 		if (le32_to_cpu(f[1])) {
1262 			/* SDB FIS containing spurious completions
1263 			 * might be dangerous, whine and fail commands
1264 			 * with HSM violation.  EH will turn off NCQ
1265 			 * after several such failures.
1266 			 */
1267 			ata_ehi_push_desc(ehi,
1268 				"spurious completions during NCQ "
1269 				"issue=0x%x SAct=0x%x FIS=%08x:%08x",
1270 				readl(port_mmio + PORT_CMD_ISSUE),
1271 				readl(port_mmio + PORT_SCR_ACT),
1272 				le32_to_cpu(f[0]), le32_to_cpu(f[1]));
1273 			ehi->err_mask |= AC_ERR_HSM;
1274 			ehi->action |= ATA_EH_SOFTRESET;
1275 			ata_port_freeze(ap);
1276 		} else {
1277 			if (!pp->ncq_saw_sdb)
1278 				ata_port_printk(ap, KERN_INFO,
1279 					"spurious SDB FIS %08x:%08x during NCQ, "
1280 					"this message won't be printed again\n",
1281 					le32_to_cpu(f[0]), le32_to_cpu(f[1]));
1282 			pp->ncq_saw_sdb = 1;
1283 		}
1284 		known_irq = 1;
1285 	}
1286 
1287 	if (!known_irq)
1288 		ata_port_printk(ap, KERN_INFO, "spurious interrupt "
1289 				"(irq_stat 0x%x active_tag 0x%x sactive 0x%x)\n",
1290 				status, ap->active_tag, ap->sactive);
1291 }
1292 
1293 static void ahci_irq_clear(struct ata_port *ap)
1294 {
1295 	/* TODO */
1296 }
1297 
1298 static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
1299 {
1300 	struct ata_host *host = dev_instance;
1301 	struct ahci_host_priv *hpriv;
1302 	unsigned int i, handled = 0;
1303 	void __iomem *mmio;
1304 	u32 irq_stat, irq_ack = 0;
1305 
1306 	VPRINTK("ENTER\n");
1307 
1308 	hpriv = host->private_data;
1309 	mmio = host->iomap[AHCI_PCI_BAR];
1310 
1311 	/* sigh.  0xffffffff is a valid return from h/w */
1312 	irq_stat = readl(mmio + HOST_IRQ_STAT);
1313 	irq_stat &= hpriv->port_map;
1314 	if (!irq_stat)
1315 		return IRQ_NONE;
1316 
1317         spin_lock(&host->lock);
1318 
1319         for (i = 0; i < host->n_ports; i++) {
1320 		struct ata_port *ap;
1321 
1322 		if (!(irq_stat & (1 << i)))
1323 			continue;
1324 
1325 		ap = host->ports[i];
1326 		if (ap) {
1327 			ahci_host_intr(ap);
1328 			VPRINTK("port %u\n", i);
1329 		} else {
1330 			VPRINTK("port %u (no irq)\n", i);
1331 			if (ata_ratelimit())
1332 				dev_printk(KERN_WARNING, host->dev,
1333 					"interrupt on disabled port %u\n", i);
1334 		}
1335 
1336 		irq_ack |= (1 << i);
1337 	}
1338 
1339 	if (irq_ack) {
1340 		writel(irq_ack, mmio + HOST_IRQ_STAT);
1341 		handled = 1;
1342 	}
1343 
1344 	spin_unlock(&host->lock);
1345 
1346 	VPRINTK("EXIT\n");
1347 
1348 	return IRQ_RETVAL(handled);
1349 }
1350 
1351 static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
1352 {
1353 	struct ata_port *ap = qc->ap;
1354 	void __iomem *port_mmio = ahci_port_base(ap);
1355 
1356 	if (qc->tf.protocol == ATA_PROT_NCQ)
1357 		writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
1358 	writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE);
1359 	readl(port_mmio + PORT_CMD_ISSUE);	/* flush */
1360 
1361 	return 0;
1362 }
1363 
1364 static void ahci_freeze(struct ata_port *ap)
1365 {
1366 	void __iomem *port_mmio = ahci_port_base(ap);
1367 
1368 	/* turn IRQ off */
1369 	writel(0, port_mmio + PORT_IRQ_MASK);
1370 }
1371 
1372 static void ahci_thaw(struct ata_port *ap)
1373 {
1374 	void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
1375 	void __iomem *port_mmio = ahci_port_base(ap);
1376 	u32 tmp;
1377 
1378 	/* clear IRQ */
1379 	tmp = readl(port_mmio + PORT_IRQ_STAT);
1380 	writel(tmp, port_mmio + PORT_IRQ_STAT);
1381 	writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
1382 
1383 	/* turn IRQ back on */
1384 	writel(DEF_PORT_IRQ, port_mmio + PORT_IRQ_MASK);
1385 }
1386 
1387 static void ahci_error_handler(struct ata_port *ap)
1388 {
1389 	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
1390 		/* restart engine */
1391 		ahci_stop_engine(ap);
1392 		ahci_start_engine(ap);
1393 	}
1394 
1395 	/* perform recovery */
1396 	ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_hardreset,
1397 		  ahci_postreset);
1398 }
1399 
1400 static void ahci_vt8251_error_handler(struct ata_port *ap)
1401 {
1402 	if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
1403 		/* restart engine */
1404 		ahci_stop_engine(ap);
1405 		ahci_start_engine(ap);
1406 	}
1407 
1408 	/* perform recovery */
1409 	ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_vt8251_hardreset,
1410 		  ahci_postreset);
1411 }
1412 
1413 static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
1414 {
1415 	struct ata_port *ap = qc->ap;
1416 
1417 	if (qc->flags & ATA_QCFLAG_FAILED) {
1418 		/* make DMA engine forget about the failed command */
1419 		ahci_stop_engine(ap);
1420 		ahci_start_engine(ap);
1421 	}
1422 }
1423 
1424 #ifdef CONFIG_PM
1425 static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
1426 {
1427 	const char *emsg = NULL;
1428 	int rc;
1429 
1430 	rc = ahci_deinit_port(ap, &emsg);
1431 	if (rc == 0)
1432 		ahci_power_down(ap);
1433 	else {
1434 		ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc);
1435 		ahci_init_port(ap);
1436 	}
1437 
1438 	return rc;
1439 }
1440 
1441 static int ahci_port_resume(struct ata_port *ap)
1442 {
1443 	ahci_power_up(ap);
1444 	ahci_init_port(ap);
1445 
1446 	return 0;
1447 }
1448 
1449 static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
1450 {
1451 	struct ata_host *host = dev_get_drvdata(&pdev->dev);
1452 	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1453 	u32 ctl;
1454 
1455 	if (mesg.event == PM_EVENT_SUSPEND) {
1456 		/* AHCI spec rev1.1 section 8.3.3:
1457 		 * Software must disable interrupts prior to requesting a
1458 		 * transition of the HBA to D3 state.
1459 		 */
1460 		ctl = readl(mmio + HOST_CTL);
1461 		ctl &= ~HOST_IRQ_EN;
1462 		writel(ctl, mmio + HOST_CTL);
1463 		readl(mmio + HOST_CTL); /* flush */
1464 	}
1465 
1466 	return ata_pci_device_suspend(pdev, mesg);
1467 }
1468 
1469 static int ahci_pci_device_resume(struct pci_dev *pdev)
1470 {
1471 	struct ata_host *host = dev_get_drvdata(&pdev->dev);
1472 	int rc;
1473 
1474 	rc = ata_pci_device_do_resume(pdev);
1475 	if (rc)
1476 		return rc;
1477 
1478 	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
1479 		rc = ahci_reset_controller(host);
1480 		if (rc)
1481 			return rc;
1482 
1483 		ahci_init_controller(host);
1484 	}
1485 
1486 	ata_host_resume(host);
1487 
1488 	return 0;
1489 }
1490 #endif
1491 
1492 static int ahci_port_start(struct ata_port *ap)
1493 {
1494 	struct device *dev = ap->host->dev;
1495 	struct ahci_port_priv *pp;
1496 	void *mem;
1497 	dma_addr_t mem_dma;
1498 	int rc;
1499 
1500 	pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
1501 	if (!pp)
1502 		return -ENOMEM;
1503 
1504 	rc = ata_pad_alloc(ap, dev);
1505 	if (rc)
1506 		return rc;
1507 
1508 	mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma,
1509 				  GFP_KERNEL);
1510 	if (!mem)
1511 		return -ENOMEM;
1512 	memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ);
1513 
1514 	/*
1515 	 * First item in chunk of DMA memory: 32-slot command table,
1516 	 * 32 bytes each in size
1517 	 */
1518 	pp->cmd_slot = mem;
1519 	pp->cmd_slot_dma = mem_dma;
1520 
1521 	mem += AHCI_CMD_SLOT_SZ;
1522 	mem_dma += AHCI_CMD_SLOT_SZ;
1523 
1524 	/*
1525 	 * Second item: Received-FIS area
1526 	 */
1527 	pp->rx_fis = mem;
1528 	pp->rx_fis_dma = mem_dma;
1529 
1530 	mem += AHCI_RX_FIS_SZ;
1531 	mem_dma += AHCI_RX_FIS_SZ;
1532 
1533 	/*
1534 	 * Third item: data area for storing a single command
1535 	 * and its scatter-gather table
1536 	 */
1537 	pp->cmd_tbl = mem;
1538 	pp->cmd_tbl_dma = mem_dma;
1539 
1540 	ap->private_data = pp;
1541 
1542 	/* power up port */
1543 	ahci_power_up(ap);
1544 
1545 	/* initialize port */
1546 	ahci_init_port(ap);
1547 
1548 	return 0;
1549 }
1550 
1551 static void ahci_port_stop(struct ata_port *ap)
1552 {
1553 	const char *emsg = NULL;
1554 	int rc;
1555 
1556 	/* de-initialize port */
1557 	rc = ahci_deinit_port(ap, &emsg);
1558 	if (rc)
1559 		ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc);
1560 }
1561 
1562 static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
1563 {
1564 	int rc;
1565 
1566 	if (using_dac &&
1567 	    !pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
1568 		rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1569 		if (rc) {
1570 			rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
1571 			if (rc) {
1572 				dev_printk(KERN_ERR, &pdev->dev,
1573 					   "64-bit DMA enable failed\n");
1574 				return rc;
1575 			}
1576 		}
1577 	} else {
1578 		rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
1579 		if (rc) {
1580 			dev_printk(KERN_ERR, &pdev->dev,
1581 				   "32-bit DMA enable failed\n");
1582 			return rc;
1583 		}
1584 		rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
1585 		if (rc) {
1586 			dev_printk(KERN_ERR, &pdev->dev,
1587 				   "32-bit consistent DMA enable failed\n");
1588 			return rc;
1589 		}
1590 	}
1591 	return 0;
1592 }
1593 
1594 static void ahci_print_info(struct ata_host *host)
1595 {
1596 	struct ahci_host_priv *hpriv = host->private_data;
1597 	struct pci_dev *pdev = to_pci_dev(host->dev);
1598 	void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
1599 	u32 vers, cap, impl, speed;
1600 	const char *speed_s;
1601 	u16 cc;
1602 	const char *scc_s;
1603 
1604 	vers = readl(mmio + HOST_VERSION);
1605 	cap = hpriv->cap;
1606 	impl = hpriv->port_map;
1607 
1608 	speed = (cap >> 20) & 0xf;
1609 	if (speed == 1)
1610 		speed_s = "1.5";
1611 	else if (speed == 2)
1612 		speed_s = "3";
1613 	else
1614 		speed_s = "?";
1615 
1616 	pci_read_config_word(pdev, 0x0a, &cc);
1617 	if (cc == PCI_CLASS_STORAGE_IDE)
1618 		scc_s = "IDE";
1619 	else if (cc == PCI_CLASS_STORAGE_SATA)
1620 		scc_s = "SATA";
1621 	else if (cc == PCI_CLASS_STORAGE_RAID)
1622 		scc_s = "RAID";
1623 	else
1624 		scc_s = "unknown";
1625 
1626 	dev_printk(KERN_INFO, &pdev->dev,
1627 		"AHCI %02x%02x.%02x%02x "
1628 		"%u slots %u ports %s Gbps 0x%x impl %s mode\n"
1629 	       	,
1630 
1631 	       	(vers >> 24) & 0xff,
1632 	       	(vers >> 16) & 0xff,
1633 	       	(vers >> 8) & 0xff,
1634 	       	vers & 0xff,
1635 
1636 		((cap >> 8) & 0x1f) + 1,
1637 		(cap & 0x1f) + 1,
1638 		speed_s,
1639 		impl,
1640 		scc_s);
1641 
1642 	dev_printk(KERN_INFO, &pdev->dev,
1643 		"flags: "
1644 	       	"%s%s%s%s%s%s"
1645 	       	"%s%s%s%s%s%s%s\n"
1646 	       	,
1647 
1648 		cap & (1 << 31) ? "64bit " : "",
1649 		cap & (1 << 30) ? "ncq " : "",
1650 		cap & (1 << 28) ? "ilck " : "",
1651 		cap & (1 << 27) ? "stag " : "",
1652 		cap & (1 << 26) ? "pm " : "",
1653 		cap & (1 << 25) ? "led " : "",
1654 
1655 		cap & (1 << 24) ? "clo " : "",
1656 		cap & (1 << 19) ? "nz " : "",
1657 		cap & (1 << 18) ? "only " : "",
1658 		cap & (1 << 17) ? "pmp " : "",
1659 		cap & (1 << 15) ? "pio " : "",
1660 		cap & (1 << 14) ? "slum " : "",
1661 		cap & (1 << 13) ? "part " : ""
1662 		);
1663 }
1664 
1665 static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1666 {
1667 	static int printed_version;
1668 	struct ata_port_info pi = ahci_port_info[ent->driver_data];
1669 	const struct ata_port_info *ppi[] = { &pi, NULL };
1670 	struct device *dev = &pdev->dev;
1671 	struct ahci_host_priv *hpriv;
1672 	struct ata_host *host;
1673 	int i, rc;
1674 
1675 	VPRINTK("ENTER\n");
1676 
1677 	WARN_ON(ATA_MAX_QUEUE > AHCI_MAX_CMDS);
1678 
1679 	if (!printed_version++)
1680 		dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
1681 
1682 	/* acquire resources */
1683 	rc = pcim_enable_device(pdev);
1684 	if (rc)
1685 		return rc;
1686 
1687 	rc = pcim_iomap_regions(pdev, 1 << AHCI_PCI_BAR, DRV_NAME);
1688 	if (rc == -EBUSY)
1689 		pcim_pin_device(pdev);
1690 	if (rc)
1691 		return rc;
1692 
1693 	if (pci_enable_msi(pdev))
1694 		pci_intx(pdev, 1);
1695 
1696 	hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
1697 	if (!hpriv)
1698 		return -ENOMEM;
1699 
1700 	/* save initial config */
1701 	ahci_save_initial_config(pdev, &pi, hpriv);
1702 
1703 	/* prepare host */
1704 	if (!(pi.flags & AHCI_FLAG_NO_NCQ) && (hpriv->cap & HOST_CAP_NCQ))
1705 		pi.flags |= ATA_FLAG_NCQ;
1706 
1707 	host = ata_host_alloc_pinfo(&pdev->dev, ppi, fls(hpriv->port_map));
1708 	if (!host)
1709 		return -ENOMEM;
1710 	host->iomap = pcim_iomap_table(pdev);
1711 	host->private_data = hpriv;
1712 
1713 	for (i = 0; i < host->n_ports; i++) {
1714 		if (hpriv->port_map & (1 << i)) {
1715 			struct ata_port *ap = host->ports[i];
1716 			void __iomem *port_mmio = ahci_port_base(ap);
1717 
1718 			ap->ioaddr.cmd_addr = port_mmio;
1719 			ap->ioaddr.scr_addr = port_mmio + PORT_SCR;
1720 		} else
1721 			host->ports[i]->ops = &ata_dummy_port_ops;
1722 	}
1723 
1724 	/* initialize adapter */
1725 	rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
1726 	if (rc)
1727 		return rc;
1728 
1729 	rc = ahci_reset_controller(host);
1730 	if (rc)
1731 		return rc;
1732 
1733 	ahci_init_controller(host);
1734 	ahci_print_info(host);
1735 
1736 	pci_set_master(pdev);
1737 	return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
1738 				 &ahci_sht);
1739 }
1740 
1741 static int __init ahci_init(void)
1742 {
1743 	return pci_register_driver(&ahci_pci_driver);
1744 }
1745 
1746 static void __exit ahci_exit(void)
1747 {
1748 	pci_unregister_driver(&ahci_pci_driver);
1749 }
1750 
1751 
1752 MODULE_AUTHOR("Jeff Garzik");
1753 MODULE_DESCRIPTION("AHCI SATA low-level driver");
1754 MODULE_LICENSE("GPL");
1755 MODULE_DEVICE_TABLE(pci, ahci_pci_tbl);
1756 MODULE_VERSION(DRV_VERSION);
1757 
1758 module_init(ahci_init);
1759 module_exit(ahci_exit);
1760