xref: /openbmc/linux/drivers/ata/ahci.c (revision 8e7ae8ba)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  ahci.c - AHCI SATA support
4  *
5  *  Maintained by:  Tejun Heo <tj@kernel.org>
6  *    		    Please ALWAYS copy linux-ide@vger.kernel.org
7  *		    on emails.
8  *
9  *  Copyright 2004-2005 Red Hat, Inc.
10  *
11  * libata documentation is available via 'make {ps|pdf}docs',
12  * as Documentation/driver-api/libata.rst
13  *
14  * AHCI hardware documentation:
15  * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf
16  * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf
17  */
18 
19 #include <linux/kernel.h>
20 #include <linux/module.h>
21 #include <linux/pci.h>
22 #include <linux/blkdev.h>
23 #include <linux/delay.h>
24 #include <linux/interrupt.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/device.h>
27 #include <linux/dmi.h>
28 #include <linux/gfp.h>
29 #include <linux/msi.h>
30 #include <scsi/scsi_host.h>
31 #include <scsi/scsi_cmnd.h>
32 #include <linux/libata.h>
33 #include <linux/ahci-remap.h>
34 #include <linux/io-64-nonatomic-lo-hi.h>
35 #include "ahci.h"
36 
37 #define DRV_NAME	"ahci"
38 #define DRV_VERSION	"3.0"
39 
40 enum {
41 	AHCI_PCI_BAR_STA2X11	= 0,
42 	AHCI_PCI_BAR_CAVIUM	= 0,
43 	AHCI_PCI_BAR_LOONGSON	= 0,
44 	AHCI_PCI_BAR_ENMOTUS	= 2,
45 	AHCI_PCI_BAR_CAVIUM_GEN5	= 4,
46 	AHCI_PCI_BAR_STANDARD	= 5,
47 };
48 
49 enum board_ids {
50 	/* board IDs by feature in alphabetical order */
51 	board_ahci,
52 	board_ahci_ign_iferr,
53 	board_ahci_low_power,
54 	board_ahci_no_debounce_delay,
55 	board_ahci_nomsi,
56 	board_ahci_noncq,
57 	board_ahci_nosntf,
58 	board_ahci_yes_fbs,
59 
60 	/* board IDs for specific chipsets in alphabetical order */
61 	board_ahci_al,
62 	board_ahci_avn,
63 	board_ahci_mcp65,
64 	board_ahci_mcp77,
65 	board_ahci_mcp89,
66 	board_ahci_mv,
67 	board_ahci_sb600,
68 	board_ahci_sb700,	/* for SB700 and SB800 */
69 	board_ahci_vt8251,
70 
71 	/*
72 	 * board IDs for Intel chipsets that support more than 6 ports
73 	 * *and* end up needing the PCS quirk.
74 	 */
75 	board_ahci_pcs7,
76 
77 	/* aliases */
78 	board_ahci_mcp_linux	= board_ahci_mcp65,
79 	board_ahci_mcp67	= board_ahci_mcp65,
80 	board_ahci_mcp73	= board_ahci_mcp65,
81 	board_ahci_mcp79	= board_ahci_mcp77,
82 };
83 
84 static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
85 static void ahci_remove_one(struct pci_dev *dev);
86 static void ahci_shutdown_one(struct pci_dev *dev);
87 static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
88 				 unsigned long deadline);
89 static int ahci_avn_hardreset(struct ata_link *link, unsigned int *class,
90 			      unsigned long deadline);
91 static void ahci_mcp89_apple_enable(struct pci_dev *pdev);
92 static bool is_mcp89_apple(struct pci_dev *pdev);
93 static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
94 				unsigned long deadline);
95 #ifdef CONFIG_PM
96 static int ahci_pci_device_runtime_suspend(struct device *dev);
97 static int ahci_pci_device_runtime_resume(struct device *dev);
98 #ifdef CONFIG_PM_SLEEP
99 static int ahci_pci_device_suspend(struct device *dev);
100 static int ahci_pci_device_resume(struct device *dev);
101 #endif
102 #endif /* CONFIG_PM */
103 
104 static struct scsi_host_template ahci_sht = {
105 	AHCI_SHT("ahci"),
106 };
107 
108 static struct ata_port_operations ahci_vt8251_ops = {
109 	.inherits		= &ahci_ops,
110 	.hardreset		= ahci_vt8251_hardreset,
111 };
112 
113 static struct ata_port_operations ahci_p5wdh_ops = {
114 	.inherits		= &ahci_ops,
115 	.hardreset		= ahci_p5wdh_hardreset,
116 };
117 
118 static struct ata_port_operations ahci_avn_ops = {
119 	.inherits		= &ahci_ops,
120 	.hardreset		= ahci_avn_hardreset,
121 };
122 
123 static const struct ata_port_info ahci_port_info[] = {
124 	/* by features */
125 	[board_ahci] = {
126 		.flags		= AHCI_FLAG_COMMON,
127 		.pio_mask	= ATA_PIO4,
128 		.udma_mask	= ATA_UDMA6,
129 		.port_ops	= &ahci_ops,
130 	},
131 	[board_ahci_ign_iferr] = {
132 		AHCI_HFLAGS	(AHCI_HFLAG_IGN_IRQ_IF_ERR),
133 		.flags		= AHCI_FLAG_COMMON,
134 		.pio_mask	= ATA_PIO4,
135 		.udma_mask	= ATA_UDMA6,
136 		.port_ops	= &ahci_ops,
137 	},
138 	[board_ahci_low_power] = {
139 		AHCI_HFLAGS	(AHCI_HFLAG_USE_LPM_POLICY),
140 		.flags		= AHCI_FLAG_COMMON,
141 		.pio_mask	= ATA_PIO4,
142 		.udma_mask	= ATA_UDMA6,
143 		.port_ops	= &ahci_ops,
144 	},
145 	[board_ahci_no_debounce_delay] = {
146 		.flags		= AHCI_FLAG_COMMON,
147 		.link_flags	= ATA_LFLAG_NO_DEBOUNCE_DELAY,
148 		.pio_mask	= ATA_PIO4,
149 		.udma_mask	= ATA_UDMA6,
150 		.port_ops	= &ahci_ops,
151 	},
152 	[board_ahci_nomsi] = {
153 		AHCI_HFLAGS	(AHCI_HFLAG_NO_MSI),
154 		.flags		= AHCI_FLAG_COMMON,
155 		.pio_mask	= ATA_PIO4,
156 		.udma_mask	= ATA_UDMA6,
157 		.port_ops	= &ahci_ops,
158 	},
159 	[board_ahci_noncq] = {
160 		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ),
161 		.flags		= AHCI_FLAG_COMMON,
162 		.pio_mask	= ATA_PIO4,
163 		.udma_mask	= ATA_UDMA6,
164 		.port_ops	= &ahci_ops,
165 	},
166 	[board_ahci_nosntf] = {
167 		AHCI_HFLAGS	(AHCI_HFLAG_NO_SNTF),
168 		.flags		= AHCI_FLAG_COMMON,
169 		.pio_mask	= ATA_PIO4,
170 		.udma_mask	= ATA_UDMA6,
171 		.port_ops	= &ahci_ops,
172 	},
173 	[board_ahci_yes_fbs] = {
174 		AHCI_HFLAGS	(AHCI_HFLAG_YES_FBS),
175 		.flags		= AHCI_FLAG_COMMON,
176 		.pio_mask	= ATA_PIO4,
177 		.udma_mask	= ATA_UDMA6,
178 		.port_ops	= &ahci_ops,
179 	},
180 	/* by chipsets */
181 	[board_ahci_al] = {
182 		AHCI_HFLAGS	(AHCI_HFLAG_NO_PMP | AHCI_HFLAG_NO_MSI),
183 		.flags		= AHCI_FLAG_COMMON,
184 		.pio_mask	= ATA_PIO4,
185 		.udma_mask	= ATA_UDMA6,
186 		.port_ops	= &ahci_ops,
187 	},
188 	[board_ahci_avn] = {
189 		.flags		= AHCI_FLAG_COMMON,
190 		.pio_mask	= ATA_PIO4,
191 		.udma_mask	= ATA_UDMA6,
192 		.port_ops	= &ahci_avn_ops,
193 	},
194 	[board_ahci_mcp65] = {
195 		AHCI_HFLAGS	(AHCI_HFLAG_NO_FPDMA_AA | AHCI_HFLAG_NO_PMP |
196 				 AHCI_HFLAG_YES_NCQ),
197 		.flags		= AHCI_FLAG_COMMON | ATA_FLAG_NO_DIPM,
198 		.pio_mask	= ATA_PIO4,
199 		.udma_mask	= ATA_UDMA6,
200 		.port_ops	= &ahci_ops,
201 	},
202 	[board_ahci_mcp77] = {
203 		AHCI_HFLAGS	(AHCI_HFLAG_NO_FPDMA_AA | AHCI_HFLAG_NO_PMP),
204 		.flags		= AHCI_FLAG_COMMON,
205 		.pio_mask	= ATA_PIO4,
206 		.udma_mask	= ATA_UDMA6,
207 		.port_ops	= &ahci_ops,
208 	},
209 	[board_ahci_mcp89] = {
210 		AHCI_HFLAGS	(AHCI_HFLAG_NO_FPDMA_AA),
211 		.flags		= AHCI_FLAG_COMMON,
212 		.pio_mask	= ATA_PIO4,
213 		.udma_mask	= ATA_UDMA6,
214 		.port_ops	= &ahci_ops,
215 	},
216 	[board_ahci_mv] = {
217 		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
218 				 AHCI_HFLAG_MV_PATA | AHCI_HFLAG_NO_PMP),
219 		.flags		= ATA_FLAG_SATA | ATA_FLAG_PIO_DMA,
220 		.pio_mask	= ATA_PIO4,
221 		.udma_mask	= ATA_UDMA6,
222 		.port_ops	= &ahci_ops,
223 	},
224 	[board_ahci_sb600] = {
225 		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL |
226 				 AHCI_HFLAG_NO_MSI | AHCI_HFLAG_SECT255 |
227 				 AHCI_HFLAG_32BIT_ONLY),
228 		.flags		= AHCI_FLAG_COMMON,
229 		.pio_mask	= ATA_PIO4,
230 		.udma_mask	= ATA_UDMA6,
231 		.port_ops	= &ahci_pmp_retry_srst_ops,
232 	},
233 	[board_ahci_sb700] = {	/* for SB700 and SB800 */
234 		AHCI_HFLAGS	(AHCI_HFLAG_IGN_SERR_INTERNAL),
235 		.flags		= AHCI_FLAG_COMMON,
236 		.pio_mask	= ATA_PIO4,
237 		.udma_mask	= ATA_UDMA6,
238 		.port_ops	= &ahci_pmp_retry_srst_ops,
239 	},
240 	[board_ahci_vt8251] = {
241 		AHCI_HFLAGS	(AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
242 		.flags		= AHCI_FLAG_COMMON,
243 		.pio_mask	= ATA_PIO4,
244 		.udma_mask	= ATA_UDMA6,
245 		.port_ops	= &ahci_vt8251_ops,
246 	},
247 	[board_ahci_pcs7] = {
248 		.flags		= AHCI_FLAG_COMMON,
249 		.pio_mask	= ATA_PIO4,
250 		.udma_mask	= ATA_UDMA6,
251 		.port_ops	= &ahci_ops,
252 	},
253 };
254 
255 static const struct pci_device_id ahci_pci_tbl[] = {
256 	/* Intel */
257 	{ PCI_VDEVICE(INTEL, 0x06d6), board_ahci }, /* Comet Lake PCH-H RAID */
258 	{ PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */
259 	{ PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */
260 	{ PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
261 	{ PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
262 	{ PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
263 	{ PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
264 	{ PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
265 	{ PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
266 	{ PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
267 	{ PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */
268 	{ PCI_VDEVICE(INTEL, 0x2821), board_ahci }, /* ICH8 */
269 	{ PCI_VDEVICE(INTEL, 0x2822), board_ahci_nosntf }, /* ICH8/Lewisburg RAID*/
270 	{ PCI_VDEVICE(INTEL, 0x2824), board_ahci }, /* ICH8 */
271 	{ PCI_VDEVICE(INTEL, 0x2829), board_ahci }, /* ICH8M */
272 	{ PCI_VDEVICE(INTEL, 0x282a), board_ahci }, /* ICH8M */
273 	{ PCI_VDEVICE(INTEL, 0x2922), board_ahci }, /* ICH9 */
274 	{ PCI_VDEVICE(INTEL, 0x2923), board_ahci }, /* ICH9 */
275 	{ PCI_VDEVICE(INTEL, 0x2924), board_ahci }, /* ICH9 */
276 	{ PCI_VDEVICE(INTEL, 0x2925), board_ahci }, /* ICH9 */
277 	{ PCI_VDEVICE(INTEL, 0x2927), board_ahci }, /* ICH9 */
278 	{ PCI_VDEVICE(INTEL, 0x2929), board_ahci_low_power }, /* ICH9M */
279 	{ PCI_VDEVICE(INTEL, 0x292a), board_ahci_low_power }, /* ICH9M */
280 	{ PCI_VDEVICE(INTEL, 0x292b), board_ahci_low_power }, /* ICH9M */
281 	{ PCI_VDEVICE(INTEL, 0x292c), board_ahci_low_power }, /* ICH9M */
282 	{ PCI_VDEVICE(INTEL, 0x292f), board_ahci_low_power }, /* ICH9M */
283 	{ PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
284 	{ PCI_VDEVICE(INTEL, 0x294e), board_ahci_low_power }, /* ICH9M */
285 	{ PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
286 	{ PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
287 	{ PCI_VDEVICE(INTEL, 0x3a05), board_ahci }, /* ICH10 */
288 	{ PCI_VDEVICE(INTEL, 0x3a22), board_ahci }, /* ICH10 */
289 	{ PCI_VDEVICE(INTEL, 0x3a25), board_ahci }, /* ICH10 */
290 	{ PCI_VDEVICE(INTEL, 0x3b22), board_ahci }, /* PCH AHCI */
291 	{ PCI_VDEVICE(INTEL, 0x3b23), board_ahci }, /* PCH AHCI */
292 	{ PCI_VDEVICE(INTEL, 0x3b24), board_ahci }, /* PCH RAID */
293 	{ PCI_VDEVICE(INTEL, 0x3b25), board_ahci }, /* PCH RAID */
294 	{ PCI_VDEVICE(INTEL, 0x3b29), board_ahci_low_power }, /* PCH M AHCI */
295 	{ PCI_VDEVICE(INTEL, 0x3b2b), board_ahci }, /* PCH RAID */
296 	{ PCI_VDEVICE(INTEL, 0x3b2c), board_ahci_low_power }, /* PCH M RAID */
297 	{ PCI_VDEVICE(INTEL, 0x3b2f), board_ahci }, /* PCH AHCI */
298 	{ PCI_VDEVICE(INTEL, 0x19b0), board_ahci_pcs7 }, /* DNV AHCI */
299 	{ PCI_VDEVICE(INTEL, 0x19b1), board_ahci_pcs7 }, /* DNV AHCI */
300 	{ PCI_VDEVICE(INTEL, 0x19b2), board_ahci_pcs7 }, /* DNV AHCI */
301 	{ PCI_VDEVICE(INTEL, 0x19b3), board_ahci_pcs7 }, /* DNV AHCI */
302 	{ PCI_VDEVICE(INTEL, 0x19b4), board_ahci_pcs7 }, /* DNV AHCI */
303 	{ PCI_VDEVICE(INTEL, 0x19b5), board_ahci_pcs7 }, /* DNV AHCI */
304 	{ PCI_VDEVICE(INTEL, 0x19b6), board_ahci_pcs7 }, /* DNV AHCI */
305 	{ PCI_VDEVICE(INTEL, 0x19b7), board_ahci_pcs7 }, /* DNV AHCI */
306 	{ PCI_VDEVICE(INTEL, 0x19bE), board_ahci_pcs7 }, /* DNV AHCI */
307 	{ PCI_VDEVICE(INTEL, 0x19bF), board_ahci_pcs7 }, /* DNV AHCI */
308 	{ PCI_VDEVICE(INTEL, 0x19c0), board_ahci_pcs7 }, /* DNV AHCI */
309 	{ PCI_VDEVICE(INTEL, 0x19c1), board_ahci_pcs7 }, /* DNV AHCI */
310 	{ PCI_VDEVICE(INTEL, 0x19c2), board_ahci_pcs7 }, /* DNV AHCI */
311 	{ PCI_VDEVICE(INTEL, 0x19c3), board_ahci_pcs7 }, /* DNV AHCI */
312 	{ PCI_VDEVICE(INTEL, 0x19c4), board_ahci_pcs7 }, /* DNV AHCI */
313 	{ PCI_VDEVICE(INTEL, 0x19c5), board_ahci_pcs7 }, /* DNV AHCI */
314 	{ PCI_VDEVICE(INTEL, 0x19c6), board_ahci_pcs7 }, /* DNV AHCI */
315 	{ PCI_VDEVICE(INTEL, 0x19c7), board_ahci_pcs7 }, /* DNV AHCI */
316 	{ PCI_VDEVICE(INTEL, 0x19cE), board_ahci_pcs7 }, /* DNV AHCI */
317 	{ PCI_VDEVICE(INTEL, 0x19cF), board_ahci_pcs7 }, /* DNV AHCI */
318 	{ PCI_VDEVICE(INTEL, 0x1c02), board_ahci }, /* CPT AHCI */
319 	{ PCI_VDEVICE(INTEL, 0x1c03), board_ahci_low_power }, /* CPT M AHCI */
320 	{ PCI_VDEVICE(INTEL, 0x1c04), board_ahci }, /* CPT RAID */
321 	{ PCI_VDEVICE(INTEL, 0x1c05), board_ahci_low_power }, /* CPT M RAID */
322 	{ PCI_VDEVICE(INTEL, 0x1c06), board_ahci }, /* CPT RAID */
323 	{ PCI_VDEVICE(INTEL, 0x1c07), board_ahci }, /* CPT RAID */
324 	{ PCI_VDEVICE(INTEL, 0x1d02), board_ahci }, /* PBG AHCI */
325 	{ PCI_VDEVICE(INTEL, 0x1d04), board_ahci }, /* PBG RAID */
326 	{ PCI_VDEVICE(INTEL, 0x1d06), board_ahci }, /* PBG RAID */
327 	{ PCI_VDEVICE(INTEL, 0x2323), board_ahci }, /* DH89xxCC AHCI */
328 	{ PCI_VDEVICE(INTEL, 0x1e02), board_ahci }, /* Panther Point AHCI */
329 	{ PCI_VDEVICE(INTEL, 0x1e03), board_ahci_low_power }, /* Panther M AHCI */
330 	{ PCI_VDEVICE(INTEL, 0x1e04), board_ahci }, /* Panther Point RAID */
331 	{ PCI_VDEVICE(INTEL, 0x1e05), board_ahci }, /* Panther Point RAID */
332 	{ PCI_VDEVICE(INTEL, 0x1e06), board_ahci }, /* Panther Point RAID */
333 	{ PCI_VDEVICE(INTEL, 0x1e07), board_ahci_low_power }, /* Panther M RAID */
334 	{ PCI_VDEVICE(INTEL, 0x1e0e), board_ahci }, /* Panther Point RAID */
335 	{ PCI_VDEVICE(INTEL, 0x8c02), board_ahci }, /* Lynx Point AHCI */
336 	{ PCI_VDEVICE(INTEL, 0x8c03), board_ahci_low_power }, /* Lynx M AHCI */
337 	{ PCI_VDEVICE(INTEL, 0x8c04), board_ahci }, /* Lynx Point RAID */
338 	{ PCI_VDEVICE(INTEL, 0x8c05), board_ahci_low_power }, /* Lynx M RAID */
339 	{ PCI_VDEVICE(INTEL, 0x8c06), board_ahci }, /* Lynx Point RAID */
340 	{ PCI_VDEVICE(INTEL, 0x8c07), board_ahci_low_power }, /* Lynx M RAID */
341 	{ PCI_VDEVICE(INTEL, 0x8c0e), board_ahci }, /* Lynx Point RAID */
342 	{ PCI_VDEVICE(INTEL, 0x8c0f), board_ahci_low_power }, /* Lynx M RAID */
343 	{ PCI_VDEVICE(INTEL, 0x9c02), board_ahci_low_power }, /* Lynx LP AHCI */
344 	{ PCI_VDEVICE(INTEL, 0x9c03), board_ahci_low_power }, /* Lynx LP AHCI */
345 	{ PCI_VDEVICE(INTEL, 0x9c04), board_ahci_low_power }, /* Lynx LP RAID */
346 	{ PCI_VDEVICE(INTEL, 0x9c05), board_ahci_low_power }, /* Lynx LP RAID */
347 	{ PCI_VDEVICE(INTEL, 0x9c06), board_ahci_low_power }, /* Lynx LP RAID */
348 	{ PCI_VDEVICE(INTEL, 0x9c07), board_ahci_low_power }, /* Lynx LP RAID */
349 	{ PCI_VDEVICE(INTEL, 0x9c0e), board_ahci_low_power }, /* Lynx LP RAID */
350 	{ PCI_VDEVICE(INTEL, 0x9c0f), board_ahci_low_power }, /* Lynx LP RAID */
351 	{ PCI_VDEVICE(INTEL, 0x9dd3), board_ahci_low_power }, /* Cannon Lake PCH-LP AHCI */
352 	{ PCI_VDEVICE(INTEL, 0x1f22), board_ahci }, /* Avoton AHCI */
353 	{ PCI_VDEVICE(INTEL, 0x1f23), board_ahci }, /* Avoton AHCI */
354 	{ PCI_VDEVICE(INTEL, 0x1f24), board_ahci }, /* Avoton RAID */
355 	{ PCI_VDEVICE(INTEL, 0x1f25), board_ahci }, /* Avoton RAID */
356 	{ PCI_VDEVICE(INTEL, 0x1f26), board_ahci }, /* Avoton RAID */
357 	{ PCI_VDEVICE(INTEL, 0x1f27), board_ahci }, /* Avoton RAID */
358 	{ PCI_VDEVICE(INTEL, 0x1f2e), board_ahci }, /* Avoton RAID */
359 	{ PCI_VDEVICE(INTEL, 0x1f2f), board_ahci }, /* Avoton RAID */
360 	{ PCI_VDEVICE(INTEL, 0x1f32), board_ahci_avn }, /* Avoton AHCI */
361 	{ PCI_VDEVICE(INTEL, 0x1f33), board_ahci_avn }, /* Avoton AHCI */
362 	{ PCI_VDEVICE(INTEL, 0x1f34), board_ahci_avn }, /* Avoton RAID */
363 	{ PCI_VDEVICE(INTEL, 0x1f35), board_ahci_avn }, /* Avoton RAID */
364 	{ PCI_VDEVICE(INTEL, 0x1f36), board_ahci_avn }, /* Avoton RAID */
365 	{ PCI_VDEVICE(INTEL, 0x1f37), board_ahci_avn }, /* Avoton RAID */
366 	{ PCI_VDEVICE(INTEL, 0x1f3e), board_ahci_avn }, /* Avoton RAID */
367 	{ PCI_VDEVICE(INTEL, 0x1f3f), board_ahci_avn }, /* Avoton RAID */
368 	{ PCI_VDEVICE(INTEL, 0x2823), board_ahci }, /* Wellsburg/Lewisburg AHCI*/
369 	{ PCI_VDEVICE(INTEL, 0x2826), board_ahci }, /* *burg SATA0 'RAID' */
370 	{ PCI_VDEVICE(INTEL, 0x2827), board_ahci }, /* *burg SATA1 'RAID' */
371 	{ PCI_VDEVICE(INTEL, 0x282f), board_ahci }, /* *burg SATA2 'RAID' */
372 	{ PCI_VDEVICE(INTEL, 0x43d4), board_ahci }, /* Rocket Lake PCH-H RAID */
373 	{ PCI_VDEVICE(INTEL, 0x43d5), board_ahci }, /* Rocket Lake PCH-H RAID */
374 	{ PCI_VDEVICE(INTEL, 0x43d6), board_ahci }, /* Rocket Lake PCH-H RAID */
375 	{ PCI_VDEVICE(INTEL, 0x43d7), board_ahci }, /* Rocket Lake PCH-H RAID */
376 	{ PCI_VDEVICE(INTEL, 0x8d02), board_ahci }, /* Wellsburg AHCI */
377 	{ PCI_VDEVICE(INTEL, 0x8d04), board_ahci }, /* Wellsburg RAID */
378 	{ PCI_VDEVICE(INTEL, 0x8d06), board_ahci }, /* Wellsburg RAID */
379 	{ PCI_VDEVICE(INTEL, 0x8d0e), board_ahci }, /* Wellsburg RAID */
380 	{ PCI_VDEVICE(INTEL, 0x8d62), board_ahci }, /* Wellsburg AHCI */
381 	{ PCI_VDEVICE(INTEL, 0x8d64), board_ahci }, /* Wellsburg RAID */
382 	{ PCI_VDEVICE(INTEL, 0x8d66), board_ahci }, /* Wellsburg RAID */
383 	{ PCI_VDEVICE(INTEL, 0x8d6e), board_ahci }, /* Wellsburg RAID */
384 	{ PCI_VDEVICE(INTEL, 0x23a3), board_ahci }, /* Coleto Creek AHCI */
385 	{ PCI_VDEVICE(INTEL, 0x9c83), board_ahci_low_power }, /* Wildcat LP AHCI */
386 	{ PCI_VDEVICE(INTEL, 0x9c85), board_ahci_low_power }, /* Wildcat LP RAID */
387 	{ PCI_VDEVICE(INTEL, 0x9c87), board_ahci_low_power }, /* Wildcat LP RAID */
388 	{ PCI_VDEVICE(INTEL, 0x9c8f), board_ahci_low_power }, /* Wildcat LP RAID */
389 	{ PCI_VDEVICE(INTEL, 0x8c82), board_ahci }, /* 9 Series AHCI */
390 	{ PCI_VDEVICE(INTEL, 0x8c83), board_ahci_low_power }, /* 9 Series M AHCI */
391 	{ PCI_VDEVICE(INTEL, 0x8c84), board_ahci }, /* 9 Series RAID */
392 	{ PCI_VDEVICE(INTEL, 0x8c85), board_ahci_low_power }, /* 9 Series M RAID */
393 	{ PCI_VDEVICE(INTEL, 0x8c86), board_ahci }, /* 9 Series RAID */
394 	{ PCI_VDEVICE(INTEL, 0x8c87), board_ahci_low_power }, /* 9 Series M RAID */
395 	{ PCI_VDEVICE(INTEL, 0x8c8e), board_ahci }, /* 9 Series RAID */
396 	{ PCI_VDEVICE(INTEL, 0x8c8f), board_ahci_low_power }, /* 9 Series M RAID */
397 	{ PCI_VDEVICE(INTEL, 0x9d03), board_ahci_low_power }, /* Sunrise LP AHCI */
398 	{ PCI_VDEVICE(INTEL, 0x9d05), board_ahci_low_power }, /* Sunrise LP RAID */
399 	{ PCI_VDEVICE(INTEL, 0x9d07), board_ahci_low_power }, /* Sunrise LP RAID */
400 	{ PCI_VDEVICE(INTEL, 0xa102), board_ahci }, /* Sunrise Point-H AHCI */
401 	{ PCI_VDEVICE(INTEL, 0xa103), board_ahci_low_power }, /* Sunrise M AHCI */
402 	{ PCI_VDEVICE(INTEL, 0xa105), board_ahci }, /* Sunrise Point-H RAID */
403 	{ PCI_VDEVICE(INTEL, 0xa106), board_ahci }, /* Sunrise Point-H RAID */
404 	{ PCI_VDEVICE(INTEL, 0xa107), board_ahci_low_power }, /* Sunrise M RAID */
405 	{ PCI_VDEVICE(INTEL, 0xa10f), board_ahci }, /* Sunrise Point-H RAID */
406 	{ PCI_VDEVICE(INTEL, 0xa182), board_ahci }, /* Lewisburg AHCI*/
407 	{ PCI_VDEVICE(INTEL, 0xa186), board_ahci }, /* Lewisburg RAID*/
408 	{ PCI_VDEVICE(INTEL, 0xa1d2), board_ahci }, /* Lewisburg RAID*/
409 	{ PCI_VDEVICE(INTEL, 0xa1d6), board_ahci }, /* Lewisburg RAID*/
410 	{ PCI_VDEVICE(INTEL, 0xa202), board_ahci }, /* Lewisburg AHCI*/
411 	{ PCI_VDEVICE(INTEL, 0xa206), board_ahci }, /* Lewisburg RAID*/
412 	{ PCI_VDEVICE(INTEL, 0xa252), board_ahci }, /* Lewisburg RAID*/
413 	{ PCI_VDEVICE(INTEL, 0xa256), board_ahci }, /* Lewisburg RAID*/
414 	{ PCI_VDEVICE(INTEL, 0xa356), board_ahci }, /* Cannon Lake PCH-H RAID */
415 	{ PCI_VDEVICE(INTEL, 0x06d7), board_ahci }, /* Comet Lake-H RAID */
416 	{ PCI_VDEVICE(INTEL, 0xa386), board_ahci }, /* Comet Lake PCH-V RAID */
417 	{ PCI_VDEVICE(INTEL, 0x0f22), board_ahci_low_power }, /* Bay Trail AHCI */
418 	{ PCI_VDEVICE(INTEL, 0x0f23), board_ahci_low_power }, /* Bay Trail AHCI */
419 	{ PCI_VDEVICE(INTEL, 0x22a3), board_ahci_low_power }, /* Cherry Tr. AHCI */
420 	{ PCI_VDEVICE(INTEL, 0x5ae3), board_ahci_low_power }, /* ApolloLake AHCI */
421 	{ PCI_VDEVICE(INTEL, 0x34d3), board_ahci_low_power }, /* Ice Lake LP AHCI */
422 	{ PCI_VDEVICE(INTEL, 0x02d3), board_ahci_low_power }, /* Comet Lake PCH-U AHCI */
423 	{ PCI_VDEVICE(INTEL, 0x02d7), board_ahci_low_power }, /* Comet Lake PCH RAID */
424 
425 	/* JMicron 360/1/3/5/6, match class to avoid IDE function */
426 	{ PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
427 	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
428 	/* JMicron 362B and 362C have an AHCI function with IDE class code */
429 	{ PCI_VDEVICE(JMICRON, 0x2362), board_ahci_ign_iferr },
430 	{ PCI_VDEVICE(JMICRON, 0x236f), board_ahci_ign_iferr },
431 	/* May need to update quirk_jmicron_async_suspend() for additions */
432 
433 	/* ATI */
434 	{ PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
435 	{ PCI_VDEVICE(ATI, 0x4390), board_ahci_sb700 }, /* ATI SB700/800 */
436 	{ PCI_VDEVICE(ATI, 0x4391), board_ahci_sb700 }, /* ATI SB700/800 */
437 	{ PCI_VDEVICE(ATI, 0x4392), board_ahci_sb700 }, /* ATI SB700/800 */
438 	{ PCI_VDEVICE(ATI, 0x4393), board_ahci_sb700 }, /* ATI SB700/800 */
439 	{ PCI_VDEVICE(ATI, 0x4394), board_ahci_sb700 }, /* ATI SB700/800 */
440 	{ PCI_VDEVICE(ATI, 0x4395), board_ahci_sb700 }, /* ATI SB700/800 */
441 
442 	/* Amazon's Annapurna Labs support */
443 	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON_ANNAPURNA_LABS, 0x0031),
444 		.class = PCI_CLASS_STORAGE_SATA_AHCI,
445 		.class_mask = 0xffffff,
446 		board_ahci_al },
447 	/* AMD */
448 	{ PCI_VDEVICE(AMD, 0x7800), board_ahci }, /* AMD Hudson-2 */
449 	{ PCI_VDEVICE(AMD, 0x7801), board_ahci_no_debounce_delay }, /* AMD Hudson-2 (AHCI mode) */
450 	{ PCI_VDEVICE(AMD, 0x7900), board_ahci }, /* AMD CZ */
451 	{ PCI_VDEVICE(AMD, 0x7901), board_ahci_low_power }, /* AMD Green Sardine */
452 	/* AMD is using RAID class only for ahci controllers */
453 	{ PCI_VENDOR_ID_AMD, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
454 	  PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci },
455 
456 	/* Dell S140/S150 */
457 	{ PCI_VENDOR_ID_INTEL, PCI_ANY_ID, PCI_SUBVENDOR_ID_DELL, PCI_ANY_ID,
458 	  PCI_CLASS_STORAGE_RAID << 8, 0xffffff, board_ahci },
459 
460 	/* VIA */
461 	{ PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
462 	{ PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
463 
464 	/* NVIDIA */
465 	{ PCI_VDEVICE(NVIDIA, 0x044c), board_ahci_mcp65 },	/* MCP65 */
466 	{ PCI_VDEVICE(NVIDIA, 0x044d), board_ahci_mcp65 },	/* MCP65 */
467 	{ PCI_VDEVICE(NVIDIA, 0x044e), board_ahci_mcp65 },	/* MCP65 */
468 	{ PCI_VDEVICE(NVIDIA, 0x044f), board_ahci_mcp65 },	/* MCP65 */
469 	{ PCI_VDEVICE(NVIDIA, 0x045c), board_ahci_mcp65 },	/* MCP65 */
470 	{ PCI_VDEVICE(NVIDIA, 0x045d), board_ahci_mcp65 },	/* MCP65 */
471 	{ PCI_VDEVICE(NVIDIA, 0x045e), board_ahci_mcp65 },	/* MCP65 */
472 	{ PCI_VDEVICE(NVIDIA, 0x045f), board_ahci_mcp65 },	/* MCP65 */
473 	{ PCI_VDEVICE(NVIDIA, 0x0550), board_ahci_mcp67 },	/* MCP67 */
474 	{ PCI_VDEVICE(NVIDIA, 0x0551), board_ahci_mcp67 },	/* MCP67 */
475 	{ PCI_VDEVICE(NVIDIA, 0x0552), board_ahci_mcp67 },	/* MCP67 */
476 	{ PCI_VDEVICE(NVIDIA, 0x0553), board_ahci_mcp67 },	/* MCP67 */
477 	{ PCI_VDEVICE(NVIDIA, 0x0554), board_ahci_mcp67 },	/* MCP67 */
478 	{ PCI_VDEVICE(NVIDIA, 0x0555), board_ahci_mcp67 },	/* MCP67 */
479 	{ PCI_VDEVICE(NVIDIA, 0x0556), board_ahci_mcp67 },	/* MCP67 */
480 	{ PCI_VDEVICE(NVIDIA, 0x0557), board_ahci_mcp67 },	/* MCP67 */
481 	{ PCI_VDEVICE(NVIDIA, 0x0558), board_ahci_mcp67 },	/* MCP67 */
482 	{ PCI_VDEVICE(NVIDIA, 0x0559), board_ahci_mcp67 },	/* MCP67 */
483 	{ PCI_VDEVICE(NVIDIA, 0x055a), board_ahci_mcp67 },	/* MCP67 */
484 	{ PCI_VDEVICE(NVIDIA, 0x055b), board_ahci_mcp67 },	/* MCP67 */
485 	{ PCI_VDEVICE(NVIDIA, 0x0580), board_ahci_mcp_linux },	/* Linux ID */
486 	{ PCI_VDEVICE(NVIDIA, 0x0581), board_ahci_mcp_linux },	/* Linux ID */
487 	{ PCI_VDEVICE(NVIDIA, 0x0582), board_ahci_mcp_linux },	/* Linux ID */
488 	{ PCI_VDEVICE(NVIDIA, 0x0583), board_ahci_mcp_linux },	/* Linux ID */
489 	{ PCI_VDEVICE(NVIDIA, 0x0584), board_ahci_mcp_linux },	/* Linux ID */
490 	{ PCI_VDEVICE(NVIDIA, 0x0585), board_ahci_mcp_linux },	/* Linux ID */
491 	{ PCI_VDEVICE(NVIDIA, 0x0586), board_ahci_mcp_linux },	/* Linux ID */
492 	{ PCI_VDEVICE(NVIDIA, 0x0587), board_ahci_mcp_linux },	/* Linux ID */
493 	{ PCI_VDEVICE(NVIDIA, 0x0588), board_ahci_mcp_linux },	/* Linux ID */
494 	{ PCI_VDEVICE(NVIDIA, 0x0589), board_ahci_mcp_linux },	/* Linux ID */
495 	{ PCI_VDEVICE(NVIDIA, 0x058a), board_ahci_mcp_linux },	/* Linux ID */
496 	{ PCI_VDEVICE(NVIDIA, 0x058b), board_ahci_mcp_linux },	/* Linux ID */
497 	{ PCI_VDEVICE(NVIDIA, 0x058c), board_ahci_mcp_linux },	/* Linux ID */
498 	{ PCI_VDEVICE(NVIDIA, 0x058d), board_ahci_mcp_linux },	/* Linux ID */
499 	{ PCI_VDEVICE(NVIDIA, 0x058e), board_ahci_mcp_linux },	/* Linux ID */
500 	{ PCI_VDEVICE(NVIDIA, 0x058f), board_ahci_mcp_linux },	/* Linux ID */
501 	{ PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci_mcp73 },	/* MCP73 */
502 	{ PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci_mcp73 },	/* MCP73 */
503 	{ PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci_mcp73 },	/* MCP73 */
504 	{ PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci_mcp73 },	/* MCP73 */
505 	{ PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci_mcp73 },	/* MCP73 */
506 	{ PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci_mcp73 },	/* MCP73 */
507 	{ PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci_mcp73 },	/* MCP73 */
508 	{ PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci_mcp73 },	/* MCP73 */
509 	{ PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci_mcp73 },	/* MCP73 */
510 	{ PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci_mcp73 },	/* MCP73 */
511 	{ PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci_mcp73 },	/* MCP73 */
512 	{ PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci_mcp73 },	/* MCP73 */
513 	{ PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci_mcp77 },	/* MCP77 */
514 	{ PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci_mcp77 },	/* MCP77 */
515 	{ PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci_mcp77 },	/* MCP77 */
516 	{ PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci_mcp77 },	/* MCP77 */
517 	{ PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci_mcp77 },	/* MCP77 */
518 	{ PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci_mcp77 },	/* MCP77 */
519 	{ PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci_mcp77 },	/* MCP77 */
520 	{ PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci_mcp77 },	/* MCP77 */
521 	{ PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci_mcp77 },	/* MCP77 */
522 	{ PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci_mcp77 },	/* MCP77 */
523 	{ PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci_mcp77 },	/* MCP77 */
524 	{ PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci_mcp77 },	/* MCP77 */
525 	{ PCI_VDEVICE(NVIDIA, 0x0ab4), board_ahci_mcp79 },	/* MCP79 */
526 	{ PCI_VDEVICE(NVIDIA, 0x0ab5), board_ahci_mcp79 },	/* MCP79 */
527 	{ PCI_VDEVICE(NVIDIA, 0x0ab6), board_ahci_mcp79 },	/* MCP79 */
528 	{ PCI_VDEVICE(NVIDIA, 0x0ab7), board_ahci_mcp79 },	/* MCP79 */
529 	{ PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci_mcp79 },	/* MCP79 */
530 	{ PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci_mcp79 },	/* MCP79 */
531 	{ PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci_mcp79 },	/* MCP79 */
532 	{ PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci_mcp79 },	/* MCP79 */
533 	{ PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci_mcp79 },	/* MCP79 */
534 	{ PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci_mcp79 },	/* MCP79 */
535 	{ PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci_mcp79 },	/* MCP79 */
536 	{ PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci_mcp79 },	/* MCP79 */
537 	{ PCI_VDEVICE(NVIDIA, 0x0d84), board_ahci_mcp89 },	/* MCP89 */
538 	{ PCI_VDEVICE(NVIDIA, 0x0d85), board_ahci_mcp89 },	/* MCP89 */
539 	{ PCI_VDEVICE(NVIDIA, 0x0d86), board_ahci_mcp89 },	/* MCP89 */
540 	{ PCI_VDEVICE(NVIDIA, 0x0d87), board_ahci_mcp89 },	/* MCP89 */
541 	{ PCI_VDEVICE(NVIDIA, 0x0d88), board_ahci_mcp89 },	/* MCP89 */
542 	{ PCI_VDEVICE(NVIDIA, 0x0d89), board_ahci_mcp89 },	/* MCP89 */
543 	{ PCI_VDEVICE(NVIDIA, 0x0d8a), board_ahci_mcp89 },	/* MCP89 */
544 	{ PCI_VDEVICE(NVIDIA, 0x0d8b), board_ahci_mcp89 },	/* MCP89 */
545 	{ PCI_VDEVICE(NVIDIA, 0x0d8c), board_ahci_mcp89 },	/* MCP89 */
546 	{ PCI_VDEVICE(NVIDIA, 0x0d8d), board_ahci_mcp89 },	/* MCP89 */
547 	{ PCI_VDEVICE(NVIDIA, 0x0d8e), board_ahci_mcp89 },	/* MCP89 */
548 	{ PCI_VDEVICE(NVIDIA, 0x0d8f), board_ahci_mcp89 },	/* MCP89 */
549 
550 	/* SiS */
551 	{ PCI_VDEVICE(SI, 0x1184), board_ahci },		/* SiS 966 */
552 	{ PCI_VDEVICE(SI, 0x1185), board_ahci },		/* SiS 968 */
553 	{ PCI_VDEVICE(SI, 0x0186), board_ahci },		/* SiS 968 */
554 
555 	/* ST Microelectronics */
556 	{ PCI_VDEVICE(STMICRO, 0xCC06), board_ahci },		/* ST ConneXt */
557 
558 	/* Marvell */
559 	{ PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },	/* 6145 */
560 	{ PCI_VDEVICE(MARVELL, 0x6121), board_ahci_mv },	/* 6121 */
561 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9123),
562 	  .class = PCI_CLASS_STORAGE_SATA_AHCI,
563 	  .class_mask = 0xffffff,
564 	  .driver_data = board_ahci_yes_fbs },			/* 88se9128 */
565 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9125),
566 	  .driver_data = board_ahci_yes_fbs },			/* 88se9125 */
567 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_MARVELL_EXT, 0x9178,
568 			 PCI_VENDOR_ID_MARVELL_EXT, 0x9170),
569 	  .driver_data = board_ahci_yes_fbs },			/* 88se9170 */
570 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x917a),
571 	  .driver_data = board_ahci_yes_fbs },			/* 88se9172 */
572 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9172),
573 	  .driver_data = board_ahci_yes_fbs },			/* 88se9182 */
574 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9182),
575 	  .driver_data = board_ahci_yes_fbs },			/* 88se9172 */
576 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9192),
577 	  .driver_data = board_ahci_yes_fbs },			/* 88se9172 on some Gigabyte */
578 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x91a0),
579 	  .driver_data = board_ahci_yes_fbs },
580 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x91a2), 	/* 88se91a2 */
581 	  .driver_data = board_ahci_yes_fbs },
582 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x91a3),
583 	  .driver_data = board_ahci_yes_fbs },
584 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9230),
585 	  .driver_data = board_ahci_yes_fbs },
586 	{ PCI_DEVICE(PCI_VENDOR_ID_MARVELL_EXT, 0x9235),
587 	  .driver_data = board_ahci_no_debounce_delay },
588 	{ PCI_DEVICE(PCI_VENDOR_ID_TTI, 0x0642), /* highpoint rocketraid 642L */
589 	  .driver_data = board_ahci_yes_fbs },
590 	{ PCI_DEVICE(PCI_VENDOR_ID_TTI, 0x0645), /* highpoint rocketraid 644L */
591 	  .driver_data = board_ahci_yes_fbs },
592 
593 	/* Promise */
594 	{ PCI_VDEVICE(PROMISE, 0x3f20), board_ahci },	/* PDC42819 */
595 	{ PCI_VDEVICE(PROMISE, 0x3781), board_ahci },   /* FastTrak TX8660 ahci-mode */
596 
597 	/* Asmedia */
598 	{ PCI_VDEVICE(ASMEDIA, 0x0601), board_ahci },	/* ASM1060 */
599 	{ PCI_VDEVICE(ASMEDIA, 0x0602), board_ahci },	/* ASM1060 */
600 	{ PCI_VDEVICE(ASMEDIA, 0x0611), board_ahci },	/* ASM1061 */
601 	{ PCI_VDEVICE(ASMEDIA, 0x0612), board_ahci },	/* ASM1062 */
602 	{ PCI_VDEVICE(ASMEDIA, 0x0621), board_ahci },   /* ASM1061R */
603 	{ PCI_VDEVICE(ASMEDIA, 0x0622), board_ahci },   /* ASM1062R */
604 	{ PCI_VDEVICE(ASMEDIA, 0x0624), board_ahci },   /* ASM1062+JMB575 */
605 
606 	/*
607 	 * Samsung SSDs found on some macbooks.  NCQ times out if MSI is
608 	 * enabled.  https://bugzilla.kernel.org/show_bug.cgi?id=60731
609 	 */
610 	{ PCI_VDEVICE(SAMSUNG, 0x1600), board_ahci_nomsi },
611 	{ PCI_VDEVICE(SAMSUNG, 0xa800), board_ahci_nomsi },
612 
613 	/* Enmotus */
614 	{ PCI_DEVICE(0x1c44, 0x8000), board_ahci },
615 
616 	/* Loongson */
617 	{ PCI_VDEVICE(LOONGSON, 0x7a08), board_ahci },
618 
619 	/* Generic, PCI class code for AHCI */
620 	{ PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
621 	  PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
622 
623 	{ }	/* terminate list */
624 };
625 
626 static const struct dev_pm_ops ahci_pci_pm_ops = {
627 	SET_SYSTEM_SLEEP_PM_OPS(ahci_pci_device_suspend, ahci_pci_device_resume)
628 	SET_RUNTIME_PM_OPS(ahci_pci_device_runtime_suspend,
629 			   ahci_pci_device_runtime_resume, NULL)
630 };
631 
632 static struct pci_driver ahci_pci_driver = {
633 	.name			= DRV_NAME,
634 	.id_table		= ahci_pci_tbl,
635 	.probe			= ahci_init_one,
636 	.remove			= ahci_remove_one,
637 	.shutdown		= ahci_shutdown_one,
638 	.driver = {
639 		.pm		= &ahci_pci_pm_ops,
640 	},
641 };
642 
643 #if IS_ENABLED(CONFIG_PATA_MARVELL)
644 static int marvell_enable;
645 #else
646 static int marvell_enable = 1;
647 #endif
648 module_param(marvell_enable, int, 0644);
649 MODULE_PARM_DESC(marvell_enable, "Marvell SATA via AHCI (1 = enabled)");
650 
651 static int mobile_lpm_policy = -1;
652 module_param(mobile_lpm_policy, int, 0644);
653 MODULE_PARM_DESC(mobile_lpm_policy, "Default LPM policy for mobile chipsets");
654 
655 static void ahci_pci_save_initial_config(struct pci_dev *pdev,
656 					 struct ahci_host_priv *hpriv)
657 {
658 	if (pdev->vendor == PCI_VENDOR_ID_JMICRON && pdev->device == 0x2361) {
659 		dev_info(&pdev->dev, "JMB361 has only one port\n");
660 		hpriv->force_port_map = 1;
661 	}
662 
663 	/*
664 	 * Temporary Marvell 6145 hack: PATA port presence
665 	 * is asserted through the standard AHCI port
666 	 * presence register, as bit 4 (counting from 0)
667 	 */
668 	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
669 		if (pdev->device == 0x6121)
670 			hpriv->mask_port_map = 0x3;
671 		else
672 			hpriv->mask_port_map = 0xf;
673 		dev_info(&pdev->dev,
674 			  "Disabling your PATA port. Use the boot option 'ahci.marvell_enable=0' to avoid this.\n");
675 	}
676 
677 	ahci_save_initial_config(&pdev->dev, hpriv);
678 }
679 
680 static void ahci_pci_init_controller(struct ata_host *host)
681 {
682 	struct ahci_host_priv *hpriv = host->private_data;
683 	struct pci_dev *pdev = to_pci_dev(host->dev);
684 	void __iomem *port_mmio;
685 	u32 tmp;
686 	int mv;
687 
688 	if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
689 		if (pdev->device == 0x6121)
690 			mv = 2;
691 		else
692 			mv = 4;
693 		port_mmio = __ahci_port_base(host, mv);
694 
695 		writel(0, port_mmio + PORT_IRQ_MASK);
696 
697 		/* clear port IRQ */
698 		tmp = readl(port_mmio + PORT_IRQ_STAT);
699 		dev_dbg(&pdev->dev, "PORT_IRQ_STAT 0x%x\n", tmp);
700 		if (tmp)
701 			writel(tmp, port_mmio + PORT_IRQ_STAT);
702 	}
703 
704 	ahci_init_controller(host);
705 }
706 
707 static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
708 				 unsigned long deadline)
709 {
710 	struct ata_port *ap = link->ap;
711 	struct ahci_host_priv *hpriv = ap->host->private_data;
712 	bool online;
713 	int rc;
714 
715 	hpriv->stop_engine(ap);
716 
717 	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
718 				 deadline, &online, NULL);
719 
720 	hpriv->start_engine(ap);
721 
722 	/* vt8251 doesn't clear BSY on signature FIS reception,
723 	 * request follow-up softreset.
724 	 */
725 	return online ? -EAGAIN : rc;
726 }
727 
728 static int ahci_p5wdh_hardreset(struct ata_link *link, unsigned int *class,
729 				unsigned long deadline)
730 {
731 	struct ata_port *ap = link->ap;
732 	struct ahci_port_priv *pp = ap->private_data;
733 	struct ahci_host_priv *hpriv = ap->host->private_data;
734 	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
735 	struct ata_taskfile tf;
736 	bool online;
737 	int rc;
738 
739 	hpriv->stop_engine(ap);
740 
741 	/* clear D2H reception area to properly wait for D2H FIS */
742 	ata_tf_init(link->device, &tf);
743 	tf.status = ATA_BUSY;
744 	ata_tf_to_fis(&tf, 0, 0, d2h_fis);
745 
746 	rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
747 				 deadline, &online, NULL);
748 
749 	hpriv->start_engine(ap);
750 
751 	/* The pseudo configuration device on SIMG4726 attached to
752 	 * ASUS P5W-DH Deluxe doesn't send signature FIS after
753 	 * hardreset if no device is attached to the first downstream
754 	 * port && the pseudo device locks up on SRST w/ PMP==0.  To
755 	 * work around this, wait for !BSY only briefly.  If BSY isn't
756 	 * cleared, perform CLO and proceed to IDENTIFY (achieved by
757 	 * ATA_LFLAG_NO_SRST and ATA_LFLAG_ASSUME_ATA).
758 	 *
759 	 * Wait for two seconds.  Devices attached to downstream port
760 	 * which can't process the following IDENTIFY after this will
761 	 * have to be reset again.  For most cases, this should
762 	 * suffice while making probing snappish enough.
763 	 */
764 	if (online) {
765 		rc = ata_wait_after_reset(link, jiffies + 2 * HZ,
766 					  ahci_check_ready);
767 		if (rc)
768 			ahci_kick_engine(ap);
769 	}
770 	return rc;
771 }
772 
773 /*
774  * ahci_avn_hardreset - attempt more aggressive recovery of Avoton ports.
775  *
776  * It has been observed with some SSDs that the timing of events in the
777  * link synchronization phase can leave the port in a state that can not
778  * be recovered by a SATA-hard-reset alone.  The failing signature is
779  * SStatus.DET stuck at 1 ("Device presence detected but Phy
780  * communication not established").  It was found that unloading and
781  * reloading the driver when this problem occurs allows the drive
782  * connection to be recovered (DET advanced to 0x3).  The critical
783  * component of reloading the driver is that the port state machines are
784  * reset by bouncing "port enable" in the AHCI PCS configuration
785  * register.  So, reproduce that effect by bouncing a port whenever we
786  * see DET==1 after a reset.
787  */
788 static int ahci_avn_hardreset(struct ata_link *link, unsigned int *class,
789 			      unsigned long deadline)
790 {
791 	const unsigned long *timing = sata_ehc_deb_timing(&link->eh_context);
792 	struct ata_port *ap = link->ap;
793 	struct ahci_port_priv *pp = ap->private_data;
794 	struct ahci_host_priv *hpriv = ap->host->private_data;
795 	u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
796 	unsigned long tmo = deadline - jiffies;
797 	struct ata_taskfile tf;
798 	bool online;
799 	int rc, i;
800 
801 	hpriv->stop_engine(ap);
802 
803 	for (i = 0; i < 2; i++) {
804 		u16 val;
805 		u32 sstatus;
806 		int port = ap->port_no;
807 		struct ata_host *host = ap->host;
808 		struct pci_dev *pdev = to_pci_dev(host->dev);
809 
810 		/* clear D2H reception area to properly wait for D2H FIS */
811 		ata_tf_init(link->device, &tf);
812 		tf.status = ATA_BUSY;
813 		ata_tf_to_fis(&tf, 0, 0, d2h_fis);
814 
815 		rc = sata_link_hardreset(link, timing, deadline, &online,
816 				ahci_check_ready);
817 
818 		if (sata_scr_read(link, SCR_STATUS, &sstatus) != 0 ||
819 				(sstatus & 0xf) != 1)
820 			break;
821 
822 		ata_link_info(link,  "avn bounce port%d\n", port);
823 
824 		pci_read_config_word(pdev, 0x92, &val);
825 		val &= ~(1 << port);
826 		pci_write_config_word(pdev, 0x92, val);
827 		ata_msleep(ap, 1000);
828 		val |= 1 << port;
829 		pci_write_config_word(pdev, 0x92, val);
830 		deadline += tmo;
831 	}
832 
833 	hpriv->start_engine(ap);
834 
835 	if (online)
836 		*class = ahci_dev_classify(ap);
837 
838 	return rc;
839 }
840 
841 
842 #ifdef CONFIG_PM
843 static void ahci_pci_disable_interrupts(struct ata_host *host)
844 {
845 	struct ahci_host_priv *hpriv = host->private_data;
846 	void __iomem *mmio = hpriv->mmio;
847 	u32 ctl;
848 
849 	/* AHCI spec rev1.1 section 8.3.3:
850 	 * Software must disable interrupts prior to requesting a
851 	 * transition of the HBA to D3 state.
852 	 */
853 	ctl = readl(mmio + HOST_CTL);
854 	ctl &= ~HOST_IRQ_EN;
855 	writel(ctl, mmio + HOST_CTL);
856 	readl(mmio + HOST_CTL); /* flush */
857 }
858 
859 static int ahci_pci_device_runtime_suspend(struct device *dev)
860 {
861 	struct pci_dev *pdev = to_pci_dev(dev);
862 	struct ata_host *host = pci_get_drvdata(pdev);
863 
864 	ahci_pci_disable_interrupts(host);
865 	return 0;
866 }
867 
868 static int ahci_pci_device_runtime_resume(struct device *dev)
869 {
870 	struct pci_dev *pdev = to_pci_dev(dev);
871 	struct ata_host *host = pci_get_drvdata(pdev);
872 	int rc;
873 
874 	rc = ahci_reset_controller(host);
875 	if (rc)
876 		return rc;
877 	ahci_pci_init_controller(host);
878 	return 0;
879 }
880 
881 #ifdef CONFIG_PM_SLEEP
882 static int ahci_pci_device_suspend(struct device *dev)
883 {
884 	struct pci_dev *pdev = to_pci_dev(dev);
885 	struct ata_host *host = pci_get_drvdata(pdev);
886 	struct ahci_host_priv *hpriv = host->private_data;
887 
888 	if (hpriv->flags & AHCI_HFLAG_NO_SUSPEND) {
889 		dev_err(&pdev->dev,
890 			"BIOS update required for suspend/resume\n");
891 		return -EIO;
892 	}
893 
894 	ahci_pci_disable_interrupts(host);
895 	ata_host_suspend(host, PMSG_SUSPEND);
896 	return 0;
897 }
898 
899 static int ahci_pci_device_resume(struct device *dev)
900 {
901 	struct pci_dev *pdev = to_pci_dev(dev);
902 	struct ata_host *host = pci_get_drvdata(pdev);
903 	int rc;
904 
905 	/* Apple BIOS helpfully mangles the registers on resume */
906 	if (is_mcp89_apple(pdev))
907 		ahci_mcp89_apple_enable(pdev);
908 
909 	if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
910 		rc = ahci_reset_controller(host);
911 		if (rc)
912 			return rc;
913 
914 		ahci_pci_init_controller(host);
915 	}
916 
917 	ata_host_resume(host);
918 
919 	return 0;
920 }
921 #endif
922 
923 #endif /* CONFIG_PM */
924 
925 static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
926 {
927 	const int dma_bits = using_dac ? 64 : 32;
928 	int rc;
929 
930 	/*
931 	 * If the device fixup already set the dma_mask to some non-standard
932 	 * value, don't extend it here. This happens on STA2X11, for example.
933 	 *
934 	 * XXX: manipulating the DMA mask from platform code is completely
935 	 * bogus, platform code should use dev->bus_dma_limit instead..
936 	 */
937 	if (pdev->dma_mask && pdev->dma_mask < DMA_BIT_MASK(32))
938 		return 0;
939 
940 	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_bits));
941 	if (rc)
942 		dev_err(&pdev->dev, "DMA enable failed\n");
943 	return rc;
944 }
945 
946 static void ahci_pci_print_info(struct ata_host *host)
947 {
948 	struct pci_dev *pdev = to_pci_dev(host->dev);
949 	u16 cc;
950 	const char *scc_s;
951 
952 	pci_read_config_word(pdev, 0x0a, &cc);
953 	if (cc == PCI_CLASS_STORAGE_IDE)
954 		scc_s = "IDE";
955 	else if (cc == PCI_CLASS_STORAGE_SATA)
956 		scc_s = "SATA";
957 	else if (cc == PCI_CLASS_STORAGE_RAID)
958 		scc_s = "RAID";
959 	else
960 		scc_s = "unknown";
961 
962 	ahci_print_info(host, scc_s);
963 }
964 
965 /* On ASUS P5W DH Deluxe, the second port of PCI device 00:1f.2 is
966  * hardwired to on-board SIMG 4726.  The chipset is ICH8 and doesn't
967  * support PMP and the 4726 either directly exports the device
968  * attached to the first downstream port or acts as a hardware storage
969  * controller and emulate a single ATA device (can be RAID 0/1 or some
970  * other configuration).
971  *
972  * When there's no device attached to the first downstream port of the
973  * 4726, "Config Disk" appears, which is a pseudo ATA device to
974  * configure the 4726.  However, ATA emulation of the device is very
975  * lame.  It doesn't send signature D2H Reg FIS after the initial
976  * hardreset, pukes on SRST w/ PMP==0 and has bunch of other issues.
977  *
978  * The following function works around the problem by always using
979  * hardreset on the port and not depending on receiving signature FIS
980  * afterward.  If signature FIS isn't received soon, ATA class is
981  * assumed without follow-up softreset.
982  */
983 static void ahci_p5wdh_workaround(struct ata_host *host)
984 {
985 	static const struct dmi_system_id sysids[] = {
986 		{
987 			.ident = "P5W DH Deluxe",
988 			.matches = {
989 				DMI_MATCH(DMI_SYS_VENDOR,
990 					  "ASUSTEK COMPUTER INC"),
991 				DMI_MATCH(DMI_PRODUCT_NAME, "P5W DH Deluxe"),
992 			},
993 		},
994 		{ }
995 	};
996 	struct pci_dev *pdev = to_pci_dev(host->dev);
997 
998 	if (pdev->bus->number == 0 && pdev->devfn == PCI_DEVFN(0x1f, 2) &&
999 	    dmi_check_system(sysids)) {
1000 		struct ata_port *ap = host->ports[1];
1001 
1002 		dev_info(&pdev->dev,
1003 			 "enabling ASUS P5W DH Deluxe on-board SIMG4726 workaround\n");
1004 
1005 		ap->ops = &ahci_p5wdh_ops;
1006 		ap->link.flags |= ATA_LFLAG_NO_SRST | ATA_LFLAG_ASSUME_ATA;
1007 	}
1008 }
1009 
1010 /*
1011  * Macbook7,1 firmware forcibly disables MCP89 AHCI and changes PCI ID when
1012  * booting in BIOS compatibility mode.  We restore the registers but not ID.
1013  */
1014 static void ahci_mcp89_apple_enable(struct pci_dev *pdev)
1015 {
1016 	u32 val;
1017 
1018 	printk(KERN_INFO "ahci: enabling MCP89 AHCI mode\n");
1019 
1020 	pci_read_config_dword(pdev, 0xf8, &val);
1021 	val |= 1 << 0x1b;
1022 	/* the following changes the device ID, but appears not to affect function */
1023 	/* val = (val & ~0xf0000000) | 0x80000000; */
1024 	pci_write_config_dword(pdev, 0xf8, val);
1025 
1026 	pci_read_config_dword(pdev, 0x54c, &val);
1027 	val |= 1 << 0xc;
1028 	pci_write_config_dword(pdev, 0x54c, val);
1029 
1030 	pci_read_config_dword(pdev, 0x4a4, &val);
1031 	val &= 0xff;
1032 	val |= 0x01060100;
1033 	pci_write_config_dword(pdev, 0x4a4, val);
1034 
1035 	pci_read_config_dword(pdev, 0x54c, &val);
1036 	val &= ~(1 << 0xc);
1037 	pci_write_config_dword(pdev, 0x54c, val);
1038 
1039 	pci_read_config_dword(pdev, 0xf8, &val);
1040 	val &= ~(1 << 0x1b);
1041 	pci_write_config_dword(pdev, 0xf8, val);
1042 }
1043 
1044 static bool is_mcp89_apple(struct pci_dev *pdev)
1045 {
1046 	return pdev->vendor == PCI_VENDOR_ID_NVIDIA &&
1047 		pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP89_SATA &&
1048 		pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE &&
1049 		pdev->subsystem_device == 0xcb89;
1050 }
1051 
1052 /* only some SB600 ahci controllers can do 64bit DMA */
1053 static bool ahci_sb600_enable_64bit(struct pci_dev *pdev)
1054 {
1055 	static const struct dmi_system_id sysids[] = {
1056 		/*
1057 		 * The oldest version known to be broken is 0901 and
1058 		 * working is 1501 which was released on 2007-10-26.
1059 		 * Enable 64bit DMA on 1501 and anything newer.
1060 		 *
1061 		 * Please read bko#9412 for more info.
1062 		 */
1063 		{
1064 			.ident = "ASUS M2A-VM",
1065 			.matches = {
1066 				DMI_MATCH(DMI_BOARD_VENDOR,
1067 					  "ASUSTeK Computer INC."),
1068 				DMI_MATCH(DMI_BOARD_NAME, "M2A-VM"),
1069 			},
1070 			.driver_data = "20071026",	/* yyyymmdd */
1071 		},
1072 		/*
1073 		 * All BIOS versions for the MSI K9A2 Platinum (MS-7376)
1074 		 * support 64bit DMA.
1075 		 *
1076 		 * BIOS versions earlier than 1.5 had the Manufacturer DMI
1077 		 * fields as "MICRO-STAR INTERANTIONAL CO.,LTD".
1078 		 * This spelling mistake was fixed in BIOS version 1.5, so
1079 		 * 1.5 and later have the Manufacturer as
1080 		 * "MICRO-STAR INTERNATIONAL CO.,LTD".
1081 		 * So try to match on DMI_BOARD_VENDOR of "MICRO-STAR INTER".
1082 		 *
1083 		 * BIOS versions earlier than 1.9 had a Board Product Name
1084 		 * DMI field of "MS-7376". This was changed to be
1085 		 * "K9A2 Platinum (MS-7376)" in version 1.9, but we can still
1086 		 * match on DMI_BOARD_NAME of "MS-7376".
1087 		 */
1088 		{
1089 			.ident = "MSI K9A2 Platinum",
1090 			.matches = {
1091 				DMI_MATCH(DMI_BOARD_VENDOR,
1092 					  "MICRO-STAR INTER"),
1093 				DMI_MATCH(DMI_BOARD_NAME, "MS-7376"),
1094 			},
1095 		},
1096 		/*
1097 		 * All BIOS versions for the MSI K9AGM2 (MS-7327) support
1098 		 * 64bit DMA.
1099 		 *
1100 		 * This board also had the typo mentioned above in the
1101 		 * Manufacturer DMI field (fixed in BIOS version 1.5), so
1102 		 * match on DMI_BOARD_VENDOR of "MICRO-STAR INTER" again.
1103 		 */
1104 		{
1105 			.ident = "MSI K9AGM2",
1106 			.matches = {
1107 				DMI_MATCH(DMI_BOARD_VENDOR,
1108 					  "MICRO-STAR INTER"),
1109 				DMI_MATCH(DMI_BOARD_NAME, "MS-7327"),
1110 			},
1111 		},
1112 		/*
1113 		 * All BIOS versions for the Asus M3A support 64bit DMA.
1114 		 * (all release versions from 0301 to 1206 were tested)
1115 		 */
1116 		{
1117 			.ident = "ASUS M3A",
1118 			.matches = {
1119 				DMI_MATCH(DMI_BOARD_VENDOR,
1120 					  "ASUSTeK Computer INC."),
1121 				DMI_MATCH(DMI_BOARD_NAME, "M3A"),
1122 			},
1123 		},
1124 		{ }
1125 	};
1126 	const struct dmi_system_id *match;
1127 	int year, month, date;
1128 	char buf[9];
1129 
1130 	match = dmi_first_match(sysids);
1131 	if (pdev->bus->number != 0 || pdev->devfn != PCI_DEVFN(0x12, 0) ||
1132 	    !match)
1133 		return false;
1134 
1135 	if (!match->driver_data)
1136 		goto enable_64bit;
1137 
1138 	dmi_get_date(DMI_BIOS_DATE, &year, &month, &date);
1139 	snprintf(buf, sizeof(buf), "%04d%02d%02d", year, month, date);
1140 
1141 	if (strcmp(buf, match->driver_data) >= 0)
1142 		goto enable_64bit;
1143 	else {
1144 		dev_warn(&pdev->dev,
1145 			 "%s: BIOS too old, forcing 32bit DMA, update BIOS\n",
1146 			 match->ident);
1147 		return false;
1148 	}
1149 
1150 enable_64bit:
1151 	dev_warn(&pdev->dev, "%s: enabling 64bit DMA\n", match->ident);
1152 	return true;
1153 }
1154 
1155 static bool ahci_broken_system_poweroff(struct pci_dev *pdev)
1156 {
1157 	static const struct dmi_system_id broken_systems[] = {
1158 		{
1159 			.ident = "HP Compaq nx6310",
1160 			.matches = {
1161 				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1162 				DMI_MATCH(DMI_PRODUCT_NAME, "HP Compaq nx6310"),
1163 			},
1164 			/* PCI slot number of the controller */
1165 			.driver_data = (void *)0x1FUL,
1166 		},
1167 		{
1168 			.ident = "HP Compaq 6720s",
1169 			.matches = {
1170 				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1171 				DMI_MATCH(DMI_PRODUCT_NAME, "HP Compaq 6720s"),
1172 			},
1173 			/* PCI slot number of the controller */
1174 			.driver_data = (void *)0x1FUL,
1175 		},
1176 
1177 		{ }	/* terminate list */
1178 	};
1179 	const struct dmi_system_id *dmi = dmi_first_match(broken_systems);
1180 
1181 	if (dmi) {
1182 		unsigned long slot = (unsigned long)dmi->driver_data;
1183 		/* apply the quirk only to on-board controllers */
1184 		return slot == PCI_SLOT(pdev->devfn);
1185 	}
1186 
1187 	return false;
1188 }
1189 
1190 static bool ahci_broken_suspend(struct pci_dev *pdev)
1191 {
1192 	static const struct dmi_system_id sysids[] = {
1193 		/*
1194 		 * On HP dv[4-6] and HDX18 with earlier BIOSen, link
1195 		 * to the harddisk doesn't become online after
1196 		 * resuming from STR.  Warn and fail suspend.
1197 		 *
1198 		 * http://bugzilla.kernel.org/show_bug.cgi?id=12276
1199 		 *
1200 		 * Use dates instead of versions to match as HP is
1201 		 * apparently recycling both product and version
1202 		 * strings.
1203 		 *
1204 		 * http://bugzilla.kernel.org/show_bug.cgi?id=15462
1205 		 */
1206 		{
1207 			.ident = "dv4",
1208 			.matches = {
1209 				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1210 				DMI_MATCH(DMI_PRODUCT_NAME,
1211 					  "HP Pavilion dv4 Notebook PC"),
1212 			},
1213 			.driver_data = "20090105",	/* F.30 */
1214 		},
1215 		{
1216 			.ident = "dv5",
1217 			.matches = {
1218 				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1219 				DMI_MATCH(DMI_PRODUCT_NAME,
1220 					  "HP Pavilion dv5 Notebook PC"),
1221 			},
1222 			.driver_data = "20090506",	/* F.16 */
1223 		},
1224 		{
1225 			.ident = "dv6",
1226 			.matches = {
1227 				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1228 				DMI_MATCH(DMI_PRODUCT_NAME,
1229 					  "HP Pavilion dv6 Notebook PC"),
1230 			},
1231 			.driver_data = "20090423",	/* F.21 */
1232 		},
1233 		{
1234 			.ident = "HDX18",
1235 			.matches = {
1236 				DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1237 				DMI_MATCH(DMI_PRODUCT_NAME,
1238 					  "HP HDX18 Notebook PC"),
1239 			},
1240 			.driver_data = "20090430",	/* F.23 */
1241 		},
1242 		/*
1243 		 * Acer eMachines G725 has the same problem.  BIOS
1244 		 * V1.03 is known to be broken.  V3.04 is known to
1245 		 * work.  Between, there are V1.06, V2.06 and V3.03
1246 		 * that we don't have much idea about.  For now,
1247 		 * blacklist anything older than V3.04.
1248 		 *
1249 		 * http://bugzilla.kernel.org/show_bug.cgi?id=15104
1250 		 */
1251 		{
1252 			.ident = "G725",
1253 			.matches = {
1254 				DMI_MATCH(DMI_SYS_VENDOR, "eMachines"),
1255 				DMI_MATCH(DMI_PRODUCT_NAME, "eMachines G725"),
1256 			},
1257 			.driver_data = "20091216",	/* V3.04 */
1258 		},
1259 		{ }	/* terminate list */
1260 	};
1261 	const struct dmi_system_id *dmi = dmi_first_match(sysids);
1262 	int year, month, date;
1263 	char buf[9];
1264 
1265 	if (!dmi || pdev->bus->number || pdev->devfn != PCI_DEVFN(0x1f, 2))
1266 		return false;
1267 
1268 	dmi_get_date(DMI_BIOS_DATE, &year, &month, &date);
1269 	snprintf(buf, sizeof(buf), "%04d%02d%02d", year, month, date);
1270 
1271 	return strcmp(buf, dmi->driver_data) < 0;
1272 }
1273 
1274 static bool ahci_broken_lpm(struct pci_dev *pdev)
1275 {
1276 	static const struct dmi_system_id sysids[] = {
1277 		/* Various Lenovo 50 series have LPM issues with older BIOSen */
1278 		{
1279 			.matches = {
1280 				DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
1281 				DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad X250"),
1282 			},
1283 			.driver_data = "20180406", /* 1.31 */
1284 		},
1285 		{
1286 			.matches = {
1287 				DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
1288 				DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad L450"),
1289 			},
1290 			.driver_data = "20180420", /* 1.28 */
1291 		},
1292 		{
1293 			.matches = {
1294 				DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
1295 				DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad T450s"),
1296 			},
1297 			.driver_data = "20180315", /* 1.33 */
1298 		},
1299 		{
1300 			.matches = {
1301 				DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
1302 				DMI_MATCH(DMI_PRODUCT_VERSION, "ThinkPad W541"),
1303 			},
1304 			/*
1305 			 * Note date based on release notes, 2.35 has been
1306 			 * reported to be good, but I've been unable to get
1307 			 * a hold of the reporter to get the DMI BIOS date.
1308 			 * TODO: fix this.
1309 			 */
1310 			.driver_data = "20180310", /* 2.35 */
1311 		},
1312 		{ }	/* terminate list */
1313 	};
1314 	const struct dmi_system_id *dmi = dmi_first_match(sysids);
1315 	int year, month, date;
1316 	char buf[9];
1317 
1318 	if (!dmi)
1319 		return false;
1320 
1321 	dmi_get_date(DMI_BIOS_DATE, &year, &month, &date);
1322 	snprintf(buf, sizeof(buf), "%04d%02d%02d", year, month, date);
1323 
1324 	return strcmp(buf, dmi->driver_data) < 0;
1325 }
1326 
1327 static bool ahci_broken_online(struct pci_dev *pdev)
1328 {
1329 #define ENCODE_BUSDEVFN(bus, slot, func)			\
1330 	(void *)(unsigned long)(((bus) << 8) | PCI_DEVFN((slot), (func)))
1331 	static const struct dmi_system_id sysids[] = {
1332 		/*
1333 		 * There are several gigabyte boards which use
1334 		 * SIMG5723s configured as hardware RAID.  Certain
1335 		 * 5723 firmware revisions shipped there keep the link
1336 		 * online but fail to answer properly to SRST or
1337 		 * IDENTIFY when no device is attached downstream
1338 		 * causing libata to retry quite a few times leading
1339 		 * to excessive detection delay.
1340 		 *
1341 		 * As these firmwares respond to the second reset try
1342 		 * with invalid device signature, considering unknown
1343 		 * sig as offline works around the problem acceptably.
1344 		 */
1345 		{
1346 			.ident = "EP45-DQ6",
1347 			.matches = {
1348 				DMI_MATCH(DMI_BOARD_VENDOR,
1349 					  "Gigabyte Technology Co., Ltd."),
1350 				DMI_MATCH(DMI_BOARD_NAME, "EP45-DQ6"),
1351 			},
1352 			.driver_data = ENCODE_BUSDEVFN(0x0a, 0x00, 0),
1353 		},
1354 		{
1355 			.ident = "EP45-DS5",
1356 			.matches = {
1357 				DMI_MATCH(DMI_BOARD_VENDOR,
1358 					  "Gigabyte Technology Co., Ltd."),
1359 				DMI_MATCH(DMI_BOARD_NAME, "EP45-DS5"),
1360 			},
1361 			.driver_data = ENCODE_BUSDEVFN(0x03, 0x00, 0),
1362 		},
1363 		{ }	/* terminate list */
1364 	};
1365 #undef ENCODE_BUSDEVFN
1366 	const struct dmi_system_id *dmi = dmi_first_match(sysids);
1367 	unsigned int val;
1368 
1369 	if (!dmi)
1370 		return false;
1371 
1372 	val = (unsigned long)dmi->driver_data;
1373 
1374 	return pdev->bus->number == (val >> 8) && pdev->devfn == (val & 0xff);
1375 }
1376 
1377 static bool ahci_broken_devslp(struct pci_dev *pdev)
1378 {
1379 	/* device with broken DEVSLP but still showing SDS capability */
1380 	static const struct pci_device_id ids[] = {
1381 		{ PCI_VDEVICE(INTEL, 0x0f23)}, /* Valleyview SoC */
1382 		{}
1383 	};
1384 
1385 	return pci_match_id(ids, pdev);
1386 }
1387 
1388 #ifdef CONFIG_ATA_ACPI
1389 static void ahci_gtf_filter_workaround(struct ata_host *host)
1390 {
1391 	static const struct dmi_system_id sysids[] = {
1392 		/*
1393 		 * Aspire 3810T issues a bunch of SATA enable commands
1394 		 * via _GTF including an invalid one and one which is
1395 		 * rejected by the device.  Among the successful ones
1396 		 * is FPDMA non-zero offset enable which when enabled
1397 		 * only on the drive side leads to NCQ command
1398 		 * failures.  Filter it out.
1399 		 */
1400 		{
1401 			.ident = "Aspire 3810T",
1402 			.matches = {
1403 				DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
1404 				DMI_MATCH(DMI_PRODUCT_NAME, "Aspire 3810T"),
1405 			},
1406 			.driver_data = (void *)ATA_ACPI_FILTER_FPDMA_OFFSET,
1407 		},
1408 		{ }
1409 	};
1410 	const struct dmi_system_id *dmi = dmi_first_match(sysids);
1411 	unsigned int filter;
1412 	int i;
1413 
1414 	if (!dmi)
1415 		return;
1416 
1417 	filter = (unsigned long)dmi->driver_data;
1418 	dev_info(host->dev, "applying extra ACPI _GTF filter 0x%x for %s\n",
1419 		 filter, dmi->ident);
1420 
1421 	for (i = 0; i < host->n_ports; i++) {
1422 		struct ata_port *ap = host->ports[i];
1423 		struct ata_link *link;
1424 		struct ata_device *dev;
1425 
1426 		ata_for_each_link(link, ap, EDGE)
1427 			ata_for_each_dev(dev, link, ALL)
1428 				dev->gtf_filter |= filter;
1429 	}
1430 }
1431 #else
1432 static inline void ahci_gtf_filter_workaround(struct ata_host *host)
1433 {}
1434 #endif
1435 
1436 /*
1437  * On the Acer Aspire Switch Alpha 12, sometimes all SATA ports are detected
1438  * as DUMMY, or detected but eventually get a "link down" and never get up
1439  * again. When this happens, CAP.NP may hold a value of 0x00 or 0x01, and the
1440  * port_map may hold a value of 0x00.
1441  *
1442  * Overriding CAP.NP to 0x02 and the port_map to 0x7 will reveal all 3 ports
1443  * and can significantly reduce the occurrence of the problem.
1444  *
1445  * https://bugzilla.kernel.org/show_bug.cgi?id=189471
1446  */
1447 static void acer_sa5_271_workaround(struct ahci_host_priv *hpriv,
1448 				    struct pci_dev *pdev)
1449 {
1450 	static const struct dmi_system_id sysids[] = {
1451 		{
1452 			.ident = "Acer Switch Alpha 12",
1453 			.matches = {
1454 				DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
1455 				DMI_MATCH(DMI_PRODUCT_NAME, "Switch SA5-271")
1456 			},
1457 		},
1458 		{ }
1459 	};
1460 
1461 	if (dmi_check_system(sysids)) {
1462 		dev_info(&pdev->dev, "enabling Acer Switch Alpha 12 workaround\n");
1463 		if ((hpriv->saved_cap & 0xC734FF00) == 0xC734FF00) {
1464 			hpriv->port_map = 0x7;
1465 			hpriv->cap = 0xC734FF02;
1466 		}
1467 	}
1468 }
1469 
1470 #ifdef CONFIG_ARM64
1471 /*
1472  * Due to ERRATA#22536, ThunderX needs to handle HOST_IRQ_STAT differently.
1473  * Workaround is to make sure all pending IRQs are served before leaving
1474  * handler.
1475  */
1476 static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance)
1477 {
1478 	struct ata_host *host = dev_instance;
1479 	struct ahci_host_priv *hpriv;
1480 	unsigned int rc = 0;
1481 	void __iomem *mmio;
1482 	u32 irq_stat, irq_masked;
1483 	unsigned int handled = 1;
1484 
1485 	hpriv = host->private_data;
1486 	mmio = hpriv->mmio;
1487 	irq_stat = readl(mmio + HOST_IRQ_STAT);
1488 	if (!irq_stat)
1489 		return IRQ_NONE;
1490 
1491 	do {
1492 		irq_masked = irq_stat & hpriv->port_map;
1493 		spin_lock(&host->lock);
1494 		rc = ahci_handle_port_intr(host, irq_masked);
1495 		if (!rc)
1496 			handled = 0;
1497 		writel(irq_stat, mmio + HOST_IRQ_STAT);
1498 		irq_stat = readl(mmio + HOST_IRQ_STAT);
1499 		spin_unlock(&host->lock);
1500 	} while (irq_stat);
1501 
1502 	return IRQ_RETVAL(handled);
1503 }
1504 #endif
1505 
1506 static void ahci_remap_check(struct pci_dev *pdev, int bar,
1507 		struct ahci_host_priv *hpriv)
1508 {
1509 	int i;
1510 	u32 cap;
1511 
1512 	/*
1513 	 * Check if this device might have remapped nvme devices.
1514 	 */
1515 	if (pdev->vendor != PCI_VENDOR_ID_INTEL ||
1516 	    pci_resource_len(pdev, bar) < SZ_512K ||
1517 	    bar != AHCI_PCI_BAR_STANDARD ||
1518 	    !(readl(hpriv->mmio + AHCI_VSCAP) & 1))
1519 		return;
1520 
1521 	cap = readq(hpriv->mmio + AHCI_REMAP_CAP);
1522 	for (i = 0; i < AHCI_MAX_REMAP; i++) {
1523 		if ((cap & (1 << i)) == 0)
1524 			continue;
1525 		if (readl(hpriv->mmio + ahci_remap_dcc(i))
1526 				!= PCI_CLASS_STORAGE_EXPRESS)
1527 			continue;
1528 
1529 		/* We've found a remapped device */
1530 		hpriv->remapped_nvme++;
1531 	}
1532 
1533 	if (!hpriv->remapped_nvme)
1534 		return;
1535 
1536 	dev_warn(&pdev->dev, "Found %u remapped NVMe devices.\n",
1537 		 hpriv->remapped_nvme);
1538 	dev_warn(&pdev->dev,
1539 		 "Switch your BIOS from RAID to AHCI mode to use them.\n");
1540 
1541 	/*
1542 	 * Don't rely on the msi-x capability in the remap case,
1543 	 * share the legacy interrupt across ahci and remapped devices.
1544 	 */
1545 	hpriv->flags |= AHCI_HFLAG_NO_MSI;
1546 }
1547 
1548 static int ahci_get_irq_vector(struct ata_host *host, int port)
1549 {
1550 	return pci_irq_vector(to_pci_dev(host->dev), port);
1551 }
1552 
1553 static int ahci_init_msi(struct pci_dev *pdev, unsigned int n_ports,
1554 			struct ahci_host_priv *hpriv)
1555 {
1556 	int nvec;
1557 
1558 	if (hpriv->flags & AHCI_HFLAG_NO_MSI)
1559 		return -ENODEV;
1560 
1561 	/*
1562 	 * If number of MSIs is less than number of ports then Sharing Last
1563 	 * Message mode could be enforced. In this case assume that advantage
1564 	 * of multipe MSIs is negated and use single MSI mode instead.
1565 	 */
1566 	if (n_ports > 1) {
1567 		nvec = pci_alloc_irq_vectors(pdev, n_ports, INT_MAX,
1568 				PCI_IRQ_MSIX | PCI_IRQ_MSI);
1569 		if (nvec > 0) {
1570 			if (!(readl(hpriv->mmio + HOST_CTL) & HOST_MRSM)) {
1571 				hpriv->get_irq_vector = ahci_get_irq_vector;
1572 				hpriv->flags |= AHCI_HFLAG_MULTI_MSI;
1573 				return nvec;
1574 			}
1575 
1576 			/*
1577 			 * Fallback to single MSI mode if the controller
1578 			 * enforced MRSM mode.
1579 			 */
1580 			printk(KERN_INFO
1581 				"ahci: MRSM is on, fallback to single MSI\n");
1582 			pci_free_irq_vectors(pdev);
1583 		}
1584 	}
1585 
1586 	/*
1587 	 * If the host is not capable of supporting per-port vectors, fall
1588 	 * back to single MSI before finally attempting single MSI-X.
1589 	 */
1590 	nvec = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
1591 	if (nvec == 1)
1592 		return nvec;
1593 	return pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSIX);
1594 }
1595 
1596 static void ahci_update_initial_lpm_policy(struct ata_port *ap,
1597 					   struct ahci_host_priv *hpriv)
1598 {
1599 	int policy = CONFIG_SATA_MOBILE_LPM_POLICY;
1600 
1601 
1602 	/* Ignore processing for chipsets that don't use policy */
1603 	if (!(hpriv->flags & AHCI_HFLAG_USE_LPM_POLICY))
1604 		return;
1605 
1606 	/* user modified policy via module param */
1607 	if (mobile_lpm_policy != -1) {
1608 		policy = mobile_lpm_policy;
1609 		goto update_policy;
1610 	}
1611 
1612 #ifdef CONFIG_ACPI
1613 	if (policy > ATA_LPM_MED_POWER &&
1614 	    (acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0)) {
1615 		if (hpriv->cap & HOST_CAP_PART)
1616 			policy = ATA_LPM_MIN_POWER_WITH_PARTIAL;
1617 		else if (hpriv->cap & HOST_CAP_SSC)
1618 			policy = ATA_LPM_MIN_POWER;
1619 	}
1620 #endif
1621 
1622 update_policy:
1623 	if (policy >= ATA_LPM_UNKNOWN && policy <= ATA_LPM_MIN_POWER)
1624 		ap->target_lpm_policy = policy;
1625 }
1626 
1627 static void ahci_intel_pcs_quirk(struct pci_dev *pdev, struct ahci_host_priv *hpriv)
1628 {
1629 	const struct pci_device_id *id = pci_match_id(ahci_pci_tbl, pdev);
1630 	u16 tmp16;
1631 
1632 	/*
1633 	 * Only apply the 6-port PCS quirk for known legacy platforms.
1634 	 */
1635 	if (!id || id->vendor != PCI_VENDOR_ID_INTEL)
1636 		return;
1637 
1638 	/* Skip applying the quirk on Denverton and beyond */
1639 	if (((enum board_ids) id->driver_data) >= board_ahci_pcs7)
1640 		return;
1641 
1642 	/*
1643 	 * port_map is determined from PORTS_IMPL PCI register which is
1644 	 * implemented as write or write-once register.  If the register
1645 	 * isn't programmed, ahci automatically generates it from number
1646 	 * of ports, which is good enough for PCS programming. It is
1647 	 * otherwise expected that platform firmware enables the ports
1648 	 * before the OS boots.
1649 	 */
1650 	pci_read_config_word(pdev, PCS_6, &tmp16);
1651 	if ((tmp16 & hpriv->port_map) != hpriv->port_map) {
1652 		tmp16 |= hpriv->port_map;
1653 		pci_write_config_word(pdev, PCS_6, tmp16);
1654 	}
1655 }
1656 
1657 static ssize_t remapped_nvme_show(struct device *dev,
1658 				  struct device_attribute *attr,
1659 				  char *buf)
1660 {
1661 	struct ata_host *host = dev_get_drvdata(dev);
1662 	struct ahci_host_priv *hpriv = host->private_data;
1663 
1664 	return sysfs_emit(buf, "%u\n", hpriv->remapped_nvme);
1665 }
1666 
1667 static DEVICE_ATTR_RO(remapped_nvme);
1668 
1669 static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1670 {
1671 	unsigned int board_id = ent->driver_data;
1672 	struct ata_port_info pi = ahci_port_info[board_id];
1673 	const struct ata_port_info *ppi[] = { &pi, NULL };
1674 	struct device *dev = &pdev->dev;
1675 	struct ahci_host_priv *hpriv;
1676 	struct ata_host *host;
1677 	int n_ports, i, rc;
1678 	int ahci_pci_bar = AHCI_PCI_BAR_STANDARD;
1679 
1680 	WARN_ON((int)ATA_MAX_QUEUE > AHCI_MAX_CMDS);
1681 
1682 	ata_print_version_once(&pdev->dev, DRV_VERSION);
1683 
1684 	/* The AHCI driver can only drive the SATA ports, the PATA driver
1685 	   can drive them all so if both drivers are selected make sure
1686 	   AHCI stays out of the way */
1687 	if (pdev->vendor == PCI_VENDOR_ID_MARVELL && !marvell_enable)
1688 		return -ENODEV;
1689 
1690 	/* Apple BIOS on MCP89 prevents us using AHCI */
1691 	if (is_mcp89_apple(pdev))
1692 		ahci_mcp89_apple_enable(pdev);
1693 
1694 	/* Promise's PDC42819 is a SAS/SATA controller that has an AHCI mode.
1695 	 * At the moment, we can only use the AHCI mode. Let the users know
1696 	 * that for SAS drives they're out of luck.
1697 	 */
1698 	if (pdev->vendor == PCI_VENDOR_ID_PROMISE)
1699 		dev_info(&pdev->dev,
1700 			 "PDC42819 can only drive SATA devices with this driver\n");
1701 
1702 	/* Some devices use non-standard BARs */
1703 	if (pdev->vendor == PCI_VENDOR_ID_STMICRO && pdev->device == 0xCC06)
1704 		ahci_pci_bar = AHCI_PCI_BAR_STA2X11;
1705 	else if (pdev->vendor == 0x1c44 && pdev->device == 0x8000)
1706 		ahci_pci_bar = AHCI_PCI_BAR_ENMOTUS;
1707 	else if (pdev->vendor == PCI_VENDOR_ID_CAVIUM) {
1708 		if (pdev->device == 0xa01c)
1709 			ahci_pci_bar = AHCI_PCI_BAR_CAVIUM;
1710 		if (pdev->device == 0xa084)
1711 			ahci_pci_bar = AHCI_PCI_BAR_CAVIUM_GEN5;
1712 	} else if (pdev->vendor == PCI_VENDOR_ID_LOONGSON) {
1713 		if (pdev->device == 0x7a08)
1714 			ahci_pci_bar = AHCI_PCI_BAR_LOONGSON;
1715 	}
1716 
1717 	/* acquire resources */
1718 	rc = pcim_enable_device(pdev);
1719 	if (rc)
1720 		return rc;
1721 
1722 	if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
1723 	    (pdev->device == 0x2652 || pdev->device == 0x2653)) {
1724 		u8 map;
1725 
1726 		/* ICH6s share the same PCI ID for both piix and ahci
1727 		 * modes.  Enabling ahci mode while MAP indicates
1728 		 * combined mode is a bad idea.  Yield to ata_piix.
1729 		 */
1730 		pci_read_config_byte(pdev, ICH_MAP, &map);
1731 		if (map & 0x3) {
1732 			dev_info(&pdev->dev,
1733 				 "controller is in combined mode, can't enable AHCI mode\n");
1734 			return -ENODEV;
1735 		}
1736 	}
1737 
1738 	/* AHCI controllers often implement SFF compatible interface.
1739 	 * Grab all PCI BARs just in case.
1740 	 */
1741 	rc = pcim_iomap_regions_request_all(pdev, 1 << ahci_pci_bar, DRV_NAME);
1742 	if (rc == -EBUSY)
1743 		pcim_pin_device(pdev);
1744 	if (rc)
1745 		return rc;
1746 
1747 	hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
1748 	if (!hpriv)
1749 		return -ENOMEM;
1750 	hpriv->flags |= (unsigned long)pi.private_data;
1751 
1752 	/* MCP65 revision A1 and A2 can't do MSI */
1753 	if (board_id == board_ahci_mcp65 &&
1754 	    (pdev->revision == 0xa1 || pdev->revision == 0xa2))
1755 		hpriv->flags |= AHCI_HFLAG_NO_MSI;
1756 
1757 	/* SB800 does NOT need the workaround to ignore SERR_INTERNAL */
1758 	if (board_id == board_ahci_sb700 && pdev->revision >= 0x40)
1759 		hpriv->flags &= ~AHCI_HFLAG_IGN_SERR_INTERNAL;
1760 
1761 	/* only some SB600s can do 64bit DMA */
1762 	if (ahci_sb600_enable_64bit(pdev))
1763 		hpriv->flags &= ~AHCI_HFLAG_32BIT_ONLY;
1764 
1765 	hpriv->mmio = pcim_iomap_table(pdev)[ahci_pci_bar];
1766 
1767 	/* detect remapped nvme devices */
1768 	ahci_remap_check(pdev, ahci_pci_bar, hpriv);
1769 
1770 	sysfs_add_file_to_group(&pdev->dev.kobj,
1771 				&dev_attr_remapped_nvme.attr,
1772 				NULL);
1773 
1774 	/* must set flag prior to save config in order to take effect */
1775 	if (ahci_broken_devslp(pdev))
1776 		hpriv->flags |= AHCI_HFLAG_NO_DEVSLP;
1777 
1778 #ifdef CONFIG_ARM64
1779 	if (pdev->vendor == PCI_VENDOR_ID_HUAWEI &&
1780 	    pdev->device == 0xa235 &&
1781 	    pdev->revision < 0x30)
1782 		hpriv->flags |= AHCI_HFLAG_NO_SXS;
1783 
1784 	if (pdev->vendor == 0x177d && pdev->device == 0xa01c)
1785 		hpriv->irq_handler = ahci_thunderx_irq_handler;
1786 #endif
1787 
1788 	/* save initial config */
1789 	ahci_pci_save_initial_config(pdev, hpriv);
1790 
1791 	/*
1792 	 * If platform firmware failed to enable ports, try to enable
1793 	 * them here.
1794 	 */
1795 	ahci_intel_pcs_quirk(pdev, hpriv);
1796 
1797 	/* prepare host */
1798 	if (hpriv->cap & HOST_CAP_NCQ) {
1799 		pi.flags |= ATA_FLAG_NCQ;
1800 		/*
1801 		 * Auto-activate optimization is supposed to be
1802 		 * supported on all AHCI controllers indicating NCQ
1803 		 * capability, but it seems to be broken on some
1804 		 * chipsets including NVIDIAs.
1805 		 */
1806 		if (!(hpriv->flags & AHCI_HFLAG_NO_FPDMA_AA))
1807 			pi.flags |= ATA_FLAG_FPDMA_AA;
1808 
1809 		/*
1810 		 * All AHCI controllers should be forward-compatible
1811 		 * with the new auxiliary field. This code should be
1812 		 * conditionalized if any buggy AHCI controllers are
1813 		 * encountered.
1814 		 */
1815 		pi.flags |= ATA_FLAG_FPDMA_AUX;
1816 	}
1817 
1818 	if (hpriv->cap & HOST_CAP_PMP)
1819 		pi.flags |= ATA_FLAG_PMP;
1820 
1821 	ahci_set_em_messages(hpriv, &pi);
1822 
1823 	if (ahci_broken_system_poweroff(pdev)) {
1824 		pi.flags |= ATA_FLAG_NO_POWEROFF_SPINDOWN;
1825 		dev_info(&pdev->dev,
1826 			"quirky BIOS, skipping spindown on poweroff\n");
1827 	}
1828 
1829 	if (ahci_broken_lpm(pdev)) {
1830 		pi.flags |= ATA_FLAG_NO_LPM;
1831 		dev_warn(&pdev->dev,
1832 			 "BIOS update required for Link Power Management support\n");
1833 	}
1834 
1835 	if (ahci_broken_suspend(pdev)) {
1836 		hpriv->flags |= AHCI_HFLAG_NO_SUSPEND;
1837 		dev_warn(&pdev->dev,
1838 			 "BIOS update required for suspend/resume\n");
1839 	}
1840 
1841 	if (ahci_broken_online(pdev)) {
1842 		hpriv->flags |= AHCI_HFLAG_SRST_TOUT_IS_OFFLINE;
1843 		dev_info(&pdev->dev,
1844 			 "online status unreliable, applying workaround\n");
1845 	}
1846 
1847 
1848 	/* Acer SA5-271 workaround modifies private_data */
1849 	acer_sa5_271_workaround(hpriv, pdev);
1850 
1851 	/* CAP.NP sometimes indicate the index of the last enabled
1852 	 * port, at other times, that of the last possible port, so
1853 	 * determining the maximum port number requires looking at
1854 	 * both CAP.NP and port_map.
1855 	 */
1856 	n_ports = max(ahci_nr_ports(hpriv->cap), fls(hpriv->port_map));
1857 
1858 	host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
1859 	if (!host)
1860 		return -ENOMEM;
1861 	host->private_data = hpriv;
1862 
1863 	if (ahci_init_msi(pdev, n_ports, hpriv) < 0) {
1864 		/* legacy intx interrupts */
1865 		pci_intx(pdev, 1);
1866 	}
1867 	hpriv->irq = pci_irq_vector(pdev, 0);
1868 
1869 	if (!(hpriv->cap & HOST_CAP_SSS) || ahci_ignore_sss)
1870 		host->flags |= ATA_HOST_PARALLEL_SCAN;
1871 	else
1872 		dev_info(&pdev->dev, "SSS flag set, parallel bus scan disabled\n");
1873 
1874 	if (pi.flags & ATA_FLAG_EM)
1875 		ahci_reset_em(host);
1876 
1877 	for (i = 0; i < host->n_ports; i++) {
1878 		struct ata_port *ap = host->ports[i];
1879 
1880 		ata_port_pbar_desc(ap, ahci_pci_bar, -1, "abar");
1881 		ata_port_pbar_desc(ap, ahci_pci_bar,
1882 				   0x100 + ap->port_no * 0x80, "port");
1883 
1884 		/* set enclosure management message type */
1885 		if (ap->flags & ATA_FLAG_EM)
1886 			ap->em_message_type = hpriv->em_msg_type;
1887 
1888 		ahci_update_initial_lpm_policy(ap, hpriv);
1889 
1890 		/* disabled/not-implemented port */
1891 		if (!(hpriv->port_map & (1 << i)))
1892 			ap->ops = &ata_dummy_port_ops;
1893 	}
1894 
1895 	/* apply workaround for ASUS P5W DH Deluxe mainboard */
1896 	ahci_p5wdh_workaround(host);
1897 
1898 	/* apply gtf filter quirk */
1899 	ahci_gtf_filter_workaround(host);
1900 
1901 	/* initialize adapter */
1902 	rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
1903 	if (rc)
1904 		return rc;
1905 
1906 	rc = ahci_reset_controller(host);
1907 	if (rc)
1908 		return rc;
1909 
1910 	ahci_pci_init_controller(host);
1911 	ahci_pci_print_info(host);
1912 
1913 	pci_set_master(pdev);
1914 
1915 	rc = ahci_host_activate(host, &ahci_sht);
1916 	if (rc)
1917 		return rc;
1918 
1919 	pm_runtime_put_noidle(&pdev->dev);
1920 	return 0;
1921 }
1922 
1923 static void ahci_shutdown_one(struct pci_dev *pdev)
1924 {
1925 	ata_pci_shutdown_one(pdev);
1926 }
1927 
1928 static void ahci_remove_one(struct pci_dev *pdev)
1929 {
1930 	sysfs_remove_file_from_group(&pdev->dev.kobj,
1931 				     &dev_attr_remapped_nvme.attr,
1932 				     NULL);
1933 	pm_runtime_get_noresume(&pdev->dev);
1934 	ata_pci_remove_one(pdev);
1935 }
1936 
1937 module_pci_driver(ahci_pci_driver);
1938 
1939 MODULE_AUTHOR("Jeff Garzik");
1940 MODULE_DESCRIPTION("AHCI SATA low-level driver");
1941 MODULE_LICENSE("GPL");
1942 MODULE_DEVICE_TABLE(pci, ahci_pci_tbl);
1943 MODULE_VERSION(DRV_VERSION);
1944