1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Driver for the Micron P320 SSD
4  *   Copyright (C) 2011 Micron Technology, Inc.
5  *
6  * Portions of this code were derived from works subjected to the
7  * following copyright:
8  *    Copyright (C) 2009 Integrated Device Technology, Inc.
9  */
10 
11 #include <linux/pci.h>
12 #include <linux/interrupt.h>
13 #include <linux/ata.h>
14 #include <linux/delay.h>
15 #include <linux/hdreg.h>
16 #include <linux/uaccess.h>
17 #include <linux/random.h>
18 #include <linux/smp.h>
19 #include <linux/compat.h>
20 #include <linux/fs.h>
21 #include <linux/module.h>
22 #include <linux/blkdev.h>
23 #include <linux/blk-mq.h>
24 #include <linux/bio.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/idr.h>
27 #include <linux/kthread.h>
28 #include <../drivers/ata/ahci.h>
29 #include <linux/export.h>
30 #include <linux/debugfs.h>
31 #include <linux/prefetch.h>
32 #include <linux/numa.h>
33 #include "mtip32xx.h"
34 
35 #define HW_CMD_SLOT_SZ		(MTIP_MAX_COMMAND_SLOTS * 32)
36 
37 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */
38 #define AHCI_RX_FIS_SZ          0x100
39 #define AHCI_RX_FIS_OFFSET      0x0
40 #define AHCI_IDFY_SZ            ATA_SECT_SIZE
41 #define AHCI_IDFY_OFFSET        0x400
42 #define AHCI_SECTBUF_SZ         ATA_SECT_SIZE
43 #define AHCI_SECTBUF_OFFSET     0x800
44 #define AHCI_SMARTBUF_SZ        ATA_SECT_SIZE
45 #define AHCI_SMARTBUF_OFFSET    0xC00
46 /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */
47 #define BLOCK_DMA_ALLOC_SZ      4096
48 
49 /* DMA region containing command table (should be 8192 bytes) */
50 #define AHCI_CMD_SLOT_SZ        sizeof(struct mtip_cmd_hdr)
51 #define AHCI_CMD_TBL_SZ         (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ)
52 #define AHCI_CMD_TBL_OFFSET     0x0
53 
54 /* DMA region per command (contains header and SGL) */
55 #define AHCI_CMD_TBL_HDR_SZ     0x80
56 #define AHCI_CMD_TBL_HDR_OFFSET 0x0
57 #define AHCI_CMD_TBL_SGL_SZ     (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg))
58 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ
59 #define CMD_DMA_ALLOC_SZ        (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ)
60 
61 
62 #define HOST_CAP_NZDMA		(1 << 19)
63 #define HOST_HSORG		0xFC
64 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24)
65 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16)
66 #define HSORG_HWREV		0xFF00
67 #define HSORG_STYLE		0x8
68 #define HSORG_SLOTGROUPS	0x7
69 
70 #define PORT_COMMAND_ISSUE	0x38
71 #define PORT_SDBV		0x7C
72 
73 #define PORT_OFFSET		0x100
74 #define PORT_MEM_SIZE		0x80
75 
76 #define PORT_IRQ_ERR \
77 	(PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \
78 	 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \
79 	 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \
80 	 PORT_IRQ_OVERFLOW)
81 #define PORT_IRQ_LEGACY \
82 	(PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
83 #define PORT_IRQ_HANDLED \
84 	(PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \
85 	 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \
86 	 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)
87 #define DEF_PORT_IRQ \
88 	(PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS)
89 
90 /* product numbers */
91 #define MTIP_PRODUCT_UNKNOWN	0x00
92 #define MTIP_PRODUCT_ASICFPGA	0x11
93 
94 /* Device instance number, incremented each time a device is probed. */
95 static int instance;
96 
97 /*
98  * Global variable used to hold the major block device number
99  * allocated in mtip_init().
100  */
101 static int mtip_major;
102 static struct dentry *dfs_parent;
103 
104 static u32 cpu_use[NR_CPUS];
105 
106 static DEFINE_IDA(rssd_index_ida);
107 
108 static int mtip_block_initialize(struct driver_data *dd);
109 
110 #ifdef CONFIG_COMPAT
111 struct mtip_compat_ide_task_request_s {
112 	__u8		io_ports[8];
113 	__u8		hob_ports[8];
114 	ide_reg_valid_t	out_flags;
115 	ide_reg_valid_t	in_flags;
116 	int		data_phase;
117 	int		req_cmd;
118 	compat_ulong_t	out_size;
119 	compat_ulong_t	in_size;
120 };
121 #endif
122 
123 /*
124  * This function check_for_surprise_removal is called
125  * while card is removed from the system and it will
126  * read the vendor id from the configuration space
127  *
128  * @pdev Pointer to the pci_dev structure.
129  *
130  * return value
131  *	 true if device removed, else false
132  */
133 static bool mtip_check_surprise_removal(struct driver_data *dd)
134 {
135 	u16 vendor_id = 0;
136 
137 	if (dd->sr)
138 		return true;
139 
140        /* Read the vendorID from the configuration space */
141 	pci_read_config_word(dd->pdev, 0x00, &vendor_id);
142 	if (vendor_id == 0xFFFF) {
143 		dd->sr = true;
144 		if (dd->disk)
145 			blk_mark_disk_dead(dd->disk);
146 		return true; /* device removed */
147 	}
148 
149 	return false; /* device present */
150 }
151 
152 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd,
153 					  unsigned int tag)
154 {
155 	return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(dd->tags.tags[0], tag));
156 }
157 
158 /*
159  * Reset the HBA (without sleeping)
160  *
161  * @dd Pointer to the driver data structure.
162  *
163  * return value
164  *	0	The reset was successful.
165  *	-1	The HBA Reset bit did not clear.
166  */
167 static int mtip_hba_reset(struct driver_data *dd)
168 {
169 	unsigned long timeout;
170 
171 	/* Set the reset bit */
172 	writel(HOST_RESET, dd->mmio + HOST_CTL);
173 
174 	/* Flush */
175 	readl(dd->mmio + HOST_CTL);
176 
177 	/*
178 	 * Spin for up to 10 seconds waiting for reset acknowledgement. Spec
179 	 * is 1 sec but in LUN failure conditions, up to 10 secs are required
180 	 */
181 	timeout = jiffies + msecs_to_jiffies(10000);
182 	do {
183 		mdelay(10);
184 		if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
185 			return -1;
186 
187 	} while ((readl(dd->mmio + HOST_CTL) & HOST_RESET)
188 		 && time_before(jiffies, timeout));
189 
190 	if (readl(dd->mmio + HOST_CTL) & HOST_RESET)
191 		return -1;
192 
193 	return 0;
194 }
195 
196 /*
197  * Issue a command to the hardware.
198  *
199  * Set the appropriate bit in the s_active and Command Issue hardware
200  * registers, causing hardware command processing to begin.
201  *
202  * @port Pointer to the port structure.
203  * @tag  The tag of the command to be issued.
204  *
205  * return value
206  *      None
207  */
208 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag)
209 {
210 	int group = tag >> 5;
211 
212 	/* guard SACT and CI registers */
213 	spin_lock(&port->cmd_issue_lock[group]);
214 	writel((1 << MTIP_TAG_BIT(tag)),
215 			port->s_active[MTIP_TAG_INDEX(tag)]);
216 	writel((1 << MTIP_TAG_BIT(tag)),
217 			port->cmd_issue[MTIP_TAG_INDEX(tag)]);
218 	spin_unlock(&port->cmd_issue_lock[group]);
219 }
220 
221 /*
222  * Enable/disable the reception of FIS
223  *
224  * @port   Pointer to the port data structure
225  * @enable 1 to enable, 0 to disable
226  *
227  * return value
228  *	Previous state: 1 enabled, 0 disabled
229  */
230 static int mtip_enable_fis(struct mtip_port *port, int enable)
231 {
232 	u32 tmp;
233 
234 	/* enable FIS reception */
235 	tmp = readl(port->mmio + PORT_CMD);
236 	if (enable)
237 		writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
238 	else
239 		writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
240 
241 	/* Flush */
242 	readl(port->mmio + PORT_CMD);
243 
244 	return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX));
245 }
246 
247 /*
248  * Enable/disable the DMA engine
249  *
250  * @port   Pointer to the port data structure
251  * @enable 1 to enable, 0 to disable
252  *
253  * return value
254  *	Previous state: 1 enabled, 0 disabled.
255  */
256 static int mtip_enable_engine(struct mtip_port *port, int enable)
257 {
258 	u32 tmp;
259 
260 	/* enable FIS reception */
261 	tmp = readl(port->mmio + PORT_CMD);
262 	if (enable)
263 		writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD);
264 	else
265 		writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD);
266 
267 	readl(port->mmio + PORT_CMD);
268 	return (((tmp & PORT_CMD_START) == PORT_CMD_START));
269 }
270 
271 /*
272  * Enables the port DMA engine and FIS reception.
273  *
274  * return value
275  *	None
276  */
277 static inline void mtip_start_port(struct mtip_port *port)
278 {
279 	/* Enable FIS reception */
280 	mtip_enable_fis(port, 1);
281 
282 	/* Enable the DMA engine */
283 	mtip_enable_engine(port, 1);
284 }
285 
286 /*
287  * Deinitialize a port by disabling port interrupts, the DMA engine,
288  * and FIS reception.
289  *
290  * @port Pointer to the port structure
291  *
292  * return value
293  *	None
294  */
295 static inline void mtip_deinit_port(struct mtip_port *port)
296 {
297 	/* Disable interrupts on this port */
298 	writel(0, port->mmio + PORT_IRQ_MASK);
299 
300 	/* Disable the DMA engine */
301 	mtip_enable_engine(port, 0);
302 
303 	/* Disable FIS reception */
304 	mtip_enable_fis(port, 0);
305 }
306 
307 /*
308  * Initialize a port.
309  *
310  * This function deinitializes the port by calling mtip_deinit_port() and
311  * then initializes it by setting the command header and RX FIS addresses,
312  * clearing the SError register and any pending port interrupts before
313  * re-enabling the default set of port interrupts.
314  *
315  * @port Pointer to the port structure.
316  *
317  * return value
318  *	None
319  */
320 static void mtip_init_port(struct mtip_port *port)
321 {
322 	int i;
323 	mtip_deinit_port(port);
324 
325 	/* Program the command list base and FIS base addresses */
326 	if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) {
327 		writel((port->command_list_dma >> 16) >> 16,
328 			 port->mmio + PORT_LST_ADDR_HI);
329 		writel((port->rxfis_dma >> 16) >> 16,
330 			 port->mmio + PORT_FIS_ADDR_HI);
331 		set_bit(MTIP_PF_HOST_CAP_64, &port->flags);
332 	}
333 
334 	writel(port->command_list_dma & 0xFFFFFFFF,
335 			port->mmio + PORT_LST_ADDR);
336 	writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR);
337 
338 	/* Clear SError */
339 	writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
340 
341 	/* reset the completed registers.*/
342 	for (i = 0; i < port->dd->slot_groups; i++)
343 		writel(0xFFFFFFFF, port->completed[i]);
344 
345 	/* Clear any pending interrupts for this port */
346 	writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT);
347 
348 	/* Clear any pending interrupts on the HBA. */
349 	writel(readl(port->dd->mmio + HOST_IRQ_STAT),
350 					port->dd->mmio + HOST_IRQ_STAT);
351 
352 	/* Enable port interrupts */
353 	writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK);
354 }
355 
356 /*
357  * Restart a port
358  *
359  * @port Pointer to the port data structure.
360  *
361  * return value
362  *	None
363  */
364 static void mtip_restart_port(struct mtip_port *port)
365 {
366 	unsigned long timeout;
367 
368 	/* Disable the DMA engine */
369 	mtip_enable_engine(port, 0);
370 
371 	/* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */
372 	timeout = jiffies + msecs_to_jiffies(500);
373 	while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON)
374 		 && time_before(jiffies, timeout))
375 		;
376 
377 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
378 		return;
379 
380 	/*
381 	 * Chip quirk: escalate to hba reset if
382 	 * PxCMD.CR not clear after 500 ms
383 	 */
384 	if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) {
385 		dev_warn(&port->dd->pdev->dev,
386 			"PxCMD.CR not clear, escalating reset\n");
387 
388 		if (mtip_hba_reset(port->dd))
389 			dev_err(&port->dd->pdev->dev,
390 				"HBA reset escalation failed.\n");
391 
392 		/* 30 ms delay before com reset to quiesce chip */
393 		mdelay(30);
394 	}
395 
396 	dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n");
397 
398 	/* Set PxSCTL.DET */
399 	writel(readl(port->mmio + PORT_SCR_CTL) |
400 			 1, port->mmio + PORT_SCR_CTL);
401 	readl(port->mmio + PORT_SCR_CTL);
402 
403 	/* Wait 1 ms to quiesce chip function */
404 	timeout = jiffies + msecs_to_jiffies(1);
405 	while (time_before(jiffies, timeout))
406 		;
407 
408 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
409 		return;
410 
411 	/* Clear PxSCTL.DET */
412 	writel(readl(port->mmio + PORT_SCR_CTL) & ~1,
413 			 port->mmio + PORT_SCR_CTL);
414 	readl(port->mmio + PORT_SCR_CTL);
415 
416 	/* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */
417 	timeout = jiffies + msecs_to_jiffies(500);
418 	while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
419 			 && time_before(jiffies, timeout))
420 		;
421 
422 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
423 		return;
424 
425 	if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
426 		dev_warn(&port->dd->pdev->dev,
427 			"COM reset failed\n");
428 
429 	mtip_init_port(port);
430 	mtip_start_port(port);
431 
432 }
433 
434 static int mtip_device_reset(struct driver_data *dd)
435 {
436 	int rv = 0;
437 
438 	if (mtip_check_surprise_removal(dd))
439 		return 0;
440 
441 	if (mtip_hba_reset(dd) < 0)
442 		rv = -EFAULT;
443 
444 	mdelay(1);
445 	mtip_init_port(dd->port);
446 	mtip_start_port(dd->port);
447 
448 	/* Enable interrupts on the HBA. */
449 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
450 					dd->mmio + HOST_CTL);
451 	return rv;
452 }
453 
454 /*
455  * Helper function for tag logging
456  */
457 static void print_tags(struct driver_data *dd,
458 			char *msg,
459 			unsigned long *tagbits,
460 			int cnt)
461 {
462 	unsigned char tagmap[128];
463 	int group, tagmap_len = 0;
464 
465 	memset(tagmap, 0, sizeof(tagmap));
466 	for (group = SLOTBITS_IN_LONGS; group > 0; group--)
467 		tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ",
468 						tagbits[group-1]);
469 	dev_warn(&dd->pdev->dev,
470 			"%d command(s) %s: tagmap [%s]", cnt, msg, tagmap);
471 }
472 
473 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
474 				dma_addr_t buffer_dma, unsigned int sectors);
475 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
476 						struct smart_attr *attrib);
477 
478 static void mtip_complete_command(struct mtip_cmd *cmd, blk_status_t status)
479 {
480 	struct request *req = blk_mq_rq_from_pdu(cmd);
481 
482 	cmd->status = status;
483 	if (likely(!blk_should_fake_timeout(req->q)))
484 		blk_mq_complete_request(req);
485 }
486 
487 /*
488  * Handle an error.
489  *
490  * @dd Pointer to the DRIVER_DATA structure.
491  *
492  * return value
493  *	None
494  */
495 static void mtip_handle_tfe(struct driver_data *dd)
496 {
497 	int group, tag, bit, reissue, rv;
498 	struct mtip_port *port;
499 	struct mtip_cmd  *cmd;
500 	u32 completed;
501 	struct host_to_dev_fis *fis;
502 	unsigned long tagaccum[SLOTBITS_IN_LONGS];
503 	unsigned int cmd_cnt = 0;
504 	unsigned char *buf;
505 	char *fail_reason = NULL;
506 	int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0;
507 
508 	dev_warn(&dd->pdev->dev, "Taskfile error\n");
509 
510 	port = dd->port;
511 
512 	if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) {
513 		cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
514 		dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n");
515 		mtip_complete_command(cmd, BLK_STS_IOERR);
516 		return;
517 	}
518 
519 	/* clear the tag accumulator */
520 	memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
521 
522 	/* Loop through all the groups */
523 	for (group = 0; group < dd->slot_groups; group++) {
524 		completed = readl(port->completed[group]);
525 
526 		dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed);
527 
528 		/* clear completed status register in the hardware.*/
529 		writel(completed, port->completed[group]);
530 
531 		/* Process successfully completed commands */
532 		for (bit = 0; bit < 32 && completed; bit++) {
533 			if (!(completed & (1<<bit)))
534 				continue;
535 			tag = (group << 5) + bit;
536 
537 			/* Skip the internal command slot */
538 			if (tag == MTIP_TAG_INTERNAL)
539 				continue;
540 
541 			cmd = mtip_cmd_from_tag(dd, tag);
542 			mtip_complete_command(cmd, 0);
543 			set_bit(tag, tagaccum);
544 			cmd_cnt++;
545 		}
546 	}
547 
548 	print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt);
549 
550 	/* Restart the port */
551 	mdelay(20);
552 	mtip_restart_port(port);
553 
554 	/* Trying to determine the cause of the error */
555 	rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
556 				dd->port->log_buf,
557 				dd->port->log_buf_dma, 1);
558 	if (rv) {
559 		dev_warn(&dd->pdev->dev,
560 			"Error in READ LOG EXT (10h) command\n");
561 		/* non-critical error, don't fail the load */
562 	} else {
563 		buf = (unsigned char *)dd->port->log_buf;
564 		if (buf[259] & 0x1) {
565 			dev_info(&dd->pdev->dev,
566 				"Write protect bit is set.\n");
567 			set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
568 			fail_all_ncq_write = 1;
569 			fail_reason = "write protect";
570 		}
571 		if (buf[288] == 0xF7) {
572 			dev_info(&dd->pdev->dev,
573 				"Exceeded Tmax, drive in thermal shutdown.\n");
574 			set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
575 			fail_all_ncq_cmds = 1;
576 			fail_reason = "thermal shutdown";
577 		}
578 		if (buf[288] == 0xBF) {
579 			set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
580 			dev_info(&dd->pdev->dev,
581 				"Drive indicates rebuild has failed. Secure erase required.\n");
582 			fail_all_ncq_cmds = 1;
583 			fail_reason = "rebuild failed";
584 		}
585 	}
586 
587 	/* clear the tag accumulator */
588 	memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
589 
590 	/* Loop through all the groups */
591 	for (group = 0; group < dd->slot_groups; group++) {
592 		for (bit = 0; bit < 32; bit++) {
593 			reissue = 1;
594 			tag = (group << 5) + bit;
595 			cmd = mtip_cmd_from_tag(dd, tag);
596 
597 			fis = (struct host_to_dev_fis *)cmd->command;
598 
599 			/* Should re-issue? */
600 			if (tag == MTIP_TAG_INTERNAL ||
601 			    fis->command == ATA_CMD_SET_FEATURES)
602 				reissue = 0;
603 			else {
604 				if (fail_all_ncq_cmds ||
605 					(fail_all_ncq_write &&
606 					fis->command == ATA_CMD_FPDMA_WRITE)) {
607 					dev_warn(&dd->pdev->dev,
608 					"  Fail: %s w/tag %d [%s].\n",
609 					fis->command == ATA_CMD_FPDMA_WRITE ?
610 						"write" : "read",
611 					tag,
612 					fail_reason != NULL ?
613 						fail_reason : "unknown");
614 					mtip_complete_command(cmd, BLK_STS_MEDIUM);
615 					continue;
616 				}
617 			}
618 
619 			/*
620 			 * First check if this command has
621 			 *  exceeded its retries.
622 			 */
623 			if (reissue && (cmd->retries-- > 0)) {
624 
625 				set_bit(tag, tagaccum);
626 
627 				/* Re-issue the command. */
628 				mtip_issue_ncq_command(port, tag);
629 
630 				continue;
631 			}
632 
633 			/* Retire a command that will not be reissued */
634 			dev_warn(&port->dd->pdev->dev,
635 				"retiring tag %d\n", tag);
636 
637 			mtip_complete_command(cmd, BLK_STS_IOERR);
638 		}
639 	}
640 	print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt);
641 }
642 
643 /*
644  * Handle a set device bits interrupt
645  */
646 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group,
647 							u32 completed)
648 {
649 	struct driver_data *dd = port->dd;
650 	int tag, bit;
651 	struct mtip_cmd *command;
652 
653 	if (!completed) {
654 		WARN_ON_ONCE(!completed);
655 		return;
656 	}
657 	/* clear completed status register in the hardware.*/
658 	writel(completed, port->completed[group]);
659 
660 	/* Process completed commands. */
661 	for (bit = 0; (bit < 32) && completed; bit++) {
662 		if (completed & 0x01) {
663 			tag = (group << 5) | bit;
664 
665 			/* skip internal command slot. */
666 			if (unlikely(tag == MTIP_TAG_INTERNAL))
667 				continue;
668 
669 			command = mtip_cmd_from_tag(dd, tag);
670 			mtip_complete_command(command, 0);
671 		}
672 		completed >>= 1;
673 	}
674 
675 	/* If last, re-enable interrupts */
676 	if (atomic_dec_return(&dd->irq_workers_active) == 0)
677 		writel(0xffffffff, dd->mmio + HOST_IRQ_STAT);
678 }
679 
680 /*
681  * Process legacy pio and d2h interrupts
682  */
683 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat)
684 {
685 	struct mtip_port *port = dd->port;
686 	struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
687 
688 	if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) {
689 		int group = MTIP_TAG_INDEX(MTIP_TAG_INTERNAL);
690 		int status = readl(port->cmd_issue[group]);
691 
692 		if (!(status & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))))
693 			mtip_complete_command(cmd, 0);
694 	}
695 }
696 
697 /*
698  * Demux and handle errors
699  */
700 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat)
701 {
702 	if (unlikely(port_stat & PORT_IRQ_CONNECT)) {
703 		dev_warn(&dd->pdev->dev,
704 			"Clearing PxSERR.DIAG.x\n");
705 		writel((1 << 26), dd->port->mmio + PORT_SCR_ERR);
706 	}
707 
708 	if (unlikely(port_stat & PORT_IRQ_PHYRDY)) {
709 		dev_warn(&dd->pdev->dev,
710 			"Clearing PxSERR.DIAG.n\n");
711 		writel((1 << 16), dd->port->mmio + PORT_SCR_ERR);
712 	}
713 
714 	if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) {
715 		dev_warn(&dd->pdev->dev,
716 			"Port stat errors %x unhandled\n",
717 			(port_stat & ~PORT_IRQ_HANDLED));
718 		if (mtip_check_surprise_removal(dd))
719 			return;
720 	}
721 	if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) {
722 		set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags);
723 		wake_up_interruptible(&dd->port->svc_wait);
724 	}
725 }
726 
727 static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
728 {
729 	struct driver_data *dd = (struct driver_data *) data;
730 	struct mtip_port *port = dd->port;
731 	u32 hba_stat, port_stat;
732 	int rv = IRQ_NONE;
733 	int do_irq_enable = 1, i, workers;
734 	struct mtip_work *twork;
735 
736 	hba_stat = readl(dd->mmio + HOST_IRQ_STAT);
737 	if (hba_stat) {
738 		rv = IRQ_HANDLED;
739 
740 		/* Acknowledge the interrupt status on the port.*/
741 		port_stat = readl(port->mmio + PORT_IRQ_STAT);
742 		if (unlikely(port_stat == 0xFFFFFFFF)) {
743 			mtip_check_surprise_removal(dd);
744 			return IRQ_HANDLED;
745 		}
746 		writel(port_stat, port->mmio + PORT_IRQ_STAT);
747 
748 		/* Demux port status */
749 		if (likely(port_stat & PORT_IRQ_SDB_FIS)) {
750 			do_irq_enable = 0;
751 			WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0);
752 
753 			/* Start at 1: group zero is always local? */
754 			for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS;
755 									i++) {
756 				twork = &dd->work[i];
757 				twork->completed = readl(port->completed[i]);
758 				if (twork->completed)
759 					workers++;
760 			}
761 
762 			atomic_set(&dd->irq_workers_active, workers);
763 			if (workers) {
764 				for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) {
765 					twork = &dd->work[i];
766 					if (twork->completed)
767 						queue_work_on(
768 							twork->cpu_binding,
769 							dd->isr_workq,
770 							&twork->work);
771 				}
772 
773 				if (likely(dd->work[0].completed))
774 					mtip_workq_sdbfx(port, 0,
775 							dd->work[0].completed);
776 
777 			} else {
778 				/*
779 				 * Chip quirk: SDB interrupt but nothing
780 				 * to complete
781 				 */
782 				do_irq_enable = 1;
783 			}
784 		}
785 
786 		if (unlikely(port_stat & PORT_IRQ_ERR)) {
787 			if (unlikely(mtip_check_surprise_removal(dd))) {
788 				/* don't proceed further */
789 				return IRQ_HANDLED;
790 			}
791 			if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
792 							&dd->dd_flag))
793 				return rv;
794 
795 			mtip_process_errors(dd, port_stat & PORT_IRQ_ERR);
796 		}
797 
798 		if (unlikely(port_stat & PORT_IRQ_LEGACY))
799 			mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY);
800 	}
801 
802 	/* acknowledge interrupt */
803 	if (unlikely(do_irq_enable))
804 		writel(hba_stat, dd->mmio + HOST_IRQ_STAT);
805 
806 	return rv;
807 }
808 
809 /*
810  * HBA interrupt subroutine.
811  *
812  * @irq		IRQ number.
813  * @instance	Pointer to the driver data structure.
814  *
815  * return value
816  *	IRQ_HANDLED	A HBA interrupt was pending and handled.
817  *	IRQ_NONE	This interrupt was not for the HBA.
818  */
819 static irqreturn_t mtip_irq_handler(int irq, void *instance)
820 {
821 	struct driver_data *dd = instance;
822 
823 	return mtip_handle_irq(dd);
824 }
825 
826 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag)
827 {
828 	writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]);
829 }
830 
831 static bool mtip_pause_ncq(struct mtip_port *port,
832 				struct host_to_dev_fis *fis)
833 {
834 	unsigned long task_file_data;
835 
836 	task_file_data = readl(port->mmio+PORT_TFDATA);
837 	if ((task_file_data & 1))
838 		return false;
839 
840 	if (fis->command == ATA_CMD_SEC_ERASE_PREP) {
841 		port->ic_pause_timer = jiffies;
842 		return true;
843 	} else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) &&
844 					(fis->features == 0x03)) {
845 		set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
846 		port->ic_pause_timer = jiffies;
847 		return true;
848 	} else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) ||
849 		((fis->command == 0xFC) &&
850 			(fis->features == 0x27 || fis->features == 0x72 ||
851 			 fis->features == 0x62 || fis->features == 0x26))) {
852 		clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
853 		clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag);
854 		/* Com reset after secure erase or lowlevel format */
855 		mtip_restart_port(port);
856 		clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
857 		return false;
858 	}
859 
860 	return false;
861 }
862 
863 static bool mtip_commands_active(struct mtip_port *port)
864 {
865 	unsigned int active;
866 	unsigned int n;
867 
868 	/*
869 	 * Ignore s_active bit 0 of array element 0.
870 	 * This bit will always be set
871 	 */
872 	active = readl(port->s_active[0]) & 0xFFFFFFFE;
873 	for (n = 1; n < port->dd->slot_groups; n++)
874 		active |= readl(port->s_active[n]);
875 
876 	return active != 0;
877 }
878 
879 /*
880  * Wait for port to quiesce
881  *
882  * @port    Pointer to port data structure
883  * @timeout Max duration to wait (ms)
884  *
885  * return value
886  *	0	Success
887  *	-EBUSY  Commands still active
888  */
889 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout)
890 {
891 	unsigned long to;
892 	bool active = true;
893 
894 	blk_mq_quiesce_queue(port->dd->queue);
895 
896 	to = jiffies + msecs_to_jiffies(timeout);
897 	do {
898 		if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) &&
899 			test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
900 			msleep(20);
901 			continue; /* svc thd is actively issuing commands */
902 		}
903 
904 		msleep(100);
905 
906 		if (mtip_check_surprise_removal(port->dd))
907 			goto err_fault;
908 
909 		active = mtip_commands_active(port);
910 		if (!active)
911 			break;
912 	} while (time_before(jiffies, to));
913 
914 	blk_mq_unquiesce_queue(port->dd->queue);
915 	return active ? -EBUSY : 0;
916 err_fault:
917 	blk_mq_unquiesce_queue(port->dd->queue);
918 	return -EFAULT;
919 }
920 
921 struct mtip_int_cmd {
922 	int fis_len;
923 	dma_addr_t buffer;
924 	int buf_len;
925 	u32 opts;
926 };
927 
928 /*
929  * Execute an internal command and wait for the completion.
930  *
931  * @port    Pointer to the port data structure.
932  * @fis     Pointer to the FIS that describes the command.
933  * @fis_len  Length in WORDS of the FIS.
934  * @buffer  DMA accessible for command data.
935  * @buf_len  Length, in bytes, of the data buffer.
936  * @opts    Command header options, excluding the FIS length
937  *             and the number of PRD entries.
938  * @timeout Time in ms to wait for the command to complete.
939  *
940  * return value
941  *	0	 Command completed successfully.
942  *	-EFAULT  The buffer address is not correctly aligned.
943  *	-EBUSY   Internal command or other IO in progress.
944  *	-EAGAIN  Time out waiting for command to complete.
945  */
946 static int mtip_exec_internal_command(struct mtip_port *port,
947 					struct host_to_dev_fis *fis,
948 					int fis_len,
949 					dma_addr_t buffer,
950 					int buf_len,
951 					u32 opts,
952 					unsigned long timeout)
953 {
954 	struct mtip_cmd *int_cmd;
955 	struct driver_data *dd = port->dd;
956 	struct request *rq;
957 	struct mtip_int_cmd icmd = {
958 		.fis_len = fis_len,
959 		.buffer = buffer,
960 		.buf_len = buf_len,
961 		.opts = opts
962 	};
963 	int rv = 0;
964 
965 	/* Make sure the buffer is 8 byte aligned. This is asic specific. */
966 	if (buffer & 0x00000007) {
967 		dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n");
968 		return -EFAULT;
969 	}
970 
971 	if (mtip_check_surprise_removal(dd))
972 		return -EFAULT;
973 
974 	rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED);
975 	if (IS_ERR(rq)) {
976 		dbg_printk(MTIP_DRV_NAME "Unable to allocate tag for PIO cmd\n");
977 		return -EFAULT;
978 	}
979 
980 	set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
981 
982 	if (fis->command == ATA_CMD_SEC_ERASE_PREP)
983 		set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
984 
985 	clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
986 
987 	if (fis->command != ATA_CMD_STANDBYNOW1) {
988 		/* wait for io to complete if non atomic */
989 		if (mtip_quiesce_io(port, MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) {
990 			dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n");
991 			blk_mq_free_request(rq);
992 			clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
993 			wake_up_interruptible(&port->svc_wait);
994 			return -EBUSY;
995 		}
996 	}
997 
998 	/* Copy the command to the command table */
999 	int_cmd = blk_mq_rq_to_pdu(rq);
1000 	int_cmd->icmd = &icmd;
1001 	memcpy(int_cmd->command, fis, fis_len*4);
1002 
1003 	rq->timeout = timeout;
1004 
1005 	/* insert request and run queue */
1006 	blk_execute_rq(rq, true);
1007 
1008 	if (int_cmd->status) {
1009 		dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n",
1010 				fis->command, int_cmd->status);
1011 		rv = -EIO;
1012 
1013 		if (mtip_check_surprise_removal(dd) ||
1014 			test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
1015 					&dd->dd_flag)) {
1016 			dev_err(&dd->pdev->dev,
1017 				"Internal command [%02X] wait returned due to SR\n",
1018 				fis->command);
1019 			rv = -ENXIO;
1020 			goto exec_ic_exit;
1021 		}
1022 		mtip_device_reset(dd); /* recover from timeout issue */
1023 		rv = -EAGAIN;
1024 		goto exec_ic_exit;
1025 	}
1026 
1027 	if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)])
1028 			& (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))) {
1029 		rv = -ENXIO;
1030 		if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
1031 			mtip_device_reset(dd);
1032 			rv = -EAGAIN;
1033 		}
1034 	}
1035 exec_ic_exit:
1036 	/* Clear the allocated and active bits for the internal command. */
1037 	blk_mq_free_request(rq);
1038 	clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
1039 	if (rv >= 0 && mtip_pause_ncq(port, fis)) {
1040 		/* NCQ paused */
1041 		return rv;
1042 	}
1043 	wake_up_interruptible(&port->svc_wait);
1044 
1045 	return rv;
1046 }
1047 
1048 /*
1049  * Byte-swap ATA ID strings.
1050  *
1051  * ATA identify data contains strings in byte-swapped 16-bit words.
1052  * They must be swapped (on all architectures) to be usable as C strings.
1053  * This function swaps bytes in-place.
1054  *
1055  * @buf The buffer location of the string
1056  * @len The number of bytes to swap
1057  *
1058  * return value
1059  *	None
1060  */
1061 static inline void ata_swap_string(u16 *buf, unsigned int len)
1062 {
1063 	int i;
1064 	for (i = 0; i < (len/2); i++)
1065 		be16_to_cpus(&buf[i]);
1066 }
1067 
1068 static void mtip_set_timeout(struct driver_data *dd,
1069 					struct host_to_dev_fis *fis,
1070 					unsigned int *timeout, u8 erasemode)
1071 {
1072 	switch (fis->command) {
1073 	case ATA_CMD_DOWNLOAD_MICRO:
1074 		*timeout = 120000; /* 2 minutes */
1075 		break;
1076 	case ATA_CMD_SEC_ERASE_UNIT:
1077 	case 0xFC:
1078 		if (erasemode)
1079 			*timeout = ((*(dd->port->identify + 90) * 2) * 60000);
1080 		else
1081 			*timeout = ((*(dd->port->identify + 89) * 2) * 60000);
1082 		break;
1083 	case ATA_CMD_STANDBYNOW1:
1084 		*timeout = 120000;  /* 2 minutes */
1085 		break;
1086 	case 0xF7:
1087 	case 0xFA:
1088 		*timeout = 60000;  /* 60 seconds */
1089 		break;
1090 	case ATA_CMD_SMART:
1091 		*timeout = 15000;  /* 15 seconds */
1092 		break;
1093 	default:
1094 		*timeout = MTIP_IOCTL_CMD_TIMEOUT_MS;
1095 		break;
1096 	}
1097 }
1098 
1099 /*
1100  * Request the device identity information.
1101  *
1102  * If a user space buffer is not specified, i.e. is NULL, the
1103  * identify information is still read from the drive and placed
1104  * into the identify data buffer (@e port->identify) in the
1105  * port data structure.
1106  * When the identify buffer contains valid identify information @e
1107  * port->identify_valid is non-zero.
1108  *
1109  * @port	 Pointer to the port structure.
1110  * @user_buffer  A user space buffer where the identify data should be
1111  *                    copied.
1112  *
1113  * return value
1114  *	0	Command completed successfully.
1115  *	-EFAULT An error occurred while coping data to the user buffer.
1116  *	-1	Command failed.
1117  */
1118 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer)
1119 {
1120 	int rv = 0;
1121 	struct host_to_dev_fis fis;
1122 
1123 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
1124 		return -EFAULT;
1125 
1126 	/* Build the FIS. */
1127 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1128 	fis.type	= 0x27;
1129 	fis.opts	= 1 << 7;
1130 	fis.command	= ATA_CMD_ID_ATA;
1131 
1132 	/* Set the identify information as invalid. */
1133 	port->identify_valid = 0;
1134 
1135 	/* Clear the identify information. */
1136 	memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS);
1137 
1138 	/* Execute the command. */
1139 	if (mtip_exec_internal_command(port,
1140 				&fis,
1141 				5,
1142 				port->identify_dma,
1143 				sizeof(u16) * ATA_ID_WORDS,
1144 				0,
1145 				MTIP_INT_CMD_TIMEOUT_MS)
1146 				< 0) {
1147 		rv = -1;
1148 		goto out;
1149 	}
1150 
1151 	/*
1152 	 * Perform any necessary byte-swapping.  Yes, the kernel does in fact
1153 	 * perform field-sensitive swapping on the string fields.
1154 	 * See the kernel use of ata_id_string() for proof of this.
1155 	 */
1156 #ifdef __LITTLE_ENDIAN
1157 	ata_swap_string(port->identify + 27, 40);  /* model string*/
1158 	ata_swap_string(port->identify + 23, 8);   /* firmware string*/
1159 	ata_swap_string(port->identify + 10, 20);  /* serial# string*/
1160 #else
1161 	{
1162 		int i;
1163 		for (i = 0; i < ATA_ID_WORDS; i++)
1164 			port->identify[i] = le16_to_cpu(port->identify[i]);
1165 	}
1166 #endif
1167 
1168 	/* Check security locked state */
1169 	if (port->identify[128] & 0x4)
1170 		set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
1171 	else
1172 		clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
1173 
1174 	/* Set the identify buffer as valid. */
1175 	port->identify_valid = 1;
1176 
1177 	if (user_buffer) {
1178 		if (copy_to_user(
1179 			user_buffer,
1180 			port->identify,
1181 			ATA_ID_WORDS * sizeof(u16))) {
1182 			rv = -EFAULT;
1183 			goto out;
1184 		}
1185 	}
1186 
1187 out:
1188 	return rv;
1189 }
1190 
1191 /*
1192  * Issue a standby immediate command to the device.
1193  *
1194  * @port Pointer to the port structure.
1195  *
1196  * return value
1197  *	0	Command was executed successfully.
1198  *	-1	An error occurred while executing the command.
1199  */
1200 static int mtip_standby_immediate(struct mtip_port *port)
1201 {
1202 	int rv;
1203 	struct host_to_dev_fis	fis;
1204 	unsigned long __maybe_unused start;
1205 	unsigned int timeout;
1206 
1207 	/* Build the FIS. */
1208 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1209 	fis.type	= 0x27;
1210 	fis.opts	= 1 << 7;
1211 	fis.command	= ATA_CMD_STANDBYNOW1;
1212 
1213 	mtip_set_timeout(port->dd, &fis, &timeout, 0);
1214 
1215 	start = jiffies;
1216 	rv = mtip_exec_internal_command(port,
1217 					&fis,
1218 					5,
1219 					0,
1220 					0,
1221 					0,
1222 					timeout);
1223 	dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n",
1224 			jiffies_to_msecs(jiffies - start));
1225 	if (rv)
1226 		dev_warn(&port->dd->pdev->dev,
1227 			"STANDBY IMMEDIATE command failed.\n");
1228 
1229 	return rv;
1230 }
1231 
1232 /*
1233  * Issue a READ LOG EXT command to the device.
1234  *
1235  * @port	pointer to the port structure.
1236  * @page	page number to fetch
1237  * @buffer	pointer to buffer
1238  * @buffer_dma	dma address corresponding to @buffer
1239  * @sectors	page length to fetch, in sectors
1240  *
1241  * return value
1242  *	@rv	return value from mtip_exec_internal_command()
1243  */
1244 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
1245 				dma_addr_t buffer_dma, unsigned int sectors)
1246 {
1247 	struct host_to_dev_fis fis;
1248 
1249 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1250 	fis.type	= 0x27;
1251 	fis.opts	= 1 << 7;
1252 	fis.command	= ATA_CMD_READ_LOG_EXT;
1253 	fis.sect_count	= sectors & 0xFF;
1254 	fis.sect_cnt_ex	= (sectors >> 8) & 0xFF;
1255 	fis.lba_low	= page;
1256 	fis.lba_mid	= 0;
1257 	fis.device	= ATA_DEVICE_OBS;
1258 
1259 	memset(buffer, 0, sectors * ATA_SECT_SIZE);
1260 
1261 	return mtip_exec_internal_command(port,
1262 					&fis,
1263 					5,
1264 					buffer_dma,
1265 					sectors * ATA_SECT_SIZE,
1266 					0,
1267 					MTIP_INT_CMD_TIMEOUT_MS);
1268 }
1269 
1270 /*
1271  * Issue a SMART READ DATA command to the device.
1272  *
1273  * @port	pointer to the port structure.
1274  * @buffer	pointer to buffer
1275  * @buffer_dma	dma address corresponding to @buffer
1276  *
1277  * return value
1278  *	@rv	return value from mtip_exec_internal_command()
1279  */
1280 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer,
1281 					dma_addr_t buffer_dma)
1282 {
1283 	struct host_to_dev_fis fis;
1284 
1285 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1286 	fis.type	= 0x27;
1287 	fis.opts	= 1 << 7;
1288 	fis.command	= ATA_CMD_SMART;
1289 	fis.features	= 0xD0;
1290 	fis.sect_count	= 1;
1291 	fis.lba_mid	= 0x4F;
1292 	fis.lba_hi	= 0xC2;
1293 	fis.device	= ATA_DEVICE_OBS;
1294 
1295 	return mtip_exec_internal_command(port,
1296 					&fis,
1297 					5,
1298 					buffer_dma,
1299 					ATA_SECT_SIZE,
1300 					0,
1301 					15000);
1302 }
1303 
1304 /*
1305  * Get the value of a smart attribute
1306  *
1307  * @port	pointer to the port structure
1308  * @id		attribute number
1309  * @attrib	pointer to return attrib information corresponding to @id
1310  *
1311  * return value
1312  *	-EINVAL	NULL buffer passed or unsupported attribute @id.
1313  *	-EPERM	Identify data not valid, SMART not supported or not enabled
1314  */
1315 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
1316 						struct smart_attr *attrib)
1317 {
1318 	int rv, i;
1319 	struct smart_attr *pattr;
1320 
1321 	if (!attrib)
1322 		return -EINVAL;
1323 
1324 	if (!port->identify_valid) {
1325 		dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n");
1326 		return -EPERM;
1327 	}
1328 	if (!(port->identify[82] & 0x1)) {
1329 		dev_warn(&port->dd->pdev->dev, "SMART not supported\n");
1330 		return -EPERM;
1331 	}
1332 	if (!(port->identify[85] & 0x1)) {
1333 		dev_warn(&port->dd->pdev->dev, "SMART not enabled\n");
1334 		return -EPERM;
1335 	}
1336 
1337 	memset(port->smart_buf, 0, ATA_SECT_SIZE);
1338 	rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma);
1339 	if (rv) {
1340 		dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n");
1341 		return rv;
1342 	}
1343 
1344 	pattr = (struct smart_attr *)(port->smart_buf + 2);
1345 	for (i = 0; i < 29; i++, pattr++)
1346 		if (pattr->attr_id == id) {
1347 			memcpy(attrib, pattr, sizeof(struct smart_attr));
1348 			break;
1349 		}
1350 
1351 	if (i == 29) {
1352 		dev_warn(&port->dd->pdev->dev,
1353 			"Query for invalid SMART attribute ID\n");
1354 		rv = -EINVAL;
1355 	}
1356 
1357 	return rv;
1358 }
1359 
1360 /*
1361  * Get the drive capacity.
1362  *
1363  * @dd      Pointer to the device data structure.
1364  * @sectors Pointer to the variable that will receive the sector count.
1365  *
1366  * return value
1367  *	1 Capacity was returned successfully.
1368  *	0 The identify information is invalid.
1369  */
1370 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors)
1371 {
1372 	struct mtip_port *port = dd->port;
1373 	u64 total, raw0, raw1, raw2, raw3;
1374 	raw0 = port->identify[100];
1375 	raw1 = port->identify[101];
1376 	raw2 = port->identify[102];
1377 	raw3 = port->identify[103];
1378 	total = raw0 | raw1<<16 | raw2<<32 | raw3<<48;
1379 	*sectors = total;
1380 	return (bool) !!port->identify_valid;
1381 }
1382 
1383 /*
1384  * Display the identify command data.
1385  *
1386  * @port Pointer to the port data structure.
1387  *
1388  * return value
1389  *	None
1390  */
1391 static void mtip_dump_identify(struct mtip_port *port)
1392 {
1393 	sector_t sectors;
1394 	unsigned short revid;
1395 	char cbuf[42];
1396 
1397 	if (!port->identify_valid)
1398 		return;
1399 
1400 	strscpy(cbuf, (char *)(port->identify + 10), 21);
1401 	dev_info(&port->dd->pdev->dev,
1402 		"Serial No.: %s\n", cbuf);
1403 
1404 	strscpy(cbuf, (char *)(port->identify + 23), 9);
1405 	dev_info(&port->dd->pdev->dev,
1406 		"Firmware Ver.: %s\n", cbuf);
1407 
1408 	strscpy(cbuf, (char *)(port->identify + 27), 41);
1409 	dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf);
1410 
1411 	dev_info(&port->dd->pdev->dev, "Security: %04x %s\n",
1412 		port->identify[128],
1413 		port->identify[128] & 0x4 ? "(LOCKED)" : "");
1414 
1415 	if (mtip_hw_get_capacity(port->dd, &sectors))
1416 		dev_info(&port->dd->pdev->dev,
1417 			"Capacity: %llu sectors (%llu MB)\n",
1418 			 (u64)sectors,
1419 			 ((u64)sectors) * ATA_SECT_SIZE >> 20);
1420 
1421 	pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid);
1422 	switch (revid & 0xFF) {
1423 	case 0x1:
1424 		strscpy(cbuf, "A0", 3);
1425 		break;
1426 	case 0x3:
1427 		strscpy(cbuf, "A2", 3);
1428 		break;
1429 	default:
1430 		strscpy(cbuf, "?", 2);
1431 		break;
1432 	}
1433 	dev_info(&port->dd->pdev->dev,
1434 		"Card Type: %s\n", cbuf);
1435 }
1436 
1437 /*
1438  * Map the commands scatter list into the command table.
1439  *
1440  * @command Pointer to the command.
1441  * @nents Number of scatter list entries.
1442  *
1443  * return value
1444  *	None
1445  */
1446 static inline void fill_command_sg(struct driver_data *dd,
1447 				struct mtip_cmd *command,
1448 				int nents)
1449 {
1450 	int n;
1451 	unsigned int dma_len;
1452 	struct mtip_cmd_sg *command_sg;
1453 	struct scatterlist *sg;
1454 
1455 	command_sg = command->command + AHCI_CMD_TBL_HDR_SZ;
1456 
1457 	for_each_sg(command->sg, sg, nents, n) {
1458 		dma_len = sg_dma_len(sg);
1459 		if (dma_len > 0x400000)
1460 			dev_err(&dd->pdev->dev,
1461 				"DMA segment length truncated\n");
1462 		command_sg->info = cpu_to_le32((dma_len-1) & 0x3FFFFF);
1463 		command_sg->dba	=  cpu_to_le32(sg_dma_address(sg));
1464 		command_sg->dba_upper =
1465 			cpu_to_le32((sg_dma_address(sg) >> 16) >> 16);
1466 		command_sg++;
1467 	}
1468 }
1469 
1470 /*
1471  * @brief Execute a drive command.
1472  *
1473  * return value 0 The command completed successfully.
1474  * return value -1 An error occurred while executing the command.
1475  */
1476 static int exec_drive_task(struct mtip_port *port, u8 *command)
1477 {
1478 	struct host_to_dev_fis	fis;
1479 	struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
1480 	unsigned int to;
1481 
1482 	/* Build the FIS. */
1483 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1484 	fis.type	= 0x27;
1485 	fis.opts	= 1 << 7;
1486 	fis.command	= command[0];
1487 	fis.features	= command[1];
1488 	fis.sect_count	= command[2];
1489 	fis.sector	= command[3];
1490 	fis.cyl_low	= command[4];
1491 	fis.cyl_hi	= command[5];
1492 	fis.device	= command[6] & ~0x10; /* Clear the dev bit*/
1493 
1494 	mtip_set_timeout(port->dd, &fis, &to, 0);
1495 
1496 	dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n",
1497 		__func__,
1498 		command[0],
1499 		command[1],
1500 		command[2],
1501 		command[3],
1502 		command[4],
1503 		command[5],
1504 		command[6]);
1505 
1506 	/* Execute the command. */
1507 	if (mtip_exec_internal_command(port,
1508 				 &fis,
1509 				 5,
1510 				 0,
1511 				 0,
1512 				 0,
1513 				 to) < 0) {
1514 		return -1;
1515 	}
1516 
1517 	command[0] = reply->command; /* Status*/
1518 	command[1] = reply->features; /* Error*/
1519 	command[4] = reply->cyl_low;
1520 	command[5] = reply->cyl_hi;
1521 
1522 	dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n",
1523 		__func__,
1524 		command[0],
1525 		command[1],
1526 		command[4],
1527 		command[5]);
1528 
1529 	return 0;
1530 }
1531 
1532 /*
1533  * @brief Execute a drive command.
1534  *
1535  * @param port Pointer to the port data structure.
1536  * @param command Pointer to the user specified command parameters.
1537  * @param user_buffer Pointer to the user space buffer where read sector
1538  *                   data should be copied.
1539  *
1540  * return value 0 The command completed successfully.
1541  * return value -EFAULT An error occurred while copying the completion
1542  *                 data to the user space buffer.
1543  * return value -1 An error occurred while executing the command.
1544  */
1545 static int exec_drive_command(struct mtip_port *port, u8 *command,
1546 				void __user *user_buffer)
1547 {
1548 	struct host_to_dev_fis	fis;
1549 	struct host_to_dev_fis *reply;
1550 	u8 *buf = NULL;
1551 	dma_addr_t dma_addr = 0;
1552 	int rv = 0, xfer_sz = command[3];
1553 	unsigned int to;
1554 
1555 	if (xfer_sz) {
1556 		if (!user_buffer)
1557 			return -EFAULT;
1558 
1559 		buf = dma_alloc_coherent(&port->dd->pdev->dev,
1560 				ATA_SECT_SIZE * xfer_sz,
1561 				&dma_addr,
1562 				GFP_KERNEL);
1563 		if (!buf) {
1564 			dev_err(&port->dd->pdev->dev,
1565 				"Memory allocation failed (%d bytes)\n",
1566 				ATA_SECT_SIZE * xfer_sz);
1567 			return -ENOMEM;
1568 		}
1569 	}
1570 
1571 	/* Build the FIS. */
1572 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1573 	fis.type	= 0x27;
1574 	fis.opts	= 1 << 7;
1575 	fis.command	= command[0];
1576 	fis.features	= command[2];
1577 	fis.sect_count	= command[3];
1578 	if (fis.command == ATA_CMD_SMART) {
1579 		fis.sector	= command[1];
1580 		fis.cyl_low	= 0x4F;
1581 		fis.cyl_hi	= 0xC2;
1582 	}
1583 
1584 	mtip_set_timeout(port->dd, &fis, &to, 0);
1585 
1586 	if (xfer_sz)
1587 		reply = (port->rxfis + RX_FIS_PIO_SETUP);
1588 	else
1589 		reply = (port->rxfis + RX_FIS_D2H_REG);
1590 
1591 	dbg_printk(MTIP_DRV_NAME
1592 		" %s: User Command: cmd %x, sect %x, "
1593 		"feat %x, sectcnt %x\n",
1594 		__func__,
1595 		command[0],
1596 		command[1],
1597 		command[2],
1598 		command[3]);
1599 
1600 	/* Execute the command. */
1601 	if (mtip_exec_internal_command(port,
1602 				&fis,
1603 				 5,
1604 				 (xfer_sz ? dma_addr : 0),
1605 				 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0),
1606 				 0,
1607 				 to)
1608 				 < 0) {
1609 		rv = -EFAULT;
1610 		goto exit_drive_command;
1611 	}
1612 
1613 	/* Collect the completion status. */
1614 	command[0] = reply->command; /* Status*/
1615 	command[1] = reply->features; /* Error*/
1616 	command[2] = reply->sect_count;
1617 
1618 	dbg_printk(MTIP_DRV_NAME
1619 		" %s: Completion Status: stat %x, "
1620 		"err %x, nsect %x\n",
1621 		__func__,
1622 		command[0],
1623 		command[1],
1624 		command[2]);
1625 
1626 	if (xfer_sz) {
1627 		if (copy_to_user(user_buffer,
1628 				 buf,
1629 				 ATA_SECT_SIZE * command[3])) {
1630 			rv = -EFAULT;
1631 			goto exit_drive_command;
1632 		}
1633 	}
1634 exit_drive_command:
1635 	if (buf)
1636 		dma_free_coherent(&port->dd->pdev->dev,
1637 				ATA_SECT_SIZE * xfer_sz, buf, dma_addr);
1638 	return rv;
1639 }
1640 
1641 /*
1642  *  Indicates whether a command has a single sector payload.
1643  *
1644  *  @command passed to the device to perform the certain event.
1645  *  @features passed to the device to perform the certain event.
1646  *
1647  *  return value
1648  *	1	command is one that always has a single sector payload,
1649  *		regardless of the value in the Sector Count field.
1650  *      0       otherwise
1651  *
1652  */
1653 static unsigned int implicit_sector(unsigned char command,
1654 				    unsigned char features)
1655 {
1656 	unsigned int rv = 0;
1657 
1658 	/* list of commands that have an implicit sector count of 1 */
1659 	switch (command) {
1660 	case ATA_CMD_SEC_SET_PASS:
1661 	case ATA_CMD_SEC_UNLOCK:
1662 	case ATA_CMD_SEC_ERASE_PREP:
1663 	case ATA_CMD_SEC_ERASE_UNIT:
1664 	case ATA_CMD_SEC_FREEZE_LOCK:
1665 	case ATA_CMD_SEC_DISABLE_PASS:
1666 	case ATA_CMD_PMP_READ:
1667 	case ATA_CMD_PMP_WRITE:
1668 		rv = 1;
1669 		break;
1670 	case ATA_CMD_SET_MAX:
1671 		if (features == ATA_SET_MAX_UNLOCK)
1672 			rv = 1;
1673 		break;
1674 	case ATA_CMD_SMART:
1675 		if ((features == ATA_SMART_READ_VALUES) ||
1676 				(features == ATA_SMART_READ_THRESHOLDS))
1677 			rv = 1;
1678 		break;
1679 	case ATA_CMD_CONF_OVERLAY:
1680 		if ((features == ATA_DCO_IDENTIFY) ||
1681 				(features == ATA_DCO_SET))
1682 			rv = 1;
1683 		break;
1684 	}
1685 	return rv;
1686 }
1687 
1688 /*
1689  * Executes a taskfile
1690  * See ide_taskfile_ioctl() for derivation
1691  */
1692 static int exec_drive_taskfile(struct driver_data *dd,
1693 			       void __user *buf,
1694 			       ide_task_request_t *req_task,
1695 			       int outtotal)
1696 {
1697 	struct host_to_dev_fis	fis;
1698 	struct host_to_dev_fis *reply;
1699 	u8 *outbuf = NULL;
1700 	u8 *inbuf = NULL;
1701 	dma_addr_t outbuf_dma = 0;
1702 	dma_addr_t inbuf_dma = 0;
1703 	dma_addr_t dma_buffer = 0;
1704 	int err = 0;
1705 	unsigned int taskin = 0;
1706 	unsigned int taskout = 0;
1707 	u8 nsect = 0;
1708 	unsigned int timeout;
1709 	unsigned int force_single_sector;
1710 	unsigned int transfer_size;
1711 	unsigned long task_file_data;
1712 	int intotal = outtotal + req_task->out_size;
1713 	int erasemode = 0;
1714 
1715 	taskout = req_task->out_size;
1716 	taskin = req_task->in_size;
1717 	/* 130560 = 512 * 0xFF*/
1718 	if (taskin > 130560 || taskout > 130560)
1719 		return -EINVAL;
1720 
1721 	if (taskout) {
1722 		outbuf = memdup_user(buf + outtotal, taskout);
1723 		if (IS_ERR(outbuf))
1724 			return PTR_ERR(outbuf);
1725 
1726 		outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf,
1727 					    taskout, DMA_TO_DEVICE);
1728 		if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) {
1729 			err = -ENOMEM;
1730 			goto abort;
1731 		}
1732 		dma_buffer = outbuf_dma;
1733 	}
1734 
1735 	if (taskin) {
1736 		inbuf = memdup_user(buf + intotal, taskin);
1737 		if (IS_ERR(inbuf)) {
1738 			err = PTR_ERR(inbuf);
1739 			inbuf = NULL;
1740 			goto abort;
1741 		}
1742 		inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf,
1743 					   taskin, DMA_FROM_DEVICE);
1744 		if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) {
1745 			err = -ENOMEM;
1746 			goto abort;
1747 		}
1748 		dma_buffer = inbuf_dma;
1749 	}
1750 
1751 	/* only supports PIO and non-data commands from this ioctl. */
1752 	switch (req_task->data_phase) {
1753 	case TASKFILE_OUT:
1754 		nsect = taskout / ATA_SECT_SIZE;
1755 		reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1756 		break;
1757 	case TASKFILE_IN:
1758 		reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1759 		break;
1760 	case TASKFILE_NO_DATA:
1761 		reply = (dd->port->rxfis + RX_FIS_D2H_REG);
1762 		break;
1763 	default:
1764 		err = -EINVAL;
1765 		goto abort;
1766 	}
1767 
1768 	/* Build the FIS. */
1769 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1770 
1771 	fis.type	= 0x27;
1772 	fis.opts	= 1 << 7;
1773 	fis.command	= req_task->io_ports[7];
1774 	fis.features	= req_task->io_ports[1];
1775 	fis.sect_count	= req_task->io_ports[2];
1776 	fis.lba_low	= req_task->io_ports[3];
1777 	fis.lba_mid	= req_task->io_ports[4];
1778 	fis.lba_hi	= req_task->io_ports[5];
1779 	 /* Clear the dev bit*/
1780 	fis.device	= req_task->io_ports[6] & ~0x10;
1781 
1782 	if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) {
1783 		req_task->in_flags.all	=
1784 			IDE_TASKFILE_STD_IN_FLAGS |
1785 			(IDE_HOB_STD_IN_FLAGS << 8);
1786 		fis.lba_low_ex		= req_task->hob_ports[3];
1787 		fis.lba_mid_ex		= req_task->hob_ports[4];
1788 		fis.lba_hi_ex		= req_task->hob_ports[5];
1789 		fis.features_ex		= req_task->hob_ports[1];
1790 		fis.sect_cnt_ex		= req_task->hob_ports[2];
1791 
1792 	} else {
1793 		req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS;
1794 	}
1795 
1796 	force_single_sector = implicit_sector(fis.command, fis.features);
1797 
1798 	if ((taskin || taskout) && (!fis.sect_count)) {
1799 		if (nsect)
1800 			fis.sect_count = nsect;
1801 		else {
1802 			if (!force_single_sector) {
1803 				dev_warn(&dd->pdev->dev,
1804 					"data movement but "
1805 					"sect_count is 0\n");
1806 				err = -EINVAL;
1807 				goto abort;
1808 			}
1809 		}
1810 	}
1811 
1812 	dbg_printk(MTIP_DRV_NAME
1813 		" %s: cmd %x, feat %x, nsect %x,"
1814 		" sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x,"
1815 		" head/dev %x\n",
1816 		__func__,
1817 		fis.command,
1818 		fis.features,
1819 		fis.sect_count,
1820 		fis.lba_low,
1821 		fis.lba_mid,
1822 		fis.lba_hi,
1823 		fis.device);
1824 
1825 	/* check for erase mode support during secure erase.*/
1826 	if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf &&
1827 					(outbuf[0] & MTIP_SEC_ERASE_MODE)) {
1828 		erasemode = 1;
1829 	}
1830 
1831 	mtip_set_timeout(dd, &fis, &timeout, erasemode);
1832 
1833 	/* Determine the correct transfer size.*/
1834 	if (force_single_sector)
1835 		transfer_size = ATA_SECT_SIZE;
1836 	else
1837 		transfer_size = ATA_SECT_SIZE * fis.sect_count;
1838 
1839 	/* Execute the command.*/
1840 	if (mtip_exec_internal_command(dd->port,
1841 				 &fis,
1842 				 5,
1843 				 dma_buffer,
1844 				 transfer_size,
1845 				 0,
1846 				 timeout) < 0) {
1847 		err = -EIO;
1848 		goto abort;
1849 	}
1850 
1851 	task_file_data = readl(dd->port->mmio+PORT_TFDATA);
1852 
1853 	if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) {
1854 		reply = dd->port->rxfis + RX_FIS_PIO_SETUP;
1855 		req_task->io_ports[7] = reply->control;
1856 	} else {
1857 		reply = dd->port->rxfis + RX_FIS_D2H_REG;
1858 		req_task->io_ports[7] = reply->command;
1859 	}
1860 
1861 	/* reclaim the DMA buffers.*/
1862 	if (inbuf_dma)
1863 		dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
1864 				 DMA_FROM_DEVICE);
1865 	if (outbuf_dma)
1866 		dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
1867 				 DMA_TO_DEVICE);
1868 	inbuf_dma  = 0;
1869 	outbuf_dma = 0;
1870 
1871 	/* return the ATA registers to the caller.*/
1872 	req_task->io_ports[1] = reply->features;
1873 	req_task->io_ports[2] = reply->sect_count;
1874 	req_task->io_ports[3] = reply->lba_low;
1875 	req_task->io_ports[4] = reply->lba_mid;
1876 	req_task->io_ports[5] = reply->lba_hi;
1877 	req_task->io_ports[6] = reply->device;
1878 
1879 	if (req_task->out_flags.all & 1)  {
1880 
1881 		req_task->hob_ports[3] = reply->lba_low_ex;
1882 		req_task->hob_ports[4] = reply->lba_mid_ex;
1883 		req_task->hob_ports[5] = reply->lba_hi_ex;
1884 		req_task->hob_ports[1] = reply->features_ex;
1885 		req_task->hob_ports[2] = reply->sect_cnt_ex;
1886 	}
1887 	dbg_printk(MTIP_DRV_NAME
1888 		" %s: Completion: stat %x,"
1889 		"err %x, sect_cnt %x, lbalo %x,"
1890 		"lbamid %x, lbahi %x, dev %x\n",
1891 		__func__,
1892 		req_task->io_ports[7],
1893 		req_task->io_ports[1],
1894 		req_task->io_ports[2],
1895 		req_task->io_ports[3],
1896 		req_task->io_ports[4],
1897 		req_task->io_ports[5],
1898 		req_task->io_ports[6]);
1899 
1900 	if (taskout) {
1901 		if (copy_to_user(buf + outtotal, outbuf, taskout)) {
1902 			err = -EFAULT;
1903 			goto abort;
1904 		}
1905 	}
1906 	if (taskin) {
1907 		if (copy_to_user(buf + intotal, inbuf, taskin)) {
1908 			err = -EFAULT;
1909 			goto abort;
1910 		}
1911 	}
1912 abort:
1913 	if (inbuf_dma)
1914 		dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
1915 				 DMA_FROM_DEVICE);
1916 	if (outbuf_dma)
1917 		dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
1918 				 DMA_TO_DEVICE);
1919 	kfree(outbuf);
1920 	kfree(inbuf);
1921 
1922 	return err;
1923 }
1924 
1925 /*
1926  * Handle IOCTL calls from the Block Layer.
1927  *
1928  * This function is called by the Block Layer when it receives an IOCTL
1929  * command that it does not understand. If the IOCTL command is not supported
1930  * this function returns -ENOTTY.
1931  *
1932  * @dd  Pointer to the driver data structure.
1933  * @cmd IOCTL command passed from the Block Layer.
1934  * @arg IOCTL argument passed from the Block Layer.
1935  *
1936  * return value
1937  *	0	The IOCTL completed successfully.
1938  *	-ENOTTY The specified command is not supported.
1939  *	-EFAULT An error occurred copying data to a user space buffer.
1940  *	-EIO	An error occurred while executing the command.
1941  */
1942 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd,
1943 			 unsigned long arg)
1944 {
1945 	switch (cmd) {
1946 	case HDIO_GET_IDENTITY:
1947 	{
1948 		if (copy_to_user((void __user *)arg, dd->port->identify,
1949 						sizeof(u16) * ATA_ID_WORDS))
1950 			return -EFAULT;
1951 		break;
1952 	}
1953 	case HDIO_DRIVE_CMD:
1954 	{
1955 		u8 drive_command[4];
1956 
1957 		/* Copy the user command info to our buffer. */
1958 		if (copy_from_user(drive_command,
1959 					 (void __user *) arg,
1960 					 sizeof(drive_command)))
1961 			return -EFAULT;
1962 
1963 		/* Execute the drive command. */
1964 		if (exec_drive_command(dd->port,
1965 					 drive_command,
1966 					 (void __user *) (arg+4)))
1967 			return -EIO;
1968 
1969 		/* Copy the status back to the users buffer. */
1970 		if (copy_to_user((void __user *) arg,
1971 					 drive_command,
1972 					 sizeof(drive_command)))
1973 			return -EFAULT;
1974 
1975 		break;
1976 	}
1977 	case HDIO_DRIVE_TASK:
1978 	{
1979 		u8 drive_command[7];
1980 
1981 		/* Copy the user command info to our buffer. */
1982 		if (copy_from_user(drive_command,
1983 					 (void __user *) arg,
1984 					 sizeof(drive_command)))
1985 			return -EFAULT;
1986 
1987 		/* Execute the drive command. */
1988 		if (exec_drive_task(dd->port, drive_command))
1989 			return -EIO;
1990 
1991 		/* Copy the status back to the users buffer. */
1992 		if (copy_to_user((void __user *) arg,
1993 					 drive_command,
1994 					 sizeof(drive_command)))
1995 			return -EFAULT;
1996 
1997 		break;
1998 	}
1999 	case HDIO_DRIVE_TASKFILE: {
2000 		ide_task_request_t req_task;
2001 		int ret, outtotal;
2002 
2003 		if (copy_from_user(&req_task, (void __user *) arg,
2004 					sizeof(req_task)))
2005 			return -EFAULT;
2006 
2007 		outtotal = sizeof(req_task);
2008 
2009 		ret = exec_drive_taskfile(dd, (void __user *) arg,
2010 						&req_task, outtotal);
2011 
2012 		if (copy_to_user((void __user *) arg, &req_task,
2013 							sizeof(req_task)))
2014 			return -EFAULT;
2015 
2016 		return ret;
2017 	}
2018 
2019 	default:
2020 		return -EINVAL;
2021 	}
2022 	return 0;
2023 }
2024 
2025 /*
2026  * Submit an IO to the hw
2027  *
2028  * This function is called by the block layer to issue an io
2029  * to the device. Upon completion, the callback function will
2030  * be called with the data parameter passed as the callback data.
2031  *
2032  * @dd       Pointer to the driver data structure.
2033  * @start    First sector to read.
2034  * @nsect    Number of sectors to read.
2035  * @tag      The tag of this read command.
2036  * @callback Pointer to the function that should be called
2037  *	     when the read completes.
2038  * @data     Callback data passed to the callback function
2039  *	     when the read completes.
2040  * @dir      Direction (read or write)
2041  *
2042  * return value
2043  *	None
2044  */
2045 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq,
2046 			      struct mtip_cmd *command,
2047 			      struct blk_mq_hw_ctx *hctx)
2048 {
2049 	struct mtip_cmd_hdr *hdr =
2050 		dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
2051 	struct host_to_dev_fis	*fis;
2052 	struct mtip_port *port = dd->port;
2053 	int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
2054 	u64 start = blk_rq_pos(rq);
2055 	unsigned int nsect = blk_rq_sectors(rq);
2056 	unsigned int nents;
2057 
2058 	/* Map the scatter list for DMA access */
2059 	nents = blk_rq_map_sg(hctx->queue, rq, command->sg);
2060 	nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir);
2061 
2062 	prefetch(&port->flags);
2063 
2064 	command->scatter_ents = nents;
2065 
2066 	/*
2067 	 * The number of retries for this command before it is
2068 	 * reported as a failure to the upper layers.
2069 	 */
2070 	command->retries = MTIP_MAX_RETRIES;
2071 
2072 	/* Fill out fis */
2073 	fis = command->command;
2074 	fis->type        = 0x27;
2075 	fis->opts        = 1 << 7;
2076 	if (dma_dir == DMA_FROM_DEVICE)
2077 		fis->command = ATA_CMD_FPDMA_READ;
2078 	else
2079 		fis->command = ATA_CMD_FPDMA_WRITE;
2080 	fis->lba_low     = start & 0xFF;
2081 	fis->lba_mid     = (start >> 8) & 0xFF;
2082 	fis->lba_hi      = (start >> 16) & 0xFF;
2083 	fis->lba_low_ex  = (start >> 24) & 0xFF;
2084 	fis->lba_mid_ex  = (start >> 32) & 0xFF;
2085 	fis->lba_hi_ex   = (start >> 40) & 0xFF;
2086 	fis->device	 = 1 << 6;
2087 	fis->features    = nsect & 0xFF;
2088 	fis->features_ex = (nsect >> 8) & 0xFF;
2089 	fis->sect_count  = ((rq->tag << 3) | (rq->tag >> 5));
2090 	fis->sect_cnt_ex = 0;
2091 	fis->control     = 0;
2092 	fis->res2        = 0;
2093 	fis->res3        = 0;
2094 	fill_command_sg(dd, command, nents);
2095 
2096 	if (unlikely(command->unaligned))
2097 		fis->device |= 1 << 7;
2098 
2099 	/* Populate the command header */
2100 	hdr->ctba = cpu_to_le32(command->command_dma & 0xFFFFFFFF);
2101 	if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
2102 		hdr->ctbau = cpu_to_le32((command->command_dma >> 16) >> 16);
2103 	hdr->opts = cpu_to_le32((nents << 16) | 5 | AHCI_CMD_PREFETCH);
2104 	hdr->byte_count = 0;
2105 
2106 	command->direction = dma_dir;
2107 
2108 	/*
2109 	 * To prevent this command from being issued
2110 	 * if an internal command is in progress or error handling is active.
2111 	 */
2112 	if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) {
2113 		set_bit(rq->tag, port->cmds_to_issue);
2114 		set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
2115 		return;
2116 	}
2117 
2118 	/* Issue the command to the hardware */
2119 	mtip_issue_ncq_command(port, rq->tag);
2120 }
2121 
2122 /*
2123  * Sysfs status dump.
2124  *
2125  * @dev  Pointer to the device structure, passed by the kernrel.
2126  * @attr Pointer to the device_attribute structure passed by the kernel.
2127  * @buf  Pointer to the char buffer that will receive the stats info.
2128  *
2129  * return value
2130  *	The size, in bytes, of the data copied into buf.
2131  */
2132 static ssize_t mtip_hw_show_status(struct device *dev,
2133 				struct device_attribute *attr,
2134 				char *buf)
2135 {
2136 	struct driver_data *dd = dev_to_disk(dev)->private_data;
2137 	int size = 0;
2138 
2139 	if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
2140 		size += sprintf(buf, "%s", "thermal_shutdown\n");
2141 	else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag))
2142 		size += sprintf(buf, "%s", "write_protect\n");
2143 	else
2144 		size += sprintf(buf, "%s", "online\n");
2145 
2146 	return size;
2147 }
2148 
2149 static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL);
2150 
2151 static struct attribute *mtip_disk_attrs[] = {
2152 	&dev_attr_status.attr,
2153 	NULL,
2154 };
2155 
2156 static const struct attribute_group mtip_disk_attr_group = {
2157 	.attrs = mtip_disk_attrs,
2158 };
2159 
2160 static const struct attribute_group *mtip_disk_attr_groups[] = {
2161 	&mtip_disk_attr_group,
2162 	NULL,
2163 };
2164 
2165 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf,
2166 				  size_t len, loff_t *offset)
2167 {
2168 	struct driver_data *dd =  (struct driver_data *)f->private_data;
2169 	char *buf;
2170 	u32 group_allocated;
2171 	int size = *offset;
2172 	int n, rv = 0;
2173 
2174 	if (!len || size)
2175 		return 0;
2176 
2177 	buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2178 	if (!buf)
2179 		return -ENOMEM;
2180 
2181 	size += sprintf(&buf[size], "H/ S ACTive      : [ 0x");
2182 
2183 	for (n = dd->slot_groups-1; n >= 0; n--)
2184 		size += sprintf(&buf[size], "%08X ",
2185 					 readl(dd->port->s_active[n]));
2186 
2187 	size += sprintf(&buf[size], "]\n");
2188 	size += sprintf(&buf[size], "H/ Command Issue : [ 0x");
2189 
2190 	for (n = dd->slot_groups-1; n >= 0; n--)
2191 		size += sprintf(&buf[size], "%08X ",
2192 					readl(dd->port->cmd_issue[n]));
2193 
2194 	size += sprintf(&buf[size], "]\n");
2195 	size += sprintf(&buf[size], "H/ Completed     : [ 0x");
2196 
2197 	for (n = dd->slot_groups-1; n >= 0; n--)
2198 		size += sprintf(&buf[size], "%08X ",
2199 				readl(dd->port->completed[n]));
2200 
2201 	size += sprintf(&buf[size], "]\n");
2202 	size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n",
2203 				readl(dd->port->mmio + PORT_IRQ_STAT));
2204 	size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n",
2205 				readl(dd->mmio + HOST_IRQ_STAT));
2206 	size += sprintf(&buf[size], "\n");
2207 
2208 	size += sprintf(&buf[size], "L/ Commands in Q : [ 0x");
2209 
2210 	for (n = dd->slot_groups-1; n >= 0; n--) {
2211 		if (sizeof(long) > sizeof(u32))
2212 			group_allocated =
2213 				dd->port->cmds_to_issue[n/2] >> (32*(n&1));
2214 		else
2215 			group_allocated = dd->port->cmds_to_issue[n];
2216 		size += sprintf(&buf[size], "%08X ", group_allocated);
2217 	}
2218 	size += sprintf(&buf[size], "]\n");
2219 
2220 	*offset = size <= len ? size : len;
2221 	size = copy_to_user(ubuf, buf, *offset);
2222 	if (size)
2223 		rv = -EFAULT;
2224 
2225 	kfree(buf);
2226 	return rv ? rv : *offset;
2227 }
2228 
2229 static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf,
2230 				  size_t len, loff_t *offset)
2231 {
2232 	struct driver_data *dd =  (struct driver_data *)f->private_data;
2233 	char *buf;
2234 	int size = *offset;
2235 	int rv = 0;
2236 
2237 	if (!len || size)
2238 		return 0;
2239 
2240 	buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2241 	if (!buf)
2242 		return -ENOMEM;
2243 
2244 	size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n",
2245 							dd->port->flags);
2246 	size += sprintf(&buf[size], "Flag-dd   : [ %08lX ]\n",
2247 							dd->dd_flag);
2248 
2249 	*offset = size <= len ? size : len;
2250 	size = copy_to_user(ubuf, buf, *offset);
2251 	if (size)
2252 		rv = -EFAULT;
2253 
2254 	kfree(buf);
2255 	return rv ? rv : *offset;
2256 }
2257 
2258 static const struct file_operations mtip_regs_fops = {
2259 	.owner  = THIS_MODULE,
2260 	.open   = simple_open,
2261 	.read   = mtip_hw_read_registers,
2262 	.llseek = no_llseek,
2263 };
2264 
2265 static const struct file_operations mtip_flags_fops = {
2266 	.owner  = THIS_MODULE,
2267 	.open   = simple_open,
2268 	.read   = mtip_hw_read_flags,
2269 	.llseek = no_llseek,
2270 };
2271 
2272 static int mtip_hw_debugfs_init(struct driver_data *dd)
2273 {
2274 	if (!dfs_parent)
2275 		return -1;
2276 
2277 	dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent);
2278 	if (IS_ERR_OR_NULL(dd->dfs_node)) {
2279 		dev_warn(&dd->pdev->dev,
2280 			"Error creating node %s under debugfs\n",
2281 						dd->disk->disk_name);
2282 		dd->dfs_node = NULL;
2283 		return -1;
2284 	}
2285 
2286 	debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops);
2287 	debugfs_create_file("registers", 0444, dd->dfs_node, dd,
2288 			    &mtip_regs_fops);
2289 
2290 	return 0;
2291 }
2292 
2293 static void mtip_hw_debugfs_exit(struct driver_data *dd)
2294 {
2295 	debugfs_remove_recursive(dd->dfs_node);
2296 }
2297 
2298 /*
2299  * Perform any init/resume time hardware setup
2300  *
2301  * @dd Pointer to the driver data structure.
2302  *
2303  * return value
2304  *	None
2305  */
2306 static inline void hba_setup(struct driver_data *dd)
2307 {
2308 	u32 hwdata;
2309 	hwdata = readl(dd->mmio + HOST_HSORG);
2310 
2311 	/* interrupt bug workaround: use only 1 IS bit.*/
2312 	writel(hwdata |
2313 		HSORG_DISABLE_SLOTGRP_INTR |
2314 		HSORG_DISABLE_SLOTGRP_PXIS,
2315 		dd->mmio + HOST_HSORG);
2316 }
2317 
2318 static int mtip_device_unaligned_constrained(struct driver_data *dd)
2319 {
2320 	return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0);
2321 }
2322 
2323 /*
2324  * Detect the details of the product, and store anything needed
2325  * into the driver data structure.  This includes product type and
2326  * version and number of slot groups.
2327  *
2328  * @dd Pointer to the driver data structure.
2329  *
2330  * return value
2331  *	None
2332  */
2333 static void mtip_detect_product(struct driver_data *dd)
2334 {
2335 	u32 hwdata;
2336 	unsigned int rev, slotgroups;
2337 
2338 	/*
2339 	 * HBA base + 0xFC [15:0] - vendor-specific hardware interface
2340 	 * info register:
2341 	 * [15:8] hardware/software interface rev#
2342 	 * [   3] asic-style interface
2343 	 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style).
2344 	 */
2345 	hwdata = readl(dd->mmio + HOST_HSORG);
2346 
2347 	dd->product_type = MTIP_PRODUCT_UNKNOWN;
2348 	dd->slot_groups = 1;
2349 
2350 	if (hwdata & 0x8) {
2351 		dd->product_type = MTIP_PRODUCT_ASICFPGA;
2352 		rev = (hwdata & HSORG_HWREV) >> 8;
2353 		slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1;
2354 		dev_info(&dd->pdev->dev,
2355 			"ASIC-FPGA design, HS rev 0x%x, "
2356 			"%i slot groups [%i slots]\n",
2357 			 rev,
2358 			 slotgroups,
2359 			 slotgroups * 32);
2360 
2361 		if (slotgroups > MTIP_MAX_SLOT_GROUPS) {
2362 			dev_warn(&dd->pdev->dev,
2363 				"Warning: driver only supports "
2364 				"%i slot groups.\n", MTIP_MAX_SLOT_GROUPS);
2365 			slotgroups = MTIP_MAX_SLOT_GROUPS;
2366 		}
2367 		dd->slot_groups = slotgroups;
2368 		return;
2369 	}
2370 
2371 	dev_warn(&dd->pdev->dev, "Unrecognized product id\n");
2372 }
2373 
2374 /*
2375  * Blocking wait for FTL rebuild to complete
2376  *
2377  * @dd Pointer to the DRIVER_DATA structure.
2378  *
2379  * return value
2380  *	0	FTL rebuild completed successfully
2381  *	-EFAULT FTL rebuild error/timeout/interruption
2382  */
2383 static int mtip_ftl_rebuild_poll(struct driver_data *dd)
2384 {
2385 	unsigned long timeout, cnt = 0, start;
2386 
2387 	dev_warn(&dd->pdev->dev,
2388 		"FTL rebuild in progress. Polling for completion.\n");
2389 
2390 	start = jiffies;
2391 	timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS);
2392 
2393 	do {
2394 		if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
2395 				&dd->dd_flag)))
2396 			return -EFAULT;
2397 		if (mtip_check_surprise_removal(dd))
2398 			return -EFAULT;
2399 
2400 		if (mtip_get_identify(dd->port, NULL) < 0)
2401 			return -EFAULT;
2402 
2403 		if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2404 			MTIP_FTL_REBUILD_MAGIC) {
2405 			ssleep(1);
2406 			/* Print message every 3 minutes */
2407 			if (cnt++ >= 180) {
2408 				dev_warn(&dd->pdev->dev,
2409 				"FTL rebuild in progress (%d secs).\n",
2410 				jiffies_to_msecs(jiffies - start) / 1000);
2411 				cnt = 0;
2412 			}
2413 		} else {
2414 			dev_warn(&dd->pdev->dev,
2415 				"FTL rebuild complete (%d secs).\n",
2416 			jiffies_to_msecs(jiffies - start) / 1000);
2417 			mtip_block_initialize(dd);
2418 			return 0;
2419 		}
2420 	} while (time_before(jiffies, timeout));
2421 
2422 	/* Check for timeout */
2423 	dev_err(&dd->pdev->dev,
2424 		"Timed out waiting for FTL rebuild to complete (%d secs).\n",
2425 		jiffies_to_msecs(jiffies - start) / 1000);
2426 	return -EFAULT;
2427 }
2428 
2429 static void mtip_softirq_done_fn(struct request *rq)
2430 {
2431 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
2432 	struct driver_data *dd = rq->q->queuedata;
2433 
2434 	/* Unmap the DMA scatter list entries */
2435 	dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents,
2436 							cmd->direction);
2437 
2438 	if (unlikely(cmd->unaligned))
2439 		atomic_inc(&dd->port->cmd_slot_unal);
2440 
2441 	blk_mq_end_request(rq, cmd->status);
2442 }
2443 
2444 static bool mtip_abort_cmd(struct request *req, void *data)
2445 {
2446 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
2447 	struct driver_data *dd = data;
2448 
2449 	dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag);
2450 
2451 	clear_bit(req->tag, dd->port->cmds_to_issue);
2452 	cmd->status = BLK_STS_IOERR;
2453 	mtip_softirq_done_fn(req);
2454 	return true;
2455 }
2456 
2457 static bool mtip_queue_cmd(struct request *req, void *data)
2458 {
2459 	struct driver_data *dd = data;
2460 
2461 	set_bit(req->tag, dd->port->cmds_to_issue);
2462 	blk_abort_request(req);
2463 	return true;
2464 }
2465 
2466 /*
2467  * service thread to issue queued commands
2468  *
2469  * @data Pointer to the driver data structure.
2470  *
2471  * return value
2472  *	0
2473  */
2474 
2475 static int mtip_service_thread(void *data)
2476 {
2477 	struct driver_data *dd = (struct driver_data *)data;
2478 	unsigned long slot, slot_start, slot_wrap, to;
2479 	unsigned int num_cmd_slots = dd->slot_groups * 32;
2480 	struct mtip_port *port = dd->port;
2481 
2482 	while (1) {
2483 		if (kthread_should_stop() ||
2484 			test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
2485 			goto st_out;
2486 		clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
2487 
2488 		/*
2489 		 * the condition is to check neither an internal command is
2490 		 * is in progress nor error handling is active
2491 		 */
2492 		wait_event_interruptible(port->svc_wait, (port->flags) &&
2493 			(port->flags & MTIP_PF_SVC_THD_WORK));
2494 
2495 		if (kthread_should_stop() ||
2496 			test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
2497 			goto st_out;
2498 
2499 		if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
2500 				&dd->dd_flag)))
2501 			goto st_out;
2502 
2503 		set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
2504 
2505 restart_eh:
2506 		/* Demux bits: start with error handling */
2507 		if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) {
2508 			mtip_handle_tfe(dd);
2509 			clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags);
2510 		}
2511 
2512 		if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags))
2513 			goto restart_eh;
2514 
2515 		if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) {
2516 			to = jiffies + msecs_to_jiffies(5000);
2517 
2518 			do {
2519 				mdelay(100);
2520 			} while (atomic_read(&dd->irq_workers_active) != 0 &&
2521 				time_before(jiffies, to));
2522 
2523 			if (atomic_read(&dd->irq_workers_active) != 0)
2524 				dev_warn(&dd->pdev->dev,
2525 					"Completion workers still active!");
2526 
2527 			blk_mq_quiesce_queue(dd->queue);
2528 
2529 			blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd);
2530 
2531 			set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags);
2532 
2533 			if (mtip_device_reset(dd))
2534 				blk_mq_tagset_busy_iter(&dd->tags,
2535 							mtip_abort_cmd, dd);
2536 
2537 			clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags);
2538 
2539 			blk_mq_unquiesce_queue(dd->queue);
2540 		}
2541 
2542 		if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
2543 			slot = 1;
2544 			/* used to restrict the loop to one iteration */
2545 			slot_start = num_cmd_slots;
2546 			slot_wrap = 0;
2547 			while (1) {
2548 				slot = find_next_bit(port->cmds_to_issue,
2549 						num_cmd_slots, slot);
2550 				if (slot_wrap == 1) {
2551 					if ((slot_start >= slot) ||
2552 						(slot >= num_cmd_slots))
2553 						break;
2554 				}
2555 				if (unlikely(slot_start == num_cmd_slots))
2556 					slot_start = slot;
2557 
2558 				if (unlikely(slot == num_cmd_slots)) {
2559 					slot = 1;
2560 					slot_wrap = 1;
2561 					continue;
2562 				}
2563 
2564 				/* Issue the command to the hardware */
2565 				mtip_issue_ncq_command(port, slot);
2566 
2567 				clear_bit(slot, port->cmds_to_issue);
2568 			}
2569 
2570 			clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
2571 		}
2572 
2573 		if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) {
2574 			if (mtip_ftl_rebuild_poll(dd) == 0)
2575 				clear_bit(MTIP_PF_REBUILD_BIT, &port->flags);
2576 		}
2577 	}
2578 
2579 st_out:
2580 	return 0;
2581 }
2582 
2583 /*
2584  * DMA region teardown
2585  *
2586  * @dd Pointer to driver_data structure
2587  *
2588  * return value
2589  *      None
2590  */
2591 static void mtip_dma_free(struct driver_data *dd)
2592 {
2593 	struct mtip_port *port = dd->port;
2594 
2595 	if (port->block1)
2596 		dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2597 					port->block1, port->block1_dma);
2598 
2599 	if (port->command_list) {
2600 		dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
2601 				port->command_list, port->command_list_dma);
2602 	}
2603 }
2604 
2605 /*
2606  * DMA region setup
2607  *
2608  * @dd Pointer to driver_data structure
2609  *
2610  * return value
2611  *      -ENOMEM Not enough free DMA region space to initialize driver
2612  */
2613 static int mtip_dma_alloc(struct driver_data *dd)
2614 {
2615 	struct mtip_port *port = dd->port;
2616 
2617 	/* Allocate dma memory for RX Fis, Identify, and Sector Buffer */
2618 	port->block1 =
2619 		dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2620 					&port->block1_dma, GFP_KERNEL);
2621 	if (!port->block1)
2622 		return -ENOMEM;
2623 
2624 	/* Allocate dma memory for command list */
2625 	port->command_list =
2626 		dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
2627 					&port->command_list_dma, GFP_KERNEL);
2628 	if (!port->command_list) {
2629 		dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2630 					port->block1, port->block1_dma);
2631 		port->block1 = NULL;
2632 		port->block1_dma = 0;
2633 		return -ENOMEM;
2634 	}
2635 
2636 	/* Setup all pointers into first DMA region */
2637 	port->rxfis         = port->block1 + AHCI_RX_FIS_OFFSET;
2638 	port->rxfis_dma     = port->block1_dma + AHCI_RX_FIS_OFFSET;
2639 	port->identify      = port->block1 + AHCI_IDFY_OFFSET;
2640 	port->identify_dma  = port->block1_dma + AHCI_IDFY_OFFSET;
2641 	port->log_buf       = port->block1 + AHCI_SECTBUF_OFFSET;
2642 	port->log_buf_dma   = port->block1_dma + AHCI_SECTBUF_OFFSET;
2643 	port->smart_buf     = port->block1 + AHCI_SMARTBUF_OFFSET;
2644 	port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET;
2645 
2646 	return 0;
2647 }
2648 
2649 static int mtip_hw_get_identify(struct driver_data *dd)
2650 {
2651 	struct smart_attr attr242;
2652 	unsigned char *buf;
2653 	int rv;
2654 
2655 	if (mtip_get_identify(dd->port, NULL) < 0)
2656 		return -EFAULT;
2657 
2658 	if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2659 		MTIP_FTL_REBUILD_MAGIC) {
2660 		set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags);
2661 		return MTIP_FTL_REBUILD_MAGIC;
2662 	}
2663 	mtip_dump_identify(dd->port);
2664 
2665 	/* check write protect, over temp and rebuild statuses */
2666 	rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
2667 				dd->port->log_buf,
2668 				dd->port->log_buf_dma, 1);
2669 	if (rv) {
2670 		dev_warn(&dd->pdev->dev,
2671 			"Error in READ LOG EXT (10h) command\n");
2672 		/* non-critical error, don't fail the load */
2673 	} else {
2674 		buf = (unsigned char *)dd->port->log_buf;
2675 		if (buf[259] & 0x1) {
2676 			dev_info(&dd->pdev->dev,
2677 				"Write protect bit is set.\n");
2678 			set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
2679 		}
2680 		if (buf[288] == 0xF7) {
2681 			dev_info(&dd->pdev->dev,
2682 				"Exceeded Tmax, drive in thermal shutdown.\n");
2683 			set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
2684 		}
2685 		if (buf[288] == 0xBF) {
2686 			dev_info(&dd->pdev->dev,
2687 				"Drive indicates rebuild has failed.\n");
2688 			set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
2689 		}
2690 	}
2691 
2692 	/* get write protect progess */
2693 	memset(&attr242, 0, sizeof(struct smart_attr));
2694 	if (mtip_get_smart_attr(dd->port, 242, &attr242))
2695 		dev_warn(&dd->pdev->dev,
2696 				"Unable to check write protect progress\n");
2697 	else
2698 		dev_info(&dd->pdev->dev,
2699 				"Write protect progress: %u%% (%u blocks)\n",
2700 				attr242.cur, le32_to_cpu(attr242.data));
2701 
2702 	return rv;
2703 }
2704 
2705 /*
2706  * Called once for each card.
2707  *
2708  * @dd Pointer to the driver data structure.
2709  *
2710  * return value
2711  *	0 on success, else an error code.
2712  */
2713 static int mtip_hw_init(struct driver_data *dd)
2714 {
2715 	int i;
2716 	int rv;
2717 	unsigned long timeout, timetaken;
2718 
2719 	dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR];
2720 
2721 	mtip_detect_product(dd);
2722 	if (dd->product_type == MTIP_PRODUCT_UNKNOWN) {
2723 		rv = -EIO;
2724 		goto out1;
2725 	}
2726 
2727 	hba_setup(dd);
2728 
2729 	dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL,
2730 				dd->numa_node);
2731 	if (!dd->port)
2732 		return -ENOMEM;
2733 
2734 	/* Continue workqueue setup */
2735 	for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
2736 		dd->work[i].port = dd->port;
2737 
2738 	/* Enable unaligned IO constraints for some devices */
2739 	if (mtip_device_unaligned_constrained(dd))
2740 		dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS;
2741 	else
2742 		dd->unal_qdepth = 0;
2743 
2744 	atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth);
2745 
2746 	/* Spinlock to prevent concurrent issue */
2747 	for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
2748 		spin_lock_init(&dd->port->cmd_issue_lock[i]);
2749 
2750 	/* Set the port mmio base address. */
2751 	dd->port->mmio	= dd->mmio + PORT_OFFSET;
2752 	dd->port->dd	= dd;
2753 
2754 	/* DMA allocations */
2755 	rv = mtip_dma_alloc(dd);
2756 	if (rv < 0)
2757 		goto out1;
2758 
2759 	/* Setup the pointers to the extended s_active and CI registers. */
2760 	for (i = 0; i < dd->slot_groups; i++) {
2761 		dd->port->s_active[i] =
2762 			dd->port->mmio + i*0x80 + PORT_SCR_ACT;
2763 		dd->port->cmd_issue[i] =
2764 			dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE;
2765 		dd->port->completed[i] =
2766 			dd->port->mmio + i*0x80 + PORT_SDBV;
2767 	}
2768 
2769 	timetaken = jiffies;
2770 	timeout = jiffies + msecs_to_jiffies(30000);
2771 	while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) &&
2772 		 time_before(jiffies, timeout)) {
2773 		mdelay(100);
2774 	}
2775 	if (unlikely(mtip_check_surprise_removal(dd))) {
2776 		timetaken = jiffies - timetaken;
2777 		dev_warn(&dd->pdev->dev,
2778 			"Surprise removal detected at %u ms\n",
2779 			jiffies_to_msecs(timetaken));
2780 		rv = -ENODEV;
2781 		goto out2 ;
2782 	}
2783 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) {
2784 		timetaken = jiffies - timetaken;
2785 		dev_warn(&dd->pdev->dev,
2786 			"Removal detected at %u ms\n",
2787 			jiffies_to_msecs(timetaken));
2788 		rv = -EFAULT;
2789 		goto out2;
2790 	}
2791 
2792 	/* Conditionally reset the HBA. */
2793 	if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) {
2794 		if (mtip_hba_reset(dd) < 0) {
2795 			dev_err(&dd->pdev->dev,
2796 				"Card did not reset within timeout\n");
2797 			rv = -EIO;
2798 			goto out2;
2799 		}
2800 	} else {
2801 		/* Clear any pending interrupts on the HBA */
2802 		writel(readl(dd->mmio + HOST_IRQ_STAT),
2803 			dd->mmio + HOST_IRQ_STAT);
2804 	}
2805 
2806 	mtip_init_port(dd->port);
2807 	mtip_start_port(dd->port);
2808 
2809 	/* Setup the ISR and enable interrupts. */
2810 	rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED,
2811 			 dev_driver_string(&dd->pdev->dev), dd);
2812 	if (rv) {
2813 		dev_err(&dd->pdev->dev,
2814 			"Unable to allocate IRQ %d\n", dd->pdev->irq);
2815 		goto out2;
2816 	}
2817 	irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding));
2818 
2819 	/* Enable interrupts on the HBA. */
2820 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
2821 					dd->mmio + HOST_CTL);
2822 
2823 	init_waitqueue_head(&dd->port->svc_wait);
2824 
2825 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
2826 		rv = -EFAULT;
2827 		goto out3;
2828 	}
2829 
2830 	return rv;
2831 
2832 out3:
2833 	/* Disable interrupts on the HBA. */
2834 	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
2835 			dd->mmio + HOST_CTL);
2836 
2837 	/* Release the IRQ. */
2838 	irq_set_affinity_hint(dd->pdev->irq, NULL);
2839 	free_irq(dd->pdev->irq, dd);
2840 
2841 out2:
2842 	mtip_deinit_port(dd->port);
2843 	mtip_dma_free(dd);
2844 
2845 out1:
2846 	/* Free the memory allocated for the for structure. */
2847 	kfree(dd->port);
2848 
2849 	return rv;
2850 }
2851 
2852 static int mtip_standby_drive(struct driver_data *dd)
2853 {
2854 	int rv = 0;
2855 
2856 	if (dd->sr || !dd->port)
2857 		return -ENODEV;
2858 	/*
2859 	 * Send standby immediate (E0h) to the drive so that it
2860 	 * saves its state.
2861 	 */
2862 	if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) &&
2863 	    !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) &&
2864 	    !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) {
2865 		rv = mtip_standby_immediate(dd->port);
2866 		if (rv)
2867 			dev_warn(&dd->pdev->dev,
2868 				"STANDBY IMMEDIATE failed\n");
2869 	}
2870 	return rv;
2871 }
2872 
2873 /*
2874  * Called to deinitialize an interface.
2875  *
2876  * @dd Pointer to the driver data structure.
2877  *
2878  * return value
2879  *	0
2880  */
2881 static int mtip_hw_exit(struct driver_data *dd)
2882 {
2883 	if (!dd->sr) {
2884 		/* de-initialize the port. */
2885 		mtip_deinit_port(dd->port);
2886 
2887 		/* Disable interrupts on the HBA. */
2888 		writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
2889 				dd->mmio + HOST_CTL);
2890 	}
2891 
2892 	/* Release the IRQ. */
2893 	irq_set_affinity_hint(dd->pdev->irq, NULL);
2894 	free_irq(dd->pdev->irq, dd);
2895 	msleep(1000);
2896 
2897 	/* Free dma regions */
2898 	mtip_dma_free(dd);
2899 
2900 	/* Free the memory allocated for the for structure. */
2901 	kfree(dd->port);
2902 	dd->port = NULL;
2903 
2904 	return 0;
2905 }
2906 
2907 /*
2908  * Issue a Standby Immediate command to the device.
2909  *
2910  * This function is called by the Block Layer just before the
2911  * system powers off during a shutdown.
2912  *
2913  * @dd Pointer to the driver data structure.
2914  *
2915  * return value
2916  *	0
2917  */
2918 static int mtip_hw_shutdown(struct driver_data *dd)
2919 {
2920 	/*
2921 	 * Send standby immediate (E0h) to the drive so that it
2922 	 * saves its state.
2923 	 */
2924 	mtip_standby_drive(dd);
2925 
2926 	return 0;
2927 }
2928 
2929 /*
2930  * Suspend function
2931  *
2932  * This function is called by the Block Layer just before the
2933  * system hibernates.
2934  *
2935  * @dd Pointer to the driver data structure.
2936  *
2937  * return value
2938  *	0	Suspend was successful
2939  *	-EFAULT Suspend was not successful
2940  */
2941 static int mtip_hw_suspend(struct driver_data *dd)
2942 {
2943 	/*
2944 	 * Send standby immediate (E0h) to the drive
2945 	 * so that it saves its state.
2946 	 */
2947 	if (mtip_standby_drive(dd) != 0) {
2948 		dev_err(&dd->pdev->dev,
2949 			"Failed standby-immediate command\n");
2950 		return -EFAULT;
2951 	}
2952 
2953 	/* Disable interrupts on the HBA.*/
2954 	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
2955 			dd->mmio + HOST_CTL);
2956 	mtip_deinit_port(dd->port);
2957 
2958 	return 0;
2959 }
2960 
2961 /*
2962  * Resume function
2963  *
2964  * This function is called by the Block Layer as the
2965  * system resumes.
2966  *
2967  * @dd Pointer to the driver data structure.
2968  *
2969  * return value
2970  *	0	Resume was successful
2971  *      -EFAULT Resume was not successful
2972  */
2973 static int mtip_hw_resume(struct driver_data *dd)
2974 {
2975 	/* Perform any needed hardware setup steps */
2976 	hba_setup(dd);
2977 
2978 	/* Reset the HBA */
2979 	if (mtip_hba_reset(dd) != 0) {
2980 		dev_err(&dd->pdev->dev,
2981 			"Unable to reset the HBA\n");
2982 		return -EFAULT;
2983 	}
2984 
2985 	/*
2986 	 * Enable the port, DMA engine, and FIS reception specific
2987 	 * h/w in controller.
2988 	 */
2989 	mtip_init_port(dd->port);
2990 	mtip_start_port(dd->port);
2991 
2992 	/* Enable interrupts on the HBA.*/
2993 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
2994 			dd->mmio + HOST_CTL);
2995 
2996 	return 0;
2997 }
2998 
2999 /*
3000  * Helper function for reusing disk name
3001  * upon hot insertion.
3002  */
3003 static int rssd_disk_name_format(char *prefix,
3004 				 int index,
3005 				 char *buf,
3006 				 int buflen)
3007 {
3008 	const int base = 'z' - 'a' + 1;
3009 	char *begin = buf + strlen(prefix);
3010 	char *end = buf + buflen;
3011 	char *p;
3012 	int unit;
3013 
3014 	p = end - 1;
3015 	*p = '\0';
3016 	unit = base;
3017 	do {
3018 		if (p == begin)
3019 			return -EINVAL;
3020 		*--p = 'a' + (index % unit);
3021 		index = (index / unit) - 1;
3022 	} while (index >= 0);
3023 
3024 	memmove(begin, p, end - p);
3025 	memcpy(buf, prefix, strlen(prefix));
3026 
3027 	return 0;
3028 }
3029 
3030 /*
3031  * Block layer IOCTL handler.
3032  *
3033  * @dev Pointer to the block_device structure.
3034  * @mode ignored
3035  * @cmd IOCTL command passed from the user application.
3036  * @arg Argument passed from the user application.
3037  *
3038  * return value
3039  *	0        IOCTL completed successfully.
3040  *	-ENOTTY  IOCTL not supported or invalid driver data
3041  *                 structure pointer.
3042  */
3043 static int mtip_block_ioctl(struct block_device *dev,
3044 			    fmode_t mode,
3045 			    unsigned cmd,
3046 			    unsigned long arg)
3047 {
3048 	struct driver_data *dd = dev->bd_disk->private_data;
3049 
3050 	if (!capable(CAP_SYS_ADMIN))
3051 		return -EACCES;
3052 
3053 	if (!dd)
3054 		return -ENOTTY;
3055 
3056 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
3057 		return -ENOTTY;
3058 
3059 	switch (cmd) {
3060 	case BLKFLSBUF:
3061 		return -ENOTTY;
3062 	default:
3063 		return mtip_hw_ioctl(dd, cmd, arg);
3064 	}
3065 }
3066 
3067 #ifdef CONFIG_COMPAT
3068 /*
3069  * Block layer compat IOCTL handler.
3070  *
3071  * @dev Pointer to the block_device structure.
3072  * @mode ignored
3073  * @cmd IOCTL command passed from the user application.
3074  * @arg Argument passed from the user application.
3075  *
3076  * return value
3077  *	0        IOCTL completed successfully.
3078  *	-ENOTTY  IOCTL not supported or invalid driver data
3079  *                 structure pointer.
3080  */
3081 static int mtip_block_compat_ioctl(struct block_device *dev,
3082 			    fmode_t mode,
3083 			    unsigned cmd,
3084 			    unsigned long arg)
3085 {
3086 	struct driver_data *dd = dev->bd_disk->private_data;
3087 
3088 	if (!capable(CAP_SYS_ADMIN))
3089 		return -EACCES;
3090 
3091 	if (!dd)
3092 		return -ENOTTY;
3093 
3094 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
3095 		return -ENOTTY;
3096 
3097 	switch (cmd) {
3098 	case BLKFLSBUF:
3099 		return -ENOTTY;
3100 	case HDIO_DRIVE_TASKFILE: {
3101 		struct mtip_compat_ide_task_request_s __user *compat_req_task;
3102 		ide_task_request_t req_task;
3103 		int compat_tasksize, outtotal, ret;
3104 
3105 		compat_tasksize =
3106 			sizeof(struct mtip_compat_ide_task_request_s);
3107 
3108 		compat_req_task =
3109 			(struct mtip_compat_ide_task_request_s __user *) arg;
3110 
3111 		if (copy_from_user(&req_task, (void __user *) arg,
3112 			compat_tasksize - (2 * sizeof(compat_long_t))))
3113 			return -EFAULT;
3114 
3115 		if (get_user(req_task.out_size, &compat_req_task->out_size))
3116 			return -EFAULT;
3117 
3118 		if (get_user(req_task.in_size, &compat_req_task->in_size))
3119 			return -EFAULT;
3120 
3121 		outtotal = sizeof(struct mtip_compat_ide_task_request_s);
3122 
3123 		ret = exec_drive_taskfile(dd, (void __user *) arg,
3124 						&req_task, outtotal);
3125 
3126 		if (copy_to_user((void __user *) arg, &req_task,
3127 				compat_tasksize -
3128 				(2 * sizeof(compat_long_t))))
3129 			return -EFAULT;
3130 
3131 		if (put_user(req_task.out_size, &compat_req_task->out_size))
3132 			return -EFAULT;
3133 
3134 		if (put_user(req_task.in_size, &compat_req_task->in_size))
3135 			return -EFAULT;
3136 
3137 		return ret;
3138 	}
3139 	default:
3140 		return mtip_hw_ioctl(dd, cmd, arg);
3141 	}
3142 }
3143 #endif
3144 
3145 /*
3146  * Obtain the geometry of the device.
3147  *
3148  * You may think that this function is obsolete, but some applications,
3149  * fdisk for example still used CHS values. This function describes the
3150  * device as having 224 heads and 56 sectors per cylinder. These values are
3151  * chosen so that each cylinder is aligned on a 4KB boundary. Since a
3152  * partition is described in terms of a start and end cylinder this means
3153  * that each partition is also 4KB aligned. Non-aligned partitions adversely
3154  * affects performance.
3155  *
3156  * @dev Pointer to the block_device strucutre.
3157  * @geo Pointer to a hd_geometry structure.
3158  *
3159  * return value
3160  *	0       Operation completed successfully.
3161  *	-ENOTTY An error occurred while reading the drive capacity.
3162  */
3163 static int mtip_block_getgeo(struct block_device *dev,
3164 				struct hd_geometry *geo)
3165 {
3166 	struct driver_data *dd = dev->bd_disk->private_data;
3167 	sector_t capacity;
3168 
3169 	if (!dd)
3170 		return -ENOTTY;
3171 
3172 	if (!(mtip_hw_get_capacity(dd, &capacity))) {
3173 		dev_warn(&dd->pdev->dev,
3174 			"Could not get drive capacity.\n");
3175 		return -ENOTTY;
3176 	}
3177 
3178 	geo->heads = 224;
3179 	geo->sectors = 56;
3180 	sector_div(capacity, (geo->heads * geo->sectors));
3181 	geo->cylinders = capacity;
3182 	return 0;
3183 }
3184 
3185 static void mtip_block_free_disk(struct gendisk *disk)
3186 {
3187 	struct driver_data *dd = disk->private_data;
3188 
3189 	ida_free(&rssd_index_ida, dd->index);
3190 	kfree(dd);
3191 }
3192 
3193 /*
3194  * Block device operation function.
3195  *
3196  * This structure contains pointers to the functions required by the block
3197  * layer.
3198  */
3199 static const struct block_device_operations mtip_block_ops = {
3200 	.ioctl		= mtip_block_ioctl,
3201 #ifdef CONFIG_COMPAT
3202 	.compat_ioctl	= mtip_block_compat_ioctl,
3203 #endif
3204 	.getgeo		= mtip_block_getgeo,
3205 	.free_disk	= mtip_block_free_disk,
3206 	.owner		= THIS_MODULE
3207 };
3208 
3209 static inline bool is_se_active(struct driver_data *dd)
3210 {
3211 	if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) {
3212 		if (dd->port->ic_pause_timer) {
3213 			unsigned long to = dd->port->ic_pause_timer +
3214 							msecs_to_jiffies(1000);
3215 			if (time_after(jiffies, to)) {
3216 				clear_bit(MTIP_PF_SE_ACTIVE_BIT,
3217 							&dd->port->flags);
3218 				clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag);
3219 				dd->port->ic_pause_timer = 0;
3220 				wake_up_interruptible(&dd->port->svc_wait);
3221 				return false;
3222 			}
3223 		}
3224 		return true;
3225 	}
3226 	return false;
3227 }
3228 
3229 static inline bool is_stopped(struct driver_data *dd, struct request *rq)
3230 {
3231 	if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO)))
3232 		return false;
3233 
3234 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
3235 		return true;
3236 	if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
3237 		return true;
3238 	if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) &&
3239 	    rq_data_dir(rq))
3240 		return true;
3241 	if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))
3242 		return true;
3243 	if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))
3244 		return true;
3245 
3246 	return false;
3247 }
3248 
3249 static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx,
3250 				  struct request *rq)
3251 {
3252 	struct driver_data *dd = hctx->queue->queuedata;
3253 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3254 
3255 	if (rq_data_dir(rq) == READ || !dd->unal_qdepth)
3256 		return false;
3257 
3258 	/*
3259 	 * If unaligned depth must be limited on this controller, mark it
3260 	 * as unaligned if the IO isn't on a 4k boundary (start of length).
3261 	 */
3262 	if (blk_rq_sectors(rq) <= 64) {
3263 		if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7))
3264 			cmd->unaligned = 1;
3265 	}
3266 
3267 	if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0)
3268 		return true;
3269 
3270 	return false;
3271 }
3272 
3273 static blk_status_t mtip_issue_reserved_cmd(struct blk_mq_hw_ctx *hctx,
3274 		struct request *rq)
3275 {
3276 	struct driver_data *dd = hctx->queue->queuedata;
3277 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3278 	struct mtip_int_cmd *icmd = cmd->icmd;
3279 	struct mtip_cmd_hdr *hdr =
3280 		dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
3281 	struct mtip_cmd_sg *command_sg;
3282 
3283 	if (mtip_commands_active(dd->port))
3284 		return BLK_STS_DEV_RESOURCE;
3285 
3286 	hdr->ctba = cpu_to_le32(cmd->command_dma & 0xFFFFFFFF);
3287 	if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
3288 		hdr->ctbau = cpu_to_le32((cmd->command_dma >> 16) >> 16);
3289 	/* Populate the SG list */
3290 	hdr->opts = cpu_to_le32(icmd->opts | icmd->fis_len);
3291 	if (icmd->buf_len) {
3292 		command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ;
3293 
3294 		command_sg->info = cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF);
3295 		command_sg->dba	= cpu_to_le32(icmd->buffer & 0xFFFFFFFF);
3296 		command_sg->dba_upper =
3297 			cpu_to_le32((icmd->buffer >> 16) >> 16);
3298 
3299 		hdr->opts |= cpu_to_le32((1 << 16));
3300 	}
3301 
3302 	/* Populate the command header */
3303 	hdr->byte_count = 0;
3304 
3305 	blk_mq_start_request(rq);
3306 	mtip_issue_non_ncq_command(dd->port, rq->tag);
3307 	return 0;
3308 }
3309 
3310 static blk_status_t mtip_queue_rq(struct blk_mq_hw_ctx *hctx,
3311 			 const struct blk_mq_queue_data *bd)
3312 {
3313 	struct driver_data *dd = hctx->queue->queuedata;
3314 	struct request *rq = bd->rq;
3315 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3316 
3317 	if (blk_rq_is_passthrough(rq))
3318 		return mtip_issue_reserved_cmd(hctx, rq);
3319 
3320 	if (unlikely(mtip_check_unal_depth(hctx, rq)))
3321 		return BLK_STS_DEV_RESOURCE;
3322 
3323 	if (is_se_active(dd) || is_stopped(dd, rq))
3324 		return BLK_STS_IOERR;
3325 
3326 	blk_mq_start_request(rq);
3327 
3328 	mtip_hw_submit_io(dd, rq, cmd, hctx);
3329 	return BLK_STS_OK;
3330 }
3331 
3332 static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq,
3333 			  unsigned int hctx_idx)
3334 {
3335 	struct driver_data *dd = set->driver_data;
3336 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3337 
3338 	if (!cmd->command)
3339 		return;
3340 
3341 	dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command,
3342 			  cmd->command_dma);
3343 }
3344 
3345 static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq,
3346 			 unsigned int hctx_idx, unsigned int numa_node)
3347 {
3348 	struct driver_data *dd = set->driver_data;
3349 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3350 
3351 	cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ,
3352 			&cmd->command_dma, GFP_KERNEL);
3353 	if (!cmd->command)
3354 		return -ENOMEM;
3355 
3356 	sg_init_table(cmd->sg, MTIP_MAX_SG);
3357 	return 0;
3358 }
3359 
3360 static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req)
3361 {
3362 	struct driver_data *dd = req->q->queuedata;
3363 
3364 	if (blk_mq_is_reserved_rq(req)) {
3365 		struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
3366 
3367 		cmd->status = BLK_STS_TIMEOUT;
3368 		blk_mq_complete_request(req);
3369 		return BLK_EH_DONE;
3370 	}
3371 
3372 	if (test_bit(req->tag, dd->port->cmds_to_issue))
3373 		goto exit_handler;
3374 
3375 	if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags))
3376 		goto exit_handler;
3377 
3378 	wake_up_interruptible(&dd->port->svc_wait);
3379 exit_handler:
3380 	return BLK_EH_RESET_TIMER;
3381 }
3382 
3383 static const struct blk_mq_ops mtip_mq_ops = {
3384 	.queue_rq	= mtip_queue_rq,
3385 	.init_request	= mtip_init_cmd,
3386 	.exit_request	= mtip_free_cmd,
3387 	.complete	= mtip_softirq_done_fn,
3388 	.timeout        = mtip_cmd_timeout,
3389 };
3390 
3391 /*
3392  * Block layer initialization function.
3393  *
3394  * This function is called once by the PCI layer for each P320
3395  * device that is connected to the system.
3396  *
3397  * @dd Pointer to the driver data structure.
3398  *
3399  * return value
3400  *	0 on success else an error code.
3401  */
3402 static int mtip_block_initialize(struct driver_data *dd)
3403 {
3404 	int rv = 0, wait_for_rebuild = 0;
3405 	sector_t capacity;
3406 	unsigned int index = 0;
3407 
3408 	if (dd->disk)
3409 		goto skip_create_disk; /* hw init done, before rebuild */
3410 
3411 	if (mtip_hw_init(dd)) {
3412 		rv = -EINVAL;
3413 		goto protocol_init_error;
3414 	}
3415 
3416 	memset(&dd->tags, 0, sizeof(dd->tags));
3417 	dd->tags.ops = &mtip_mq_ops;
3418 	dd->tags.nr_hw_queues = 1;
3419 	dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS;
3420 	dd->tags.reserved_tags = 1;
3421 	dd->tags.cmd_size = sizeof(struct mtip_cmd);
3422 	dd->tags.numa_node = dd->numa_node;
3423 	dd->tags.flags = BLK_MQ_F_SHOULD_MERGE;
3424 	dd->tags.driver_data = dd;
3425 	dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS;
3426 
3427 	rv = blk_mq_alloc_tag_set(&dd->tags);
3428 	if (rv) {
3429 		dev_err(&dd->pdev->dev,
3430 			"Unable to allocate request queue\n");
3431 		goto block_queue_alloc_tag_error;
3432 	}
3433 
3434 	dd->disk = blk_mq_alloc_disk(&dd->tags, dd);
3435 	if (IS_ERR(dd->disk)) {
3436 		dev_err(&dd->pdev->dev,
3437 			"Unable to allocate request queue\n");
3438 		rv = -ENOMEM;
3439 		goto block_queue_alloc_init_error;
3440 	}
3441 	dd->queue		= dd->disk->queue;
3442 
3443 	rv = ida_alloc(&rssd_index_ida, GFP_KERNEL);
3444 	if (rv < 0)
3445 		goto ida_get_error;
3446 	index = rv;
3447 
3448 	rv = rssd_disk_name_format("rssd",
3449 				index,
3450 				dd->disk->disk_name,
3451 				DISK_NAME_LEN);
3452 	if (rv)
3453 		goto disk_index_error;
3454 
3455 	dd->disk->major		= dd->major;
3456 	dd->disk->first_minor	= index * MTIP_MAX_MINORS;
3457 	dd->disk->minors 	= MTIP_MAX_MINORS;
3458 	dd->disk->fops		= &mtip_block_ops;
3459 	dd->disk->private_data	= dd;
3460 	dd->index		= index;
3461 
3462 	mtip_hw_debugfs_init(dd);
3463 
3464 skip_create_disk:
3465 	/* Initialize the protocol layer. */
3466 	wait_for_rebuild = mtip_hw_get_identify(dd);
3467 	if (wait_for_rebuild < 0) {
3468 		dev_err(&dd->pdev->dev,
3469 			"Protocol layer initialization failed\n");
3470 		rv = -EINVAL;
3471 		goto init_hw_cmds_error;
3472 	}
3473 
3474 	/*
3475 	 * if rebuild pending, start the service thread, and delay the block
3476 	 * queue creation and device_add_disk()
3477 	 */
3478 	if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
3479 		goto start_service_thread;
3480 
3481 	/* Set device limits. */
3482 	blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue);
3483 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue);
3484 	blk_queue_max_segments(dd->queue, MTIP_MAX_SG);
3485 	blk_queue_physical_block_size(dd->queue, 4096);
3486 	blk_queue_max_hw_sectors(dd->queue, 0xffff);
3487 	blk_queue_max_segment_size(dd->queue, 0x400000);
3488 	dma_set_max_seg_size(&dd->pdev->dev, 0x400000);
3489 	blk_queue_io_min(dd->queue, 4096);
3490 
3491 	/* Set the capacity of the device in 512 byte sectors. */
3492 	if (!(mtip_hw_get_capacity(dd, &capacity))) {
3493 		dev_warn(&dd->pdev->dev,
3494 			"Could not read drive capacity\n");
3495 		rv = -EIO;
3496 		goto read_capacity_error;
3497 	}
3498 	set_capacity(dd->disk, capacity);
3499 
3500 	/* Enable the block device and add it to /dev */
3501 	rv = device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups);
3502 	if (rv)
3503 		goto read_capacity_error;
3504 
3505 	if (dd->mtip_svc_handler) {
3506 		set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
3507 		return rv; /* service thread created for handling rebuild */
3508 	}
3509 
3510 start_service_thread:
3511 	dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread,
3512 						dd, dd->numa_node,
3513 						"mtip_svc_thd_%02d", index);
3514 
3515 	if (IS_ERR(dd->mtip_svc_handler)) {
3516 		dev_err(&dd->pdev->dev, "service thread failed to start\n");
3517 		dd->mtip_svc_handler = NULL;
3518 		rv = -EFAULT;
3519 		goto kthread_run_error;
3520 	}
3521 	wake_up_process(dd->mtip_svc_handler);
3522 	if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
3523 		rv = wait_for_rebuild;
3524 
3525 	return rv;
3526 
3527 kthread_run_error:
3528 	/* Delete our gendisk. This also removes the device from /dev */
3529 	del_gendisk(dd->disk);
3530 read_capacity_error:
3531 init_hw_cmds_error:
3532 	mtip_hw_debugfs_exit(dd);
3533 disk_index_error:
3534 	ida_free(&rssd_index_ida, index);
3535 ida_get_error:
3536 	put_disk(dd->disk);
3537 block_queue_alloc_init_error:
3538 	blk_mq_free_tag_set(&dd->tags);
3539 block_queue_alloc_tag_error:
3540 	mtip_hw_exit(dd); /* De-initialize the protocol layer. */
3541 protocol_init_error:
3542 	return rv;
3543 }
3544 
3545 /*
3546  * Function called by the PCI layer when just before the
3547  * machine shuts down.
3548  *
3549  * If a protocol layer shutdown function is present it will be called
3550  * by this function.
3551  *
3552  * @dd Pointer to the driver data structure.
3553  *
3554  * return value
3555  *	0
3556  */
3557 static int mtip_block_shutdown(struct driver_data *dd)
3558 {
3559 	mtip_hw_shutdown(dd);
3560 
3561 	dev_info(&dd->pdev->dev,
3562 		"Shutting down %s ...\n", dd->disk->disk_name);
3563 
3564 	if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3565 		del_gendisk(dd->disk);
3566 
3567 	blk_mq_free_tag_set(&dd->tags);
3568 	put_disk(dd->disk);
3569 	return 0;
3570 }
3571 
3572 static int mtip_block_suspend(struct driver_data *dd)
3573 {
3574 	dev_info(&dd->pdev->dev,
3575 		"Suspending %s ...\n", dd->disk->disk_name);
3576 	mtip_hw_suspend(dd);
3577 	return 0;
3578 }
3579 
3580 static int mtip_block_resume(struct driver_data *dd)
3581 {
3582 	dev_info(&dd->pdev->dev, "Resuming %s ...\n",
3583 		dd->disk->disk_name);
3584 	mtip_hw_resume(dd);
3585 	return 0;
3586 }
3587 
3588 static void drop_cpu(int cpu)
3589 {
3590 	cpu_use[cpu]--;
3591 }
3592 
3593 static int get_least_used_cpu_on_node(int node)
3594 {
3595 	int cpu, least_used_cpu, least_cnt;
3596 	const struct cpumask *node_mask;
3597 
3598 	node_mask = cpumask_of_node(node);
3599 	least_used_cpu = cpumask_first(node_mask);
3600 	least_cnt = cpu_use[least_used_cpu];
3601 	cpu = least_used_cpu;
3602 
3603 	for_each_cpu(cpu, node_mask) {
3604 		if (cpu_use[cpu] < least_cnt) {
3605 			least_used_cpu = cpu;
3606 			least_cnt = cpu_use[cpu];
3607 		}
3608 	}
3609 	cpu_use[least_used_cpu]++;
3610 	return least_used_cpu;
3611 }
3612 
3613 /* Helper for selecting a node in round robin mode */
3614 static inline int mtip_get_next_rr_node(void)
3615 {
3616 	static int next_node = NUMA_NO_NODE;
3617 
3618 	if (next_node == NUMA_NO_NODE) {
3619 		next_node = first_online_node;
3620 		return next_node;
3621 	}
3622 
3623 	next_node = next_online_node(next_node);
3624 	if (next_node == MAX_NUMNODES)
3625 		next_node = first_online_node;
3626 	return next_node;
3627 }
3628 
3629 static DEFINE_HANDLER(0);
3630 static DEFINE_HANDLER(1);
3631 static DEFINE_HANDLER(2);
3632 static DEFINE_HANDLER(3);
3633 static DEFINE_HANDLER(4);
3634 static DEFINE_HANDLER(5);
3635 static DEFINE_HANDLER(6);
3636 static DEFINE_HANDLER(7);
3637 
3638 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev)
3639 {
3640 	unsigned short pcie_dev_ctrl;
3641 
3642 	if (pci_is_pcie(pdev)) {
3643 		pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &pcie_dev_ctrl);
3644 		if (pcie_dev_ctrl & PCI_EXP_DEVCTL_NOSNOOP_EN ||
3645 		    pcie_dev_ctrl & PCI_EXP_DEVCTL_RELAX_EN) {
3646 			dev_info(&dd->pdev->dev,
3647 				"Disabling ERO/No-Snoop on bridge device %04x:%04x\n",
3648 					pdev->vendor, pdev->device);
3649 			pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN |
3650 						PCI_EXP_DEVCTL_RELAX_EN);
3651 			pcie_capability_write_word(pdev, PCI_EXP_DEVCTL,
3652 				pcie_dev_ctrl);
3653 		}
3654 	}
3655 }
3656 
3657 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev)
3658 {
3659 	/*
3660 	 * This workaround is specific to AMD/ATI chipset with a PCI upstream
3661 	 * device with device id 0x5aXX
3662 	 */
3663 	if (pdev->bus && pdev->bus->self) {
3664 		if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI &&
3665 		    ((pdev->bus->self->device & 0xff00) == 0x5a00)) {
3666 			mtip_disable_link_opts(dd, pdev->bus->self);
3667 		} else {
3668 			/* Check further up the topology */
3669 			struct pci_dev *parent_dev = pdev->bus->self;
3670 			if (parent_dev->bus &&
3671 				parent_dev->bus->parent &&
3672 				parent_dev->bus->parent->self &&
3673 				parent_dev->bus->parent->self->vendor ==
3674 					 PCI_VENDOR_ID_ATI &&
3675 				(parent_dev->bus->parent->self->device &
3676 					0xff00) == 0x5a00) {
3677 				mtip_disable_link_opts(dd,
3678 					parent_dev->bus->parent->self);
3679 			}
3680 		}
3681 	}
3682 }
3683 
3684 /*
3685  * Called for each supported PCI device detected.
3686  *
3687  * This function allocates the private data structure, enables the
3688  * PCI device and then calls the block layer initialization function.
3689  *
3690  * return value
3691  *	0 on success else an error code.
3692  */
3693 static int mtip_pci_probe(struct pci_dev *pdev,
3694 			const struct pci_device_id *ent)
3695 {
3696 	int rv = 0;
3697 	struct driver_data *dd = NULL;
3698 	char cpu_list[256];
3699 	const struct cpumask *node_mask;
3700 	int cpu, i = 0, j = 0;
3701 	int my_node = NUMA_NO_NODE;
3702 
3703 	/* Allocate memory for this devices private data. */
3704 	my_node = pcibus_to_node(pdev->bus);
3705 	if (my_node != NUMA_NO_NODE) {
3706 		if (!node_online(my_node))
3707 			my_node = mtip_get_next_rr_node();
3708 	} else {
3709 		dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n");
3710 		my_node = mtip_get_next_rr_node();
3711 	}
3712 	dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n",
3713 		my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev),
3714 		cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id());
3715 
3716 	dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node);
3717 	if (!dd)
3718 		return -ENOMEM;
3719 
3720 	/* Attach the private data to this PCI device.  */
3721 	pci_set_drvdata(pdev, dd);
3722 
3723 	rv = pcim_enable_device(pdev);
3724 	if (rv < 0) {
3725 		dev_err(&pdev->dev, "Unable to enable device\n");
3726 		goto iomap_err;
3727 	}
3728 
3729 	/* Map BAR5 to memory. */
3730 	rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME);
3731 	if (rv < 0) {
3732 		dev_err(&pdev->dev, "Unable to map regions\n");
3733 		goto iomap_err;
3734 	}
3735 
3736 	rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3737 	if (rv) {
3738 		dev_warn(&pdev->dev, "64-bit DMA enable failed\n");
3739 		goto setmask_err;
3740 	}
3741 
3742 	/* Copy the info we may need later into the private data structure. */
3743 	dd->major	= mtip_major;
3744 	dd->instance	= instance;
3745 	dd->pdev	= pdev;
3746 	dd->numa_node	= my_node;
3747 
3748 	memset(dd->workq_name, 0, 32);
3749 	snprintf(dd->workq_name, 31, "mtipq%d", dd->instance);
3750 
3751 	dd->isr_workq = create_workqueue(dd->workq_name);
3752 	if (!dd->isr_workq) {
3753 		dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance);
3754 		rv = -ENOMEM;
3755 		goto setmask_err;
3756 	}
3757 
3758 	memset(cpu_list, 0, sizeof(cpu_list));
3759 
3760 	node_mask = cpumask_of_node(dd->numa_node);
3761 	if (!cpumask_empty(node_mask)) {
3762 		for_each_cpu(cpu, node_mask)
3763 		{
3764 			snprintf(&cpu_list[j], 256 - j, "%d ", cpu);
3765 			j = strlen(cpu_list);
3766 		}
3767 
3768 		dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n",
3769 			dd->numa_node,
3770 			topology_physical_package_id(cpumask_first(node_mask)),
3771 			nr_cpus_node(dd->numa_node),
3772 			cpu_list);
3773 	} else
3774 		dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n");
3775 
3776 	dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node);
3777 	dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n",
3778 		cpu_to_node(dd->isr_binding), dd->isr_binding);
3779 
3780 	/* first worker context always runs in ISR */
3781 	dd->work[0].cpu_binding = dd->isr_binding;
3782 	dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
3783 	dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
3784 	dd->work[3].cpu_binding = dd->work[0].cpu_binding;
3785 	dd->work[4].cpu_binding = dd->work[1].cpu_binding;
3786 	dd->work[5].cpu_binding = dd->work[2].cpu_binding;
3787 	dd->work[6].cpu_binding = dd->work[2].cpu_binding;
3788 	dd->work[7].cpu_binding = dd->work[1].cpu_binding;
3789 
3790 	/* Log the bindings */
3791 	for_each_present_cpu(cpu) {
3792 		memset(cpu_list, 0, sizeof(cpu_list));
3793 		for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) {
3794 			if (dd->work[i].cpu_binding == cpu) {
3795 				snprintf(&cpu_list[j], 256 - j, "%d ", i);
3796 				j = strlen(cpu_list);
3797 			}
3798 		}
3799 		if (j)
3800 			dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list);
3801 	}
3802 
3803 	INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0);
3804 	INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1);
3805 	INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2);
3806 	INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3);
3807 	INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4);
3808 	INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5);
3809 	INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6);
3810 	INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7);
3811 
3812 	pci_set_master(pdev);
3813 	rv = pci_enable_msi(pdev);
3814 	if (rv) {
3815 		dev_warn(&pdev->dev,
3816 			"Unable to enable MSI interrupt.\n");
3817 		goto msi_initialize_err;
3818 	}
3819 
3820 	mtip_fix_ero_nosnoop(dd, pdev);
3821 
3822 	/* Initialize the block layer. */
3823 	rv = mtip_block_initialize(dd);
3824 	if (rv < 0) {
3825 		dev_err(&pdev->dev,
3826 			"Unable to initialize block layer\n");
3827 		goto block_initialize_err;
3828 	}
3829 
3830 	/*
3831 	 * Increment the instance count so that each device has a unique
3832 	 * instance number.
3833 	 */
3834 	instance++;
3835 	if (rv != MTIP_FTL_REBUILD_MAGIC)
3836 		set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
3837 	else
3838 		rv = 0; /* device in rebuild state, return 0 from probe */
3839 
3840 	goto done;
3841 
3842 block_initialize_err:
3843 	pci_disable_msi(pdev);
3844 
3845 msi_initialize_err:
3846 	if (dd->isr_workq) {
3847 		destroy_workqueue(dd->isr_workq);
3848 		drop_cpu(dd->work[0].cpu_binding);
3849 		drop_cpu(dd->work[1].cpu_binding);
3850 		drop_cpu(dd->work[2].cpu_binding);
3851 	}
3852 setmask_err:
3853 	pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
3854 
3855 iomap_err:
3856 	kfree(dd);
3857 	pci_set_drvdata(pdev, NULL);
3858 	return rv;
3859 done:
3860 	return rv;
3861 }
3862 
3863 /*
3864  * Called for each probed device when the device is removed or the
3865  * driver is unloaded.
3866  *
3867  * return value
3868  *	None
3869  */
3870 static void mtip_pci_remove(struct pci_dev *pdev)
3871 {
3872 	struct driver_data *dd = pci_get_drvdata(pdev);
3873 	unsigned long to;
3874 
3875 	mtip_check_surprise_removal(dd);
3876 	synchronize_irq(dd->pdev->irq);
3877 
3878 	/* Spin until workers are done */
3879 	to = jiffies + msecs_to_jiffies(4000);
3880 	do {
3881 		msleep(20);
3882 	} while (atomic_read(&dd->irq_workers_active) != 0 &&
3883 		time_before(jiffies, to));
3884 
3885 	if (atomic_read(&dd->irq_workers_active) != 0) {
3886 		dev_warn(&dd->pdev->dev,
3887 			"Completion workers still active!\n");
3888 	}
3889 
3890 	set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag);
3891 
3892 	if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3893 		del_gendisk(dd->disk);
3894 
3895 	mtip_hw_debugfs_exit(dd);
3896 
3897 	if (dd->mtip_svc_handler) {
3898 		set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
3899 		wake_up_interruptible(&dd->port->svc_wait);
3900 		kthread_stop(dd->mtip_svc_handler);
3901 	}
3902 
3903 	if (!dd->sr) {
3904 		/*
3905 		 * Explicitly wait here for IOs to quiesce,
3906 		 * as mtip_standby_drive usually won't wait for IOs.
3907 		 */
3908 		if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS))
3909 			mtip_standby_drive(dd);
3910 	}
3911 	else
3912 		dev_info(&dd->pdev->dev, "device %s surprise removal\n",
3913 						dd->disk->disk_name);
3914 
3915 	blk_mq_free_tag_set(&dd->tags);
3916 
3917 	/* De-initialize the protocol layer. */
3918 	mtip_hw_exit(dd);
3919 
3920 	if (dd->isr_workq) {
3921 		destroy_workqueue(dd->isr_workq);
3922 		drop_cpu(dd->work[0].cpu_binding);
3923 		drop_cpu(dd->work[1].cpu_binding);
3924 		drop_cpu(dd->work[2].cpu_binding);
3925 	}
3926 
3927 	pci_disable_msi(pdev);
3928 
3929 	pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
3930 	pci_set_drvdata(pdev, NULL);
3931 
3932 	put_disk(dd->disk);
3933 }
3934 
3935 /*
3936  * Called for each probed device when the device is suspended.
3937  *
3938  * return value
3939  *	0  Success
3940  *	<0 Error
3941  */
3942 static int __maybe_unused mtip_pci_suspend(struct device *dev)
3943 {
3944 	int rv = 0;
3945 	struct driver_data *dd = dev_get_drvdata(dev);
3946 
3947 	set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
3948 
3949 	/* Disable ports & interrupts then send standby immediate */
3950 	rv = mtip_block_suspend(dd);
3951 	if (rv < 0)
3952 		dev_err(dev, "Failed to suspend controller\n");
3953 
3954 	return rv;
3955 }
3956 
3957 /*
3958  * Called for each probed device when the device is resumed.
3959  *
3960  * return value
3961  *      0  Success
3962  *      <0 Error
3963  */
3964 static int __maybe_unused mtip_pci_resume(struct device *dev)
3965 {
3966 	int rv = 0;
3967 	struct driver_data *dd = dev_get_drvdata(dev);
3968 
3969 	/*
3970 	 * Calls hbaReset, initPort, & startPort function
3971 	 * then enables interrupts
3972 	 */
3973 	rv = mtip_block_resume(dd);
3974 	if (rv < 0)
3975 		dev_err(dev, "Unable to resume\n");
3976 
3977 	clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
3978 
3979 	return rv;
3980 }
3981 
3982 /*
3983  * Shutdown routine
3984  *
3985  * return value
3986  *      None
3987  */
3988 static void mtip_pci_shutdown(struct pci_dev *pdev)
3989 {
3990 	struct driver_data *dd = pci_get_drvdata(pdev);
3991 	if (dd)
3992 		mtip_block_shutdown(dd);
3993 }
3994 
3995 /* Table of device ids supported by this driver. */
3996 static const struct pci_device_id mtip_pci_tbl[] = {
3997 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) },
3998 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) },
3999 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) },
4000 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) },
4001 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) },
4002 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) },
4003 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) },
4004 	{ 0 }
4005 };
4006 
4007 static SIMPLE_DEV_PM_OPS(mtip_pci_pm_ops, mtip_pci_suspend, mtip_pci_resume);
4008 
4009 /* Structure that describes the PCI driver functions. */
4010 static struct pci_driver mtip_pci_driver = {
4011 	.name			= MTIP_DRV_NAME,
4012 	.id_table		= mtip_pci_tbl,
4013 	.probe			= mtip_pci_probe,
4014 	.remove			= mtip_pci_remove,
4015 	.driver.pm		= &mtip_pci_pm_ops,
4016 	.shutdown		= mtip_pci_shutdown,
4017 };
4018 
4019 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl);
4020 
4021 /*
4022  * Module initialization function.
4023  *
4024  * Called once when the module is loaded. This function allocates a major
4025  * block device number to the Cyclone devices and registers the PCI layer
4026  * of the driver.
4027  *
4028  * Return value
4029  *      0 on success else error code.
4030  */
4031 static int __init mtip_init(void)
4032 {
4033 	int error;
4034 
4035 	pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n");
4036 
4037 	/* Allocate a major block device number to use with this driver. */
4038 	error = register_blkdev(0, MTIP_DRV_NAME);
4039 	if (error <= 0) {
4040 		pr_err("Unable to register block device (%d)\n",
4041 		error);
4042 		return -EBUSY;
4043 	}
4044 	mtip_major = error;
4045 
4046 	dfs_parent = debugfs_create_dir("rssd", NULL);
4047 	if (IS_ERR_OR_NULL(dfs_parent)) {
4048 		pr_warn("Error creating debugfs parent\n");
4049 		dfs_parent = NULL;
4050 	}
4051 
4052 	/* Register our PCI operations. */
4053 	error = pci_register_driver(&mtip_pci_driver);
4054 	if (error) {
4055 		debugfs_remove(dfs_parent);
4056 		unregister_blkdev(mtip_major, MTIP_DRV_NAME);
4057 	}
4058 
4059 	return error;
4060 }
4061 
4062 /*
4063  * Module de-initialization function.
4064  *
4065  * Called once when the module is unloaded. This function deallocates
4066  * the major block device number allocated by mtip_init() and
4067  * unregisters the PCI layer of the driver.
4068  *
4069  * Return value
4070  *      none
4071  */
4072 static void __exit mtip_exit(void)
4073 {
4074 	/* Release the allocated major block device number. */
4075 	unregister_blkdev(mtip_major, MTIP_DRV_NAME);
4076 
4077 	/* Unregister the PCI driver. */
4078 	pci_unregister_driver(&mtip_pci_driver);
4079 
4080 	debugfs_remove_recursive(dfs_parent);
4081 }
4082 
4083 MODULE_AUTHOR("Micron Technology, Inc");
4084 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver");
4085 MODULE_LICENSE("GPL");
4086 MODULE_VERSION(MTIP_DRV_VERSION);
4087 
4088 module_init(mtip_init);
4089 module_exit(mtip_exit);
4090