1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Driver for the Micron P320 SSD
4  *   Copyright (C) 2011 Micron Technology, Inc.
5  *
6  * Portions of this code were derived from works subjected to the
7  * following copyright:
8  *    Copyright (C) 2009 Integrated Device Technology, Inc.
9  */
10 
11 #include <linux/pci.h>
12 #include <linux/interrupt.h>
13 #include <linux/ata.h>
14 #include <linux/delay.h>
15 #include <linux/hdreg.h>
16 #include <linux/uaccess.h>
17 #include <linux/random.h>
18 #include <linux/smp.h>
19 #include <linux/compat.h>
20 #include <linux/fs.h>
21 #include <linux/module.h>
22 #include <linux/blkdev.h>
23 #include <linux/blk-mq.h>
24 #include <linux/bio.h>
25 #include <linux/dma-mapping.h>
26 #include <linux/idr.h>
27 #include <linux/kthread.h>
28 #include <../drivers/ata/ahci.h>
29 #include <linux/export.h>
30 #include <linux/debugfs.h>
31 #include <linux/prefetch.h>
32 #include <linux/numa.h>
33 #include "mtip32xx.h"
34 
35 #define HW_CMD_SLOT_SZ		(MTIP_MAX_COMMAND_SLOTS * 32)
36 
37 /* DMA region containing RX Fis, Identify, RLE10, and SMART buffers */
38 #define AHCI_RX_FIS_SZ          0x100
39 #define AHCI_RX_FIS_OFFSET      0x0
40 #define AHCI_IDFY_SZ            ATA_SECT_SIZE
41 #define AHCI_IDFY_OFFSET        0x400
42 #define AHCI_SECTBUF_SZ         ATA_SECT_SIZE
43 #define AHCI_SECTBUF_OFFSET     0x800
44 #define AHCI_SMARTBUF_SZ        ATA_SECT_SIZE
45 #define AHCI_SMARTBUF_OFFSET    0xC00
46 /* 0x100 + 0x200 + 0x200 + 0x200 is smaller than 4k but we pad it out */
47 #define BLOCK_DMA_ALLOC_SZ      4096
48 
49 /* DMA region containing command table (should be 8192 bytes) */
50 #define AHCI_CMD_SLOT_SZ        sizeof(struct mtip_cmd_hdr)
51 #define AHCI_CMD_TBL_SZ         (MTIP_MAX_COMMAND_SLOTS * AHCI_CMD_SLOT_SZ)
52 #define AHCI_CMD_TBL_OFFSET     0x0
53 
54 /* DMA region per command (contains header and SGL) */
55 #define AHCI_CMD_TBL_HDR_SZ     0x80
56 #define AHCI_CMD_TBL_HDR_OFFSET 0x0
57 #define AHCI_CMD_TBL_SGL_SZ     (MTIP_MAX_SG * sizeof(struct mtip_cmd_sg))
58 #define AHCI_CMD_TBL_SGL_OFFSET AHCI_CMD_TBL_HDR_SZ
59 #define CMD_DMA_ALLOC_SZ        (AHCI_CMD_TBL_SGL_SZ + AHCI_CMD_TBL_HDR_SZ)
60 
61 
62 #define HOST_CAP_NZDMA		(1 << 19)
63 #define HOST_HSORG		0xFC
64 #define HSORG_DISABLE_SLOTGRP_INTR (1<<24)
65 #define HSORG_DISABLE_SLOTGRP_PXIS (1<<16)
66 #define HSORG_HWREV		0xFF00
67 #define HSORG_STYLE		0x8
68 #define HSORG_SLOTGROUPS	0x7
69 
70 #define PORT_COMMAND_ISSUE	0x38
71 #define PORT_SDBV		0x7C
72 
73 #define PORT_OFFSET		0x100
74 #define PORT_MEM_SIZE		0x80
75 
76 #define PORT_IRQ_ERR \
77 	(PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR | PORT_IRQ_CONNECT | \
78 	 PORT_IRQ_PHYRDY | PORT_IRQ_UNK_FIS | PORT_IRQ_BAD_PMP | \
79 	 PORT_IRQ_TF_ERR | PORT_IRQ_HBUS_DATA_ERR | PORT_IRQ_IF_NONFATAL | \
80 	 PORT_IRQ_OVERFLOW)
81 #define PORT_IRQ_LEGACY \
82 	(PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
83 #define PORT_IRQ_HANDLED \
84 	(PORT_IRQ_SDB_FIS | PORT_IRQ_LEGACY | \
85 	 PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR | \
86 	 PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)
87 #define DEF_PORT_IRQ \
88 	(PORT_IRQ_ERR | PORT_IRQ_LEGACY | PORT_IRQ_SDB_FIS)
89 
90 /* product numbers */
91 #define MTIP_PRODUCT_UNKNOWN	0x00
92 #define MTIP_PRODUCT_ASICFPGA	0x11
93 
94 /* Device instance number, incremented each time a device is probed. */
95 static int instance;
96 
97 static LIST_HEAD(online_list);
98 static LIST_HEAD(removing_list);
99 static DEFINE_SPINLOCK(dev_lock);
100 
101 /*
102  * Global variable used to hold the major block device number
103  * allocated in mtip_init().
104  */
105 static int mtip_major;
106 static struct dentry *dfs_parent;
107 static struct dentry *dfs_device_status;
108 
109 static u32 cpu_use[NR_CPUS];
110 
111 static DEFINE_IDA(rssd_index_ida);
112 
113 static int mtip_block_initialize(struct driver_data *dd);
114 
115 #ifdef CONFIG_COMPAT
116 struct mtip_compat_ide_task_request_s {
117 	__u8		io_ports[8];
118 	__u8		hob_ports[8];
119 	ide_reg_valid_t	out_flags;
120 	ide_reg_valid_t	in_flags;
121 	int		data_phase;
122 	int		req_cmd;
123 	compat_ulong_t	out_size;
124 	compat_ulong_t	in_size;
125 };
126 #endif
127 
128 /*
129  * This function check_for_surprise_removal is called
130  * while card is removed from the system and it will
131  * read the vendor id from the configuration space
132  *
133  * @pdev Pointer to the pci_dev structure.
134  *
135  * return value
136  *	 true if device removed, else false
137  */
138 static bool mtip_check_surprise_removal(struct driver_data *dd)
139 {
140 	u16 vendor_id = 0;
141 
142 	if (dd->sr)
143 		return true;
144 
145        /* Read the vendorID from the configuration space */
146 	pci_read_config_word(dd->pdev, 0x00, &vendor_id);
147 	if (vendor_id == 0xFFFF) {
148 		dd->sr = true;
149 		if (dd->queue)
150 			blk_queue_flag_set(QUEUE_FLAG_DEAD, dd->queue);
151 		else
152 			dev_warn(&dd->pdev->dev,
153 				"%s: dd->queue is NULL\n", __func__);
154 		return true; /* device removed */
155 	}
156 
157 	return false; /* device present */
158 }
159 
160 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd,
161 					  unsigned int tag)
162 {
163 	return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(dd->tags.tags[0], tag));
164 }
165 
166 /*
167  * Reset the HBA (without sleeping)
168  *
169  * @dd Pointer to the driver data structure.
170  *
171  * return value
172  *	0	The reset was successful.
173  *	-1	The HBA Reset bit did not clear.
174  */
175 static int mtip_hba_reset(struct driver_data *dd)
176 {
177 	unsigned long timeout;
178 
179 	/* Set the reset bit */
180 	writel(HOST_RESET, dd->mmio + HOST_CTL);
181 
182 	/* Flush */
183 	readl(dd->mmio + HOST_CTL);
184 
185 	/*
186 	 * Spin for up to 10 seconds waiting for reset acknowledgement. Spec
187 	 * is 1 sec but in LUN failure conditions, up to 10 secs are required
188 	 */
189 	timeout = jiffies + msecs_to_jiffies(10000);
190 	do {
191 		mdelay(10);
192 		if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
193 			return -1;
194 
195 	} while ((readl(dd->mmio + HOST_CTL) & HOST_RESET)
196 		 && time_before(jiffies, timeout));
197 
198 	if (readl(dd->mmio + HOST_CTL) & HOST_RESET)
199 		return -1;
200 
201 	return 0;
202 }
203 
204 /*
205  * Issue a command to the hardware.
206  *
207  * Set the appropriate bit in the s_active and Command Issue hardware
208  * registers, causing hardware command processing to begin.
209  *
210  * @port Pointer to the port structure.
211  * @tag  The tag of the command to be issued.
212  *
213  * return value
214  *      None
215  */
216 static inline void mtip_issue_ncq_command(struct mtip_port *port, int tag)
217 {
218 	int group = tag >> 5;
219 
220 	/* guard SACT and CI registers */
221 	spin_lock(&port->cmd_issue_lock[group]);
222 	writel((1 << MTIP_TAG_BIT(tag)),
223 			port->s_active[MTIP_TAG_INDEX(tag)]);
224 	writel((1 << MTIP_TAG_BIT(tag)),
225 			port->cmd_issue[MTIP_TAG_INDEX(tag)]);
226 	spin_unlock(&port->cmd_issue_lock[group]);
227 }
228 
229 /*
230  * Enable/disable the reception of FIS
231  *
232  * @port   Pointer to the port data structure
233  * @enable 1 to enable, 0 to disable
234  *
235  * return value
236  *	Previous state: 1 enabled, 0 disabled
237  */
238 static int mtip_enable_fis(struct mtip_port *port, int enable)
239 {
240 	u32 tmp;
241 
242 	/* enable FIS reception */
243 	tmp = readl(port->mmio + PORT_CMD);
244 	if (enable)
245 		writel(tmp | PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
246 	else
247 		writel(tmp & ~PORT_CMD_FIS_RX, port->mmio + PORT_CMD);
248 
249 	/* Flush */
250 	readl(port->mmio + PORT_CMD);
251 
252 	return (((tmp & PORT_CMD_FIS_RX) == PORT_CMD_FIS_RX));
253 }
254 
255 /*
256  * Enable/disable the DMA engine
257  *
258  * @port   Pointer to the port data structure
259  * @enable 1 to enable, 0 to disable
260  *
261  * return value
262  *	Previous state: 1 enabled, 0 disabled.
263  */
264 static int mtip_enable_engine(struct mtip_port *port, int enable)
265 {
266 	u32 tmp;
267 
268 	/* enable FIS reception */
269 	tmp = readl(port->mmio + PORT_CMD);
270 	if (enable)
271 		writel(tmp | PORT_CMD_START, port->mmio + PORT_CMD);
272 	else
273 		writel(tmp & ~PORT_CMD_START, port->mmio + PORT_CMD);
274 
275 	readl(port->mmio + PORT_CMD);
276 	return (((tmp & PORT_CMD_START) == PORT_CMD_START));
277 }
278 
279 /*
280  * Enables the port DMA engine and FIS reception.
281  *
282  * return value
283  *	None
284  */
285 static inline void mtip_start_port(struct mtip_port *port)
286 {
287 	/* Enable FIS reception */
288 	mtip_enable_fis(port, 1);
289 
290 	/* Enable the DMA engine */
291 	mtip_enable_engine(port, 1);
292 }
293 
294 /*
295  * Deinitialize a port by disabling port interrupts, the DMA engine,
296  * and FIS reception.
297  *
298  * @port Pointer to the port structure
299  *
300  * return value
301  *	None
302  */
303 static inline void mtip_deinit_port(struct mtip_port *port)
304 {
305 	/* Disable interrupts on this port */
306 	writel(0, port->mmio + PORT_IRQ_MASK);
307 
308 	/* Disable the DMA engine */
309 	mtip_enable_engine(port, 0);
310 
311 	/* Disable FIS reception */
312 	mtip_enable_fis(port, 0);
313 }
314 
315 /*
316  * Initialize a port.
317  *
318  * This function deinitializes the port by calling mtip_deinit_port() and
319  * then initializes it by setting the command header and RX FIS addresses,
320  * clearing the SError register and any pending port interrupts before
321  * re-enabling the default set of port interrupts.
322  *
323  * @port Pointer to the port structure.
324  *
325  * return value
326  *	None
327  */
328 static void mtip_init_port(struct mtip_port *port)
329 {
330 	int i;
331 	mtip_deinit_port(port);
332 
333 	/* Program the command list base and FIS base addresses */
334 	if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) {
335 		writel((port->command_list_dma >> 16) >> 16,
336 			 port->mmio + PORT_LST_ADDR_HI);
337 		writel((port->rxfis_dma >> 16) >> 16,
338 			 port->mmio + PORT_FIS_ADDR_HI);
339 		set_bit(MTIP_PF_HOST_CAP_64, &port->flags);
340 	}
341 
342 	writel(port->command_list_dma & 0xFFFFFFFF,
343 			port->mmio + PORT_LST_ADDR);
344 	writel(port->rxfis_dma & 0xFFFFFFFF, port->mmio + PORT_FIS_ADDR);
345 
346 	/* Clear SError */
347 	writel(readl(port->mmio + PORT_SCR_ERR), port->mmio + PORT_SCR_ERR);
348 
349 	/* reset the completed registers.*/
350 	for (i = 0; i < port->dd->slot_groups; i++)
351 		writel(0xFFFFFFFF, port->completed[i]);
352 
353 	/* Clear any pending interrupts for this port */
354 	writel(readl(port->mmio + PORT_IRQ_STAT), port->mmio + PORT_IRQ_STAT);
355 
356 	/* Clear any pending interrupts on the HBA. */
357 	writel(readl(port->dd->mmio + HOST_IRQ_STAT),
358 					port->dd->mmio + HOST_IRQ_STAT);
359 
360 	/* Enable port interrupts */
361 	writel(DEF_PORT_IRQ, port->mmio + PORT_IRQ_MASK);
362 }
363 
364 /*
365  * Restart a port
366  *
367  * @port Pointer to the port data structure.
368  *
369  * return value
370  *	None
371  */
372 static void mtip_restart_port(struct mtip_port *port)
373 {
374 	unsigned long timeout;
375 
376 	/* Disable the DMA engine */
377 	mtip_enable_engine(port, 0);
378 
379 	/* Chip quirk: wait up to 500ms for PxCMD.CR == 0 */
380 	timeout = jiffies + msecs_to_jiffies(500);
381 	while ((readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON)
382 		 && time_before(jiffies, timeout))
383 		;
384 
385 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
386 		return;
387 
388 	/*
389 	 * Chip quirk: escalate to hba reset if
390 	 * PxCMD.CR not clear after 500 ms
391 	 */
392 	if (readl(port->mmio + PORT_CMD) & PORT_CMD_LIST_ON) {
393 		dev_warn(&port->dd->pdev->dev,
394 			"PxCMD.CR not clear, escalating reset\n");
395 
396 		if (mtip_hba_reset(port->dd))
397 			dev_err(&port->dd->pdev->dev,
398 				"HBA reset escalation failed.\n");
399 
400 		/* 30 ms delay before com reset to quiesce chip */
401 		mdelay(30);
402 	}
403 
404 	dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n");
405 
406 	/* Set PxSCTL.DET */
407 	writel(readl(port->mmio + PORT_SCR_CTL) |
408 			 1, port->mmio + PORT_SCR_CTL);
409 	readl(port->mmio + PORT_SCR_CTL);
410 
411 	/* Wait 1 ms to quiesce chip function */
412 	timeout = jiffies + msecs_to_jiffies(1);
413 	while (time_before(jiffies, timeout))
414 		;
415 
416 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
417 		return;
418 
419 	/* Clear PxSCTL.DET */
420 	writel(readl(port->mmio + PORT_SCR_CTL) & ~1,
421 			 port->mmio + PORT_SCR_CTL);
422 	readl(port->mmio + PORT_SCR_CTL);
423 
424 	/* Wait 500 ms for bit 0 of PORT_SCR_STS to be set */
425 	timeout = jiffies + msecs_to_jiffies(500);
426 	while (((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
427 			 && time_before(jiffies, timeout))
428 		;
429 
430 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
431 		return;
432 
433 	if ((readl(port->mmio + PORT_SCR_STAT) & 0x01) == 0)
434 		dev_warn(&port->dd->pdev->dev,
435 			"COM reset failed\n");
436 
437 	mtip_init_port(port);
438 	mtip_start_port(port);
439 
440 }
441 
442 static int mtip_device_reset(struct driver_data *dd)
443 {
444 	int rv = 0;
445 
446 	if (mtip_check_surprise_removal(dd))
447 		return 0;
448 
449 	if (mtip_hba_reset(dd) < 0)
450 		rv = -EFAULT;
451 
452 	mdelay(1);
453 	mtip_init_port(dd->port);
454 	mtip_start_port(dd->port);
455 
456 	/* Enable interrupts on the HBA. */
457 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
458 					dd->mmio + HOST_CTL);
459 	return rv;
460 }
461 
462 /*
463  * Helper function for tag logging
464  */
465 static void print_tags(struct driver_data *dd,
466 			char *msg,
467 			unsigned long *tagbits,
468 			int cnt)
469 {
470 	unsigned char tagmap[128];
471 	int group, tagmap_len = 0;
472 
473 	memset(tagmap, 0, sizeof(tagmap));
474 	for (group = SLOTBITS_IN_LONGS; group > 0; group--)
475 		tagmap_len += sprintf(tagmap + tagmap_len, "%016lX ",
476 						tagbits[group-1]);
477 	dev_warn(&dd->pdev->dev,
478 			"%d command(s) %s: tagmap [%s]", cnt, msg, tagmap);
479 }
480 
481 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
482 				dma_addr_t buffer_dma, unsigned int sectors);
483 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
484 						struct smart_attr *attrib);
485 
486 static void mtip_complete_command(struct mtip_cmd *cmd, blk_status_t status)
487 {
488 	struct request *req = blk_mq_rq_from_pdu(cmd);
489 
490 	cmd->status = status;
491 	if (likely(!blk_should_fake_timeout(req->q)))
492 		blk_mq_complete_request(req);
493 }
494 
495 /*
496  * Handle an error.
497  *
498  * @dd Pointer to the DRIVER_DATA structure.
499  *
500  * return value
501  *	None
502  */
503 static void mtip_handle_tfe(struct driver_data *dd)
504 {
505 	int group, tag, bit, reissue, rv;
506 	struct mtip_port *port;
507 	struct mtip_cmd  *cmd;
508 	u32 completed;
509 	struct host_to_dev_fis *fis;
510 	unsigned long tagaccum[SLOTBITS_IN_LONGS];
511 	unsigned int cmd_cnt = 0;
512 	unsigned char *buf;
513 	char *fail_reason = NULL;
514 	int fail_all_ncq_write = 0, fail_all_ncq_cmds = 0;
515 
516 	dev_warn(&dd->pdev->dev, "Taskfile error\n");
517 
518 	port = dd->port;
519 
520 	if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags)) {
521 		cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
522 		dbg_printk(MTIP_DRV_NAME " TFE for the internal command\n");
523 		mtip_complete_command(cmd, BLK_STS_IOERR);
524 		return;
525 	}
526 
527 	/* clear the tag accumulator */
528 	memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
529 
530 	/* Loop through all the groups */
531 	for (group = 0; group < dd->slot_groups; group++) {
532 		completed = readl(port->completed[group]);
533 
534 		dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed);
535 
536 		/* clear completed status register in the hardware.*/
537 		writel(completed, port->completed[group]);
538 
539 		/* Process successfully completed commands */
540 		for (bit = 0; bit < 32 && completed; bit++) {
541 			if (!(completed & (1<<bit)))
542 				continue;
543 			tag = (group << 5) + bit;
544 
545 			/* Skip the internal command slot */
546 			if (tag == MTIP_TAG_INTERNAL)
547 				continue;
548 
549 			cmd = mtip_cmd_from_tag(dd, tag);
550 			mtip_complete_command(cmd, 0);
551 			set_bit(tag, tagaccum);
552 			cmd_cnt++;
553 		}
554 	}
555 
556 	print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt);
557 
558 	/* Restart the port */
559 	mdelay(20);
560 	mtip_restart_port(port);
561 
562 	/* Trying to determine the cause of the error */
563 	rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
564 				dd->port->log_buf,
565 				dd->port->log_buf_dma, 1);
566 	if (rv) {
567 		dev_warn(&dd->pdev->dev,
568 			"Error in READ LOG EXT (10h) command\n");
569 		/* non-critical error, don't fail the load */
570 	} else {
571 		buf = (unsigned char *)dd->port->log_buf;
572 		if (buf[259] & 0x1) {
573 			dev_info(&dd->pdev->dev,
574 				"Write protect bit is set.\n");
575 			set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
576 			fail_all_ncq_write = 1;
577 			fail_reason = "write protect";
578 		}
579 		if (buf[288] == 0xF7) {
580 			dev_info(&dd->pdev->dev,
581 				"Exceeded Tmax, drive in thermal shutdown.\n");
582 			set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
583 			fail_all_ncq_cmds = 1;
584 			fail_reason = "thermal shutdown";
585 		}
586 		if (buf[288] == 0xBF) {
587 			set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
588 			dev_info(&dd->pdev->dev,
589 				"Drive indicates rebuild has failed. Secure erase required.\n");
590 			fail_all_ncq_cmds = 1;
591 			fail_reason = "rebuild failed";
592 		}
593 	}
594 
595 	/* clear the tag accumulator */
596 	memset(tagaccum, 0, SLOTBITS_IN_LONGS * sizeof(long));
597 
598 	/* Loop through all the groups */
599 	for (group = 0; group < dd->slot_groups; group++) {
600 		for (bit = 0; bit < 32; bit++) {
601 			reissue = 1;
602 			tag = (group << 5) + bit;
603 			cmd = mtip_cmd_from_tag(dd, tag);
604 
605 			fis = (struct host_to_dev_fis *)cmd->command;
606 
607 			/* Should re-issue? */
608 			if (tag == MTIP_TAG_INTERNAL ||
609 			    fis->command == ATA_CMD_SET_FEATURES)
610 				reissue = 0;
611 			else {
612 				if (fail_all_ncq_cmds ||
613 					(fail_all_ncq_write &&
614 					fis->command == ATA_CMD_FPDMA_WRITE)) {
615 					dev_warn(&dd->pdev->dev,
616 					"  Fail: %s w/tag %d [%s].\n",
617 					fis->command == ATA_CMD_FPDMA_WRITE ?
618 						"write" : "read",
619 					tag,
620 					fail_reason != NULL ?
621 						fail_reason : "unknown");
622 					mtip_complete_command(cmd, BLK_STS_MEDIUM);
623 					continue;
624 				}
625 			}
626 
627 			/*
628 			 * First check if this command has
629 			 *  exceeded its retries.
630 			 */
631 			if (reissue && (cmd->retries-- > 0)) {
632 
633 				set_bit(tag, tagaccum);
634 
635 				/* Re-issue the command. */
636 				mtip_issue_ncq_command(port, tag);
637 
638 				continue;
639 			}
640 
641 			/* Retire a command that will not be reissued */
642 			dev_warn(&port->dd->pdev->dev,
643 				"retiring tag %d\n", tag);
644 
645 			mtip_complete_command(cmd, BLK_STS_IOERR);
646 		}
647 	}
648 	print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt);
649 }
650 
651 /*
652  * Handle a set device bits interrupt
653  */
654 static inline void mtip_workq_sdbfx(struct mtip_port *port, int group,
655 							u32 completed)
656 {
657 	struct driver_data *dd = port->dd;
658 	int tag, bit;
659 	struct mtip_cmd *command;
660 
661 	if (!completed) {
662 		WARN_ON_ONCE(!completed);
663 		return;
664 	}
665 	/* clear completed status register in the hardware.*/
666 	writel(completed, port->completed[group]);
667 
668 	/* Process completed commands. */
669 	for (bit = 0; (bit < 32) && completed; bit++) {
670 		if (completed & 0x01) {
671 			tag = (group << 5) | bit;
672 
673 			/* skip internal command slot. */
674 			if (unlikely(tag == MTIP_TAG_INTERNAL))
675 				continue;
676 
677 			command = mtip_cmd_from_tag(dd, tag);
678 			mtip_complete_command(command, 0);
679 		}
680 		completed >>= 1;
681 	}
682 
683 	/* If last, re-enable interrupts */
684 	if (atomic_dec_return(&dd->irq_workers_active) == 0)
685 		writel(0xffffffff, dd->mmio + HOST_IRQ_STAT);
686 }
687 
688 /*
689  * Process legacy pio and d2h interrupts
690  */
691 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat)
692 {
693 	struct mtip_port *port = dd->port;
694 	struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL);
695 
696 	if (test_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags) && cmd) {
697 		int group = MTIP_TAG_INDEX(MTIP_TAG_INTERNAL);
698 		int status = readl(port->cmd_issue[group]);
699 
700 		if (!(status & (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))))
701 			mtip_complete_command(cmd, 0);
702 	}
703 }
704 
705 /*
706  * Demux and handle errors
707  */
708 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat)
709 {
710 	if (unlikely(port_stat & PORT_IRQ_CONNECT)) {
711 		dev_warn(&dd->pdev->dev,
712 			"Clearing PxSERR.DIAG.x\n");
713 		writel((1 << 26), dd->port->mmio + PORT_SCR_ERR);
714 	}
715 
716 	if (unlikely(port_stat & PORT_IRQ_PHYRDY)) {
717 		dev_warn(&dd->pdev->dev,
718 			"Clearing PxSERR.DIAG.n\n");
719 		writel((1 << 16), dd->port->mmio + PORT_SCR_ERR);
720 	}
721 
722 	if (unlikely(port_stat & ~PORT_IRQ_HANDLED)) {
723 		dev_warn(&dd->pdev->dev,
724 			"Port stat errors %x unhandled\n",
725 			(port_stat & ~PORT_IRQ_HANDLED));
726 		if (mtip_check_surprise_removal(dd))
727 			return;
728 	}
729 	if (likely(port_stat & (PORT_IRQ_TF_ERR | PORT_IRQ_IF_ERR))) {
730 		set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags);
731 		wake_up_interruptible(&dd->port->svc_wait);
732 	}
733 }
734 
735 static inline irqreturn_t mtip_handle_irq(struct driver_data *data)
736 {
737 	struct driver_data *dd = (struct driver_data *) data;
738 	struct mtip_port *port = dd->port;
739 	u32 hba_stat, port_stat;
740 	int rv = IRQ_NONE;
741 	int do_irq_enable = 1, i, workers;
742 	struct mtip_work *twork;
743 
744 	hba_stat = readl(dd->mmio + HOST_IRQ_STAT);
745 	if (hba_stat) {
746 		rv = IRQ_HANDLED;
747 
748 		/* Acknowledge the interrupt status on the port.*/
749 		port_stat = readl(port->mmio + PORT_IRQ_STAT);
750 		if (unlikely(port_stat == 0xFFFFFFFF)) {
751 			mtip_check_surprise_removal(dd);
752 			return IRQ_HANDLED;
753 		}
754 		writel(port_stat, port->mmio + PORT_IRQ_STAT);
755 
756 		/* Demux port status */
757 		if (likely(port_stat & PORT_IRQ_SDB_FIS)) {
758 			do_irq_enable = 0;
759 			WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0);
760 
761 			/* Start at 1: group zero is always local? */
762 			for (i = 0, workers = 0; i < MTIP_MAX_SLOT_GROUPS;
763 									i++) {
764 				twork = &dd->work[i];
765 				twork->completed = readl(port->completed[i]);
766 				if (twork->completed)
767 					workers++;
768 			}
769 
770 			atomic_set(&dd->irq_workers_active, workers);
771 			if (workers) {
772 				for (i = 1; i < MTIP_MAX_SLOT_GROUPS; i++) {
773 					twork = &dd->work[i];
774 					if (twork->completed)
775 						queue_work_on(
776 							twork->cpu_binding,
777 							dd->isr_workq,
778 							&twork->work);
779 				}
780 
781 				if (likely(dd->work[0].completed))
782 					mtip_workq_sdbfx(port, 0,
783 							dd->work[0].completed);
784 
785 			} else {
786 				/*
787 				 * Chip quirk: SDB interrupt but nothing
788 				 * to complete
789 				 */
790 				do_irq_enable = 1;
791 			}
792 		}
793 
794 		if (unlikely(port_stat & PORT_IRQ_ERR)) {
795 			if (unlikely(mtip_check_surprise_removal(dd))) {
796 				/* don't proceed further */
797 				return IRQ_HANDLED;
798 			}
799 			if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
800 							&dd->dd_flag))
801 				return rv;
802 
803 			mtip_process_errors(dd, port_stat & PORT_IRQ_ERR);
804 		}
805 
806 		if (unlikely(port_stat & PORT_IRQ_LEGACY))
807 			mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY);
808 	}
809 
810 	/* acknowledge interrupt */
811 	if (unlikely(do_irq_enable))
812 		writel(hba_stat, dd->mmio + HOST_IRQ_STAT);
813 
814 	return rv;
815 }
816 
817 /*
818  * HBA interrupt subroutine.
819  *
820  * @irq		IRQ number.
821  * @instance	Pointer to the driver data structure.
822  *
823  * return value
824  *	IRQ_HANDLED	A HBA interrupt was pending and handled.
825  *	IRQ_NONE	This interrupt was not for the HBA.
826  */
827 static irqreturn_t mtip_irq_handler(int irq, void *instance)
828 {
829 	struct driver_data *dd = instance;
830 
831 	return mtip_handle_irq(dd);
832 }
833 
834 static void mtip_issue_non_ncq_command(struct mtip_port *port, int tag)
835 {
836 	writel(1 << MTIP_TAG_BIT(tag), port->cmd_issue[MTIP_TAG_INDEX(tag)]);
837 }
838 
839 static bool mtip_pause_ncq(struct mtip_port *port,
840 				struct host_to_dev_fis *fis)
841 {
842 	unsigned long task_file_data;
843 
844 	task_file_data = readl(port->mmio+PORT_TFDATA);
845 	if ((task_file_data & 1))
846 		return false;
847 
848 	if (fis->command == ATA_CMD_SEC_ERASE_PREP) {
849 		port->ic_pause_timer = jiffies;
850 		return true;
851 	} else if ((fis->command == ATA_CMD_DOWNLOAD_MICRO) &&
852 					(fis->features == 0x03)) {
853 		set_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
854 		port->ic_pause_timer = jiffies;
855 		return true;
856 	} else if ((fis->command == ATA_CMD_SEC_ERASE_UNIT) ||
857 		((fis->command == 0xFC) &&
858 			(fis->features == 0x27 || fis->features == 0x72 ||
859 			 fis->features == 0x62 || fis->features == 0x26))) {
860 		clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
861 		clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag);
862 		/* Com reset after secure erase or lowlevel format */
863 		mtip_restart_port(port);
864 		clear_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
865 		return false;
866 	}
867 
868 	return false;
869 }
870 
871 static bool mtip_commands_active(struct mtip_port *port)
872 {
873 	unsigned int active;
874 	unsigned int n;
875 
876 	/*
877 	 * Ignore s_active bit 0 of array element 0.
878 	 * This bit will always be set
879 	 */
880 	active = readl(port->s_active[0]) & 0xFFFFFFFE;
881 	for (n = 1; n < port->dd->slot_groups; n++)
882 		active |= readl(port->s_active[n]);
883 
884 	return active != 0;
885 }
886 
887 /*
888  * Wait for port to quiesce
889  *
890  * @port    Pointer to port data structure
891  * @timeout Max duration to wait (ms)
892  *
893  * return value
894  *	0	Success
895  *	-EBUSY  Commands still active
896  */
897 static int mtip_quiesce_io(struct mtip_port *port, unsigned long timeout)
898 {
899 	unsigned long to;
900 	bool active = true;
901 
902 	blk_mq_quiesce_queue(port->dd->queue);
903 
904 	to = jiffies + msecs_to_jiffies(timeout);
905 	do {
906 		if (test_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags) &&
907 			test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
908 			msleep(20);
909 			continue; /* svc thd is actively issuing commands */
910 		}
911 
912 		msleep(100);
913 
914 		if (mtip_check_surprise_removal(port->dd))
915 			goto err_fault;
916 
917 		active = mtip_commands_active(port);
918 		if (!active)
919 			break;
920 	} while (time_before(jiffies, to));
921 
922 	blk_mq_unquiesce_queue(port->dd->queue);
923 	return active ? -EBUSY : 0;
924 err_fault:
925 	blk_mq_unquiesce_queue(port->dd->queue);
926 	return -EFAULT;
927 }
928 
929 struct mtip_int_cmd {
930 	int fis_len;
931 	dma_addr_t buffer;
932 	int buf_len;
933 	u32 opts;
934 };
935 
936 /*
937  * Execute an internal command and wait for the completion.
938  *
939  * @port    Pointer to the port data structure.
940  * @fis     Pointer to the FIS that describes the command.
941  * @fis_len  Length in WORDS of the FIS.
942  * @buffer  DMA accessible for command data.
943  * @buf_len  Length, in bytes, of the data buffer.
944  * @opts    Command header options, excluding the FIS length
945  *             and the number of PRD entries.
946  * @timeout Time in ms to wait for the command to complete.
947  *
948  * return value
949  *	0	 Command completed successfully.
950  *	-EFAULT  The buffer address is not correctly aligned.
951  *	-EBUSY   Internal command or other IO in progress.
952  *	-EAGAIN  Time out waiting for command to complete.
953  */
954 static int mtip_exec_internal_command(struct mtip_port *port,
955 					struct host_to_dev_fis *fis,
956 					int fis_len,
957 					dma_addr_t buffer,
958 					int buf_len,
959 					u32 opts,
960 					unsigned long timeout)
961 {
962 	struct mtip_cmd *int_cmd;
963 	struct driver_data *dd = port->dd;
964 	struct request *rq;
965 	struct mtip_int_cmd icmd = {
966 		.fis_len = fis_len,
967 		.buffer = buffer,
968 		.buf_len = buf_len,
969 		.opts = opts
970 	};
971 	int rv = 0;
972 
973 	/* Make sure the buffer is 8 byte aligned. This is asic specific. */
974 	if (buffer & 0x00000007) {
975 		dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n");
976 		return -EFAULT;
977 	}
978 
979 	if (mtip_check_surprise_removal(dd))
980 		return -EFAULT;
981 
982 	rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED);
983 	if (IS_ERR(rq)) {
984 		dbg_printk(MTIP_DRV_NAME "Unable to allocate tag for PIO cmd\n");
985 		return -EFAULT;
986 	}
987 
988 	set_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
989 
990 	if (fis->command == ATA_CMD_SEC_ERASE_PREP)
991 		set_bit(MTIP_PF_SE_ACTIVE_BIT, &port->flags);
992 
993 	clear_bit(MTIP_PF_DM_ACTIVE_BIT, &port->flags);
994 
995 	if (fis->command != ATA_CMD_STANDBYNOW1) {
996 		/* wait for io to complete if non atomic */
997 		if (mtip_quiesce_io(port, MTIP_QUIESCE_IO_TIMEOUT_MS) < 0) {
998 			dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n");
999 			blk_mq_free_request(rq);
1000 			clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
1001 			wake_up_interruptible(&port->svc_wait);
1002 			return -EBUSY;
1003 		}
1004 	}
1005 
1006 	/* Copy the command to the command table */
1007 	int_cmd = blk_mq_rq_to_pdu(rq);
1008 	int_cmd->icmd = &icmd;
1009 	memcpy(int_cmd->command, fis, fis_len*4);
1010 
1011 	rq->timeout = timeout;
1012 
1013 	/* insert request and run queue */
1014 	blk_execute_rq(rq, true);
1015 
1016 	if (int_cmd->status) {
1017 		dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n",
1018 				fis->command, int_cmd->status);
1019 		rv = -EIO;
1020 
1021 		if (mtip_check_surprise_removal(dd) ||
1022 			test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
1023 					&dd->dd_flag)) {
1024 			dev_err(&dd->pdev->dev,
1025 				"Internal command [%02X] wait returned due to SR\n",
1026 				fis->command);
1027 			rv = -ENXIO;
1028 			goto exec_ic_exit;
1029 		}
1030 		mtip_device_reset(dd); /* recover from timeout issue */
1031 		rv = -EAGAIN;
1032 		goto exec_ic_exit;
1033 	}
1034 
1035 	if (readl(port->cmd_issue[MTIP_TAG_INDEX(MTIP_TAG_INTERNAL)])
1036 			& (1 << MTIP_TAG_BIT(MTIP_TAG_INTERNAL))) {
1037 		rv = -ENXIO;
1038 		if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
1039 			mtip_device_reset(dd);
1040 			rv = -EAGAIN;
1041 		}
1042 	}
1043 exec_ic_exit:
1044 	/* Clear the allocated and active bits for the internal command. */
1045 	blk_mq_free_request(rq);
1046 	clear_bit(MTIP_PF_IC_ACTIVE_BIT, &port->flags);
1047 	if (rv >= 0 && mtip_pause_ncq(port, fis)) {
1048 		/* NCQ paused */
1049 		return rv;
1050 	}
1051 	wake_up_interruptible(&port->svc_wait);
1052 
1053 	return rv;
1054 }
1055 
1056 /*
1057  * Byte-swap ATA ID strings.
1058  *
1059  * ATA identify data contains strings in byte-swapped 16-bit words.
1060  * They must be swapped (on all architectures) to be usable as C strings.
1061  * This function swaps bytes in-place.
1062  *
1063  * @buf The buffer location of the string
1064  * @len The number of bytes to swap
1065  *
1066  * return value
1067  *	None
1068  */
1069 static inline void ata_swap_string(u16 *buf, unsigned int len)
1070 {
1071 	int i;
1072 	for (i = 0; i < (len/2); i++)
1073 		be16_to_cpus(&buf[i]);
1074 }
1075 
1076 static void mtip_set_timeout(struct driver_data *dd,
1077 					struct host_to_dev_fis *fis,
1078 					unsigned int *timeout, u8 erasemode)
1079 {
1080 	switch (fis->command) {
1081 	case ATA_CMD_DOWNLOAD_MICRO:
1082 		*timeout = 120000; /* 2 minutes */
1083 		break;
1084 	case ATA_CMD_SEC_ERASE_UNIT:
1085 	case 0xFC:
1086 		if (erasemode)
1087 			*timeout = ((*(dd->port->identify + 90) * 2) * 60000);
1088 		else
1089 			*timeout = ((*(dd->port->identify + 89) * 2) * 60000);
1090 		break;
1091 	case ATA_CMD_STANDBYNOW1:
1092 		*timeout = 120000;  /* 2 minutes */
1093 		break;
1094 	case 0xF7:
1095 	case 0xFA:
1096 		*timeout = 60000;  /* 60 seconds */
1097 		break;
1098 	case ATA_CMD_SMART:
1099 		*timeout = 15000;  /* 15 seconds */
1100 		break;
1101 	default:
1102 		*timeout = MTIP_IOCTL_CMD_TIMEOUT_MS;
1103 		break;
1104 	}
1105 }
1106 
1107 /*
1108  * Request the device identity information.
1109  *
1110  * If a user space buffer is not specified, i.e. is NULL, the
1111  * identify information is still read from the drive and placed
1112  * into the identify data buffer (@e port->identify) in the
1113  * port data structure.
1114  * When the identify buffer contains valid identify information @e
1115  * port->identify_valid is non-zero.
1116  *
1117  * @port	 Pointer to the port structure.
1118  * @user_buffer  A user space buffer where the identify data should be
1119  *                    copied.
1120  *
1121  * return value
1122  *	0	Command completed successfully.
1123  *	-EFAULT An error occurred while coping data to the user buffer.
1124  *	-1	Command failed.
1125  */
1126 static int mtip_get_identify(struct mtip_port *port, void __user *user_buffer)
1127 {
1128 	int rv = 0;
1129 	struct host_to_dev_fis fis;
1130 
1131 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag))
1132 		return -EFAULT;
1133 
1134 	/* Build the FIS. */
1135 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1136 	fis.type	= 0x27;
1137 	fis.opts	= 1 << 7;
1138 	fis.command	= ATA_CMD_ID_ATA;
1139 
1140 	/* Set the identify information as invalid. */
1141 	port->identify_valid = 0;
1142 
1143 	/* Clear the identify information. */
1144 	memset(port->identify, 0, sizeof(u16) * ATA_ID_WORDS);
1145 
1146 	/* Execute the command. */
1147 	if (mtip_exec_internal_command(port,
1148 				&fis,
1149 				5,
1150 				port->identify_dma,
1151 				sizeof(u16) * ATA_ID_WORDS,
1152 				0,
1153 				MTIP_INT_CMD_TIMEOUT_MS)
1154 				< 0) {
1155 		rv = -1;
1156 		goto out;
1157 	}
1158 
1159 	/*
1160 	 * Perform any necessary byte-swapping.  Yes, the kernel does in fact
1161 	 * perform field-sensitive swapping on the string fields.
1162 	 * See the kernel use of ata_id_string() for proof of this.
1163 	 */
1164 #ifdef __LITTLE_ENDIAN
1165 	ata_swap_string(port->identify + 27, 40);  /* model string*/
1166 	ata_swap_string(port->identify + 23, 8);   /* firmware string*/
1167 	ata_swap_string(port->identify + 10, 20);  /* serial# string*/
1168 #else
1169 	{
1170 		int i;
1171 		for (i = 0; i < ATA_ID_WORDS; i++)
1172 			port->identify[i] = le16_to_cpu(port->identify[i]);
1173 	}
1174 #endif
1175 
1176 	/* Check security locked state */
1177 	if (port->identify[128] & 0x4)
1178 		set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
1179 	else
1180 		clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag);
1181 
1182 	/* Set the identify buffer as valid. */
1183 	port->identify_valid = 1;
1184 
1185 	if (user_buffer) {
1186 		if (copy_to_user(
1187 			user_buffer,
1188 			port->identify,
1189 			ATA_ID_WORDS * sizeof(u16))) {
1190 			rv = -EFAULT;
1191 			goto out;
1192 		}
1193 	}
1194 
1195 out:
1196 	return rv;
1197 }
1198 
1199 /*
1200  * Issue a standby immediate command to the device.
1201  *
1202  * @port Pointer to the port structure.
1203  *
1204  * return value
1205  *	0	Command was executed successfully.
1206  *	-1	An error occurred while executing the command.
1207  */
1208 static int mtip_standby_immediate(struct mtip_port *port)
1209 {
1210 	int rv;
1211 	struct host_to_dev_fis	fis;
1212 	unsigned long __maybe_unused start;
1213 	unsigned int timeout;
1214 
1215 	/* Build the FIS. */
1216 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1217 	fis.type	= 0x27;
1218 	fis.opts	= 1 << 7;
1219 	fis.command	= ATA_CMD_STANDBYNOW1;
1220 
1221 	mtip_set_timeout(port->dd, &fis, &timeout, 0);
1222 
1223 	start = jiffies;
1224 	rv = mtip_exec_internal_command(port,
1225 					&fis,
1226 					5,
1227 					0,
1228 					0,
1229 					0,
1230 					timeout);
1231 	dbg_printk(MTIP_DRV_NAME "Time taken to complete standby cmd: %d ms\n",
1232 			jiffies_to_msecs(jiffies - start));
1233 	if (rv)
1234 		dev_warn(&port->dd->pdev->dev,
1235 			"STANDBY IMMEDIATE command failed.\n");
1236 
1237 	return rv;
1238 }
1239 
1240 /*
1241  * Issue a READ LOG EXT command to the device.
1242  *
1243  * @port	pointer to the port structure.
1244  * @page	page number to fetch
1245  * @buffer	pointer to buffer
1246  * @buffer_dma	dma address corresponding to @buffer
1247  * @sectors	page length to fetch, in sectors
1248  *
1249  * return value
1250  *	@rv	return value from mtip_exec_internal_command()
1251  */
1252 static int mtip_read_log_page(struct mtip_port *port, u8 page, u16 *buffer,
1253 				dma_addr_t buffer_dma, unsigned int sectors)
1254 {
1255 	struct host_to_dev_fis fis;
1256 
1257 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1258 	fis.type	= 0x27;
1259 	fis.opts	= 1 << 7;
1260 	fis.command	= ATA_CMD_READ_LOG_EXT;
1261 	fis.sect_count	= sectors & 0xFF;
1262 	fis.sect_cnt_ex	= (sectors >> 8) & 0xFF;
1263 	fis.lba_low	= page;
1264 	fis.lba_mid	= 0;
1265 	fis.device	= ATA_DEVICE_OBS;
1266 
1267 	memset(buffer, 0, sectors * ATA_SECT_SIZE);
1268 
1269 	return mtip_exec_internal_command(port,
1270 					&fis,
1271 					5,
1272 					buffer_dma,
1273 					sectors * ATA_SECT_SIZE,
1274 					0,
1275 					MTIP_INT_CMD_TIMEOUT_MS);
1276 }
1277 
1278 /*
1279  * Issue a SMART READ DATA command to the device.
1280  *
1281  * @port	pointer to the port structure.
1282  * @buffer	pointer to buffer
1283  * @buffer_dma	dma address corresponding to @buffer
1284  *
1285  * return value
1286  *	@rv	return value from mtip_exec_internal_command()
1287  */
1288 static int mtip_get_smart_data(struct mtip_port *port, u8 *buffer,
1289 					dma_addr_t buffer_dma)
1290 {
1291 	struct host_to_dev_fis fis;
1292 
1293 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1294 	fis.type	= 0x27;
1295 	fis.opts	= 1 << 7;
1296 	fis.command	= ATA_CMD_SMART;
1297 	fis.features	= 0xD0;
1298 	fis.sect_count	= 1;
1299 	fis.lba_mid	= 0x4F;
1300 	fis.lba_hi	= 0xC2;
1301 	fis.device	= ATA_DEVICE_OBS;
1302 
1303 	return mtip_exec_internal_command(port,
1304 					&fis,
1305 					5,
1306 					buffer_dma,
1307 					ATA_SECT_SIZE,
1308 					0,
1309 					15000);
1310 }
1311 
1312 /*
1313  * Get the value of a smart attribute
1314  *
1315  * @port	pointer to the port structure
1316  * @id		attribute number
1317  * @attrib	pointer to return attrib information corresponding to @id
1318  *
1319  * return value
1320  *	-EINVAL	NULL buffer passed or unsupported attribute @id.
1321  *	-EPERM	Identify data not valid, SMART not supported or not enabled
1322  */
1323 static int mtip_get_smart_attr(struct mtip_port *port, unsigned int id,
1324 						struct smart_attr *attrib)
1325 {
1326 	int rv, i;
1327 	struct smart_attr *pattr;
1328 
1329 	if (!attrib)
1330 		return -EINVAL;
1331 
1332 	if (!port->identify_valid) {
1333 		dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n");
1334 		return -EPERM;
1335 	}
1336 	if (!(port->identify[82] & 0x1)) {
1337 		dev_warn(&port->dd->pdev->dev, "SMART not supported\n");
1338 		return -EPERM;
1339 	}
1340 	if (!(port->identify[85] & 0x1)) {
1341 		dev_warn(&port->dd->pdev->dev, "SMART not enabled\n");
1342 		return -EPERM;
1343 	}
1344 
1345 	memset(port->smart_buf, 0, ATA_SECT_SIZE);
1346 	rv = mtip_get_smart_data(port, port->smart_buf, port->smart_buf_dma);
1347 	if (rv) {
1348 		dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n");
1349 		return rv;
1350 	}
1351 
1352 	pattr = (struct smart_attr *)(port->smart_buf + 2);
1353 	for (i = 0; i < 29; i++, pattr++)
1354 		if (pattr->attr_id == id) {
1355 			memcpy(attrib, pattr, sizeof(struct smart_attr));
1356 			break;
1357 		}
1358 
1359 	if (i == 29) {
1360 		dev_warn(&port->dd->pdev->dev,
1361 			"Query for invalid SMART attribute ID\n");
1362 		rv = -EINVAL;
1363 	}
1364 
1365 	return rv;
1366 }
1367 
1368 /*
1369  * Get the drive capacity.
1370  *
1371  * @dd      Pointer to the device data structure.
1372  * @sectors Pointer to the variable that will receive the sector count.
1373  *
1374  * return value
1375  *	1 Capacity was returned successfully.
1376  *	0 The identify information is invalid.
1377  */
1378 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors)
1379 {
1380 	struct mtip_port *port = dd->port;
1381 	u64 total, raw0, raw1, raw2, raw3;
1382 	raw0 = port->identify[100];
1383 	raw1 = port->identify[101];
1384 	raw2 = port->identify[102];
1385 	raw3 = port->identify[103];
1386 	total = raw0 | raw1<<16 | raw2<<32 | raw3<<48;
1387 	*sectors = total;
1388 	return (bool) !!port->identify_valid;
1389 }
1390 
1391 /*
1392  * Display the identify command data.
1393  *
1394  * @port Pointer to the port data structure.
1395  *
1396  * return value
1397  *	None
1398  */
1399 static void mtip_dump_identify(struct mtip_port *port)
1400 {
1401 	sector_t sectors;
1402 	unsigned short revid;
1403 	char cbuf[42];
1404 
1405 	if (!port->identify_valid)
1406 		return;
1407 
1408 	strlcpy(cbuf, (char *)(port->identify+10), 21);
1409 	dev_info(&port->dd->pdev->dev,
1410 		"Serial No.: %s\n", cbuf);
1411 
1412 	strlcpy(cbuf, (char *)(port->identify+23), 9);
1413 	dev_info(&port->dd->pdev->dev,
1414 		"Firmware Ver.: %s\n", cbuf);
1415 
1416 	strlcpy(cbuf, (char *)(port->identify+27), 41);
1417 	dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf);
1418 
1419 	dev_info(&port->dd->pdev->dev, "Security: %04x %s\n",
1420 		port->identify[128],
1421 		port->identify[128] & 0x4 ? "(LOCKED)" : "");
1422 
1423 	if (mtip_hw_get_capacity(port->dd, &sectors))
1424 		dev_info(&port->dd->pdev->dev,
1425 			"Capacity: %llu sectors (%llu MB)\n",
1426 			 (u64)sectors,
1427 			 ((u64)sectors) * ATA_SECT_SIZE >> 20);
1428 
1429 	pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid);
1430 	switch (revid & 0xFF) {
1431 	case 0x1:
1432 		strlcpy(cbuf, "A0", 3);
1433 		break;
1434 	case 0x3:
1435 		strlcpy(cbuf, "A2", 3);
1436 		break;
1437 	default:
1438 		strlcpy(cbuf, "?", 2);
1439 		break;
1440 	}
1441 	dev_info(&port->dd->pdev->dev,
1442 		"Card Type: %s\n", cbuf);
1443 }
1444 
1445 /*
1446  * Map the commands scatter list into the command table.
1447  *
1448  * @command Pointer to the command.
1449  * @nents Number of scatter list entries.
1450  *
1451  * return value
1452  *	None
1453  */
1454 static inline void fill_command_sg(struct driver_data *dd,
1455 				struct mtip_cmd *command,
1456 				int nents)
1457 {
1458 	int n;
1459 	unsigned int dma_len;
1460 	struct mtip_cmd_sg *command_sg;
1461 	struct scatterlist *sg;
1462 
1463 	command_sg = command->command + AHCI_CMD_TBL_HDR_SZ;
1464 
1465 	for_each_sg(command->sg, sg, nents, n) {
1466 		dma_len = sg_dma_len(sg);
1467 		if (dma_len > 0x400000)
1468 			dev_err(&dd->pdev->dev,
1469 				"DMA segment length truncated\n");
1470 		command_sg->info = cpu_to_le32((dma_len-1) & 0x3FFFFF);
1471 		command_sg->dba	=  cpu_to_le32(sg_dma_address(sg));
1472 		command_sg->dba_upper =
1473 			cpu_to_le32((sg_dma_address(sg) >> 16) >> 16);
1474 		command_sg++;
1475 	}
1476 }
1477 
1478 /*
1479  * @brief Execute a drive command.
1480  *
1481  * return value 0 The command completed successfully.
1482  * return value -1 An error occurred while executing the command.
1483  */
1484 static int exec_drive_task(struct mtip_port *port, u8 *command)
1485 {
1486 	struct host_to_dev_fis	fis;
1487 	struct host_to_dev_fis *reply = (port->rxfis + RX_FIS_D2H_REG);
1488 	unsigned int to;
1489 
1490 	/* Build the FIS. */
1491 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1492 	fis.type	= 0x27;
1493 	fis.opts	= 1 << 7;
1494 	fis.command	= command[0];
1495 	fis.features	= command[1];
1496 	fis.sect_count	= command[2];
1497 	fis.sector	= command[3];
1498 	fis.cyl_low	= command[4];
1499 	fis.cyl_hi	= command[5];
1500 	fis.device	= command[6] & ~0x10; /* Clear the dev bit*/
1501 
1502 	mtip_set_timeout(port->dd, &fis, &to, 0);
1503 
1504 	dbg_printk(MTIP_DRV_NAME " %s: User Command: cmd %x, feat %x, nsect %x, sect %x, lcyl %x, hcyl %x, sel %x\n",
1505 		__func__,
1506 		command[0],
1507 		command[1],
1508 		command[2],
1509 		command[3],
1510 		command[4],
1511 		command[5],
1512 		command[6]);
1513 
1514 	/* Execute the command. */
1515 	if (mtip_exec_internal_command(port,
1516 				 &fis,
1517 				 5,
1518 				 0,
1519 				 0,
1520 				 0,
1521 				 to) < 0) {
1522 		return -1;
1523 	}
1524 
1525 	command[0] = reply->command; /* Status*/
1526 	command[1] = reply->features; /* Error*/
1527 	command[4] = reply->cyl_low;
1528 	command[5] = reply->cyl_hi;
1529 
1530 	dbg_printk(MTIP_DRV_NAME " %s: Completion Status: stat %x, err %x , cyl_lo %x cyl_hi %x\n",
1531 		__func__,
1532 		command[0],
1533 		command[1],
1534 		command[4],
1535 		command[5]);
1536 
1537 	return 0;
1538 }
1539 
1540 /*
1541  * @brief Execute a drive command.
1542  *
1543  * @param port Pointer to the port data structure.
1544  * @param command Pointer to the user specified command parameters.
1545  * @param user_buffer Pointer to the user space buffer where read sector
1546  *                   data should be copied.
1547  *
1548  * return value 0 The command completed successfully.
1549  * return value -EFAULT An error occurred while copying the completion
1550  *                 data to the user space buffer.
1551  * return value -1 An error occurred while executing the command.
1552  */
1553 static int exec_drive_command(struct mtip_port *port, u8 *command,
1554 				void __user *user_buffer)
1555 {
1556 	struct host_to_dev_fis	fis;
1557 	struct host_to_dev_fis *reply;
1558 	u8 *buf = NULL;
1559 	dma_addr_t dma_addr = 0;
1560 	int rv = 0, xfer_sz = command[3];
1561 	unsigned int to;
1562 
1563 	if (xfer_sz) {
1564 		if (!user_buffer)
1565 			return -EFAULT;
1566 
1567 		buf = dma_alloc_coherent(&port->dd->pdev->dev,
1568 				ATA_SECT_SIZE * xfer_sz,
1569 				&dma_addr,
1570 				GFP_KERNEL);
1571 		if (!buf) {
1572 			dev_err(&port->dd->pdev->dev,
1573 				"Memory allocation failed (%d bytes)\n",
1574 				ATA_SECT_SIZE * xfer_sz);
1575 			return -ENOMEM;
1576 		}
1577 	}
1578 
1579 	/* Build the FIS. */
1580 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1581 	fis.type	= 0x27;
1582 	fis.opts	= 1 << 7;
1583 	fis.command	= command[0];
1584 	fis.features	= command[2];
1585 	fis.sect_count	= command[3];
1586 	if (fis.command == ATA_CMD_SMART) {
1587 		fis.sector	= command[1];
1588 		fis.cyl_low	= 0x4F;
1589 		fis.cyl_hi	= 0xC2;
1590 	}
1591 
1592 	mtip_set_timeout(port->dd, &fis, &to, 0);
1593 
1594 	if (xfer_sz)
1595 		reply = (port->rxfis + RX_FIS_PIO_SETUP);
1596 	else
1597 		reply = (port->rxfis + RX_FIS_D2H_REG);
1598 
1599 	dbg_printk(MTIP_DRV_NAME
1600 		" %s: User Command: cmd %x, sect %x, "
1601 		"feat %x, sectcnt %x\n",
1602 		__func__,
1603 		command[0],
1604 		command[1],
1605 		command[2],
1606 		command[3]);
1607 
1608 	/* Execute the command. */
1609 	if (mtip_exec_internal_command(port,
1610 				&fis,
1611 				 5,
1612 				 (xfer_sz ? dma_addr : 0),
1613 				 (xfer_sz ? ATA_SECT_SIZE * xfer_sz : 0),
1614 				 0,
1615 				 to)
1616 				 < 0) {
1617 		rv = -EFAULT;
1618 		goto exit_drive_command;
1619 	}
1620 
1621 	/* Collect the completion status. */
1622 	command[0] = reply->command; /* Status*/
1623 	command[1] = reply->features; /* Error*/
1624 	command[2] = reply->sect_count;
1625 
1626 	dbg_printk(MTIP_DRV_NAME
1627 		" %s: Completion Status: stat %x, "
1628 		"err %x, nsect %x\n",
1629 		__func__,
1630 		command[0],
1631 		command[1],
1632 		command[2]);
1633 
1634 	if (xfer_sz) {
1635 		if (copy_to_user(user_buffer,
1636 				 buf,
1637 				 ATA_SECT_SIZE * command[3])) {
1638 			rv = -EFAULT;
1639 			goto exit_drive_command;
1640 		}
1641 	}
1642 exit_drive_command:
1643 	if (buf)
1644 		dma_free_coherent(&port->dd->pdev->dev,
1645 				ATA_SECT_SIZE * xfer_sz, buf, dma_addr);
1646 	return rv;
1647 }
1648 
1649 /*
1650  *  Indicates whether a command has a single sector payload.
1651  *
1652  *  @command passed to the device to perform the certain event.
1653  *  @features passed to the device to perform the certain event.
1654  *
1655  *  return value
1656  *	1	command is one that always has a single sector payload,
1657  *		regardless of the value in the Sector Count field.
1658  *      0       otherwise
1659  *
1660  */
1661 static unsigned int implicit_sector(unsigned char command,
1662 				    unsigned char features)
1663 {
1664 	unsigned int rv = 0;
1665 
1666 	/* list of commands that have an implicit sector count of 1 */
1667 	switch (command) {
1668 	case ATA_CMD_SEC_SET_PASS:
1669 	case ATA_CMD_SEC_UNLOCK:
1670 	case ATA_CMD_SEC_ERASE_PREP:
1671 	case ATA_CMD_SEC_ERASE_UNIT:
1672 	case ATA_CMD_SEC_FREEZE_LOCK:
1673 	case ATA_CMD_SEC_DISABLE_PASS:
1674 	case ATA_CMD_PMP_READ:
1675 	case ATA_CMD_PMP_WRITE:
1676 		rv = 1;
1677 		break;
1678 	case ATA_CMD_SET_MAX:
1679 		if (features == ATA_SET_MAX_UNLOCK)
1680 			rv = 1;
1681 		break;
1682 	case ATA_CMD_SMART:
1683 		if ((features == ATA_SMART_READ_VALUES) ||
1684 				(features == ATA_SMART_READ_THRESHOLDS))
1685 			rv = 1;
1686 		break;
1687 	case ATA_CMD_CONF_OVERLAY:
1688 		if ((features == ATA_DCO_IDENTIFY) ||
1689 				(features == ATA_DCO_SET))
1690 			rv = 1;
1691 		break;
1692 	}
1693 	return rv;
1694 }
1695 
1696 /*
1697  * Executes a taskfile
1698  * See ide_taskfile_ioctl() for derivation
1699  */
1700 static int exec_drive_taskfile(struct driver_data *dd,
1701 			       void __user *buf,
1702 			       ide_task_request_t *req_task,
1703 			       int outtotal)
1704 {
1705 	struct host_to_dev_fis	fis;
1706 	struct host_to_dev_fis *reply;
1707 	u8 *outbuf = NULL;
1708 	u8 *inbuf = NULL;
1709 	dma_addr_t outbuf_dma = 0;
1710 	dma_addr_t inbuf_dma = 0;
1711 	dma_addr_t dma_buffer = 0;
1712 	int err = 0;
1713 	unsigned int taskin = 0;
1714 	unsigned int taskout = 0;
1715 	u8 nsect = 0;
1716 	unsigned int timeout;
1717 	unsigned int force_single_sector;
1718 	unsigned int transfer_size;
1719 	unsigned long task_file_data;
1720 	int intotal = outtotal + req_task->out_size;
1721 	int erasemode = 0;
1722 
1723 	taskout = req_task->out_size;
1724 	taskin = req_task->in_size;
1725 	/* 130560 = 512 * 0xFF*/
1726 	if (taskin > 130560 || taskout > 130560)
1727 		return -EINVAL;
1728 
1729 	if (taskout) {
1730 		outbuf = memdup_user(buf + outtotal, taskout);
1731 		if (IS_ERR(outbuf))
1732 			return PTR_ERR(outbuf);
1733 
1734 		outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf,
1735 					    taskout, DMA_TO_DEVICE);
1736 		if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) {
1737 			err = -ENOMEM;
1738 			goto abort;
1739 		}
1740 		dma_buffer = outbuf_dma;
1741 	}
1742 
1743 	if (taskin) {
1744 		inbuf = memdup_user(buf + intotal, taskin);
1745 		if (IS_ERR(inbuf)) {
1746 			err = PTR_ERR(inbuf);
1747 			inbuf = NULL;
1748 			goto abort;
1749 		}
1750 		inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf,
1751 					   taskin, DMA_FROM_DEVICE);
1752 		if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) {
1753 			err = -ENOMEM;
1754 			goto abort;
1755 		}
1756 		dma_buffer = inbuf_dma;
1757 	}
1758 
1759 	/* only supports PIO and non-data commands from this ioctl. */
1760 	switch (req_task->data_phase) {
1761 	case TASKFILE_OUT:
1762 		nsect = taskout / ATA_SECT_SIZE;
1763 		reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1764 		break;
1765 	case TASKFILE_IN:
1766 		reply = (dd->port->rxfis + RX_FIS_PIO_SETUP);
1767 		break;
1768 	case TASKFILE_NO_DATA:
1769 		reply = (dd->port->rxfis + RX_FIS_D2H_REG);
1770 		break;
1771 	default:
1772 		err = -EINVAL;
1773 		goto abort;
1774 	}
1775 
1776 	/* Build the FIS. */
1777 	memset(&fis, 0, sizeof(struct host_to_dev_fis));
1778 
1779 	fis.type	= 0x27;
1780 	fis.opts	= 1 << 7;
1781 	fis.command	= req_task->io_ports[7];
1782 	fis.features	= req_task->io_ports[1];
1783 	fis.sect_count	= req_task->io_ports[2];
1784 	fis.lba_low	= req_task->io_ports[3];
1785 	fis.lba_mid	= req_task->io_ports[4];
1786 	fis.lba_hi	= req_task->io_ports[5];
1787 	 /* Clear the dev bit*/
1788 	fis.device	= req_task->io_ports[6] & ~0x10;
1789 
1790 	if ((req_task->in_flags.all == 0) && (req_task->out_flags.all & 1)) {
1791 		req_task->in_flags.all	=
1792 			IDE_TASKFILE_STD_IN_FLAGS |
1793 			(IDE_HOB_STD_IN_FLAGS << 8);
1794 		fis.lba_low_ex		= req_task->hob_ports[3];
1795 		fis.lba_mid_ex		= req_task->hob_ports[4];
1796 		fis.lba_hi_ex		= req_task->hob_ports[5];
1797 		fis.features_ex		= req_task->hob_ports[1];
1798 		fis.sect_cnt_ex		= req_task->hob_ports[2];
1799 
1800 	} else {
1801 		req_task->in_flags.all = IDE_TASKFILE_STD_IN_FLAGS;
1802 	}
1803 
1804 	force_single_sector = implicit_sector(fis.command, fis.features);
1805 
1806 	if ((taskin || taskout) && (!fis.sect_count)) {
1807 		if (nsect)
1808 			fis.sect_count = nsect;
1809 		else {
1810 			if (!force_single_sector) {
1811 				dev_warn(&dd->pdev->dev,
1812 					"data movement but "
1813 					"sect_count is 0\n");
1814 				err = -EINVAL;
1815 				goto abort;
1816 			}
1817 		}
1818 	}
1819 
1820 	dbg_printk(MTIP_DRV_NAME
1821 		" %s: cmd %x, feat %x, nsect %x,"
1822 		" sect/lbal %x, lcyl/lbam %x, hcyl/lbah %x,"
1823 		" head/dev %x\n",
1824 		__func__,
1825 		fis.command,
1826 		fis.features,
1827 		fis.sect_count,
1828 		fis.lba_low,
1829 		fis.lba_mid,
1830 		fis.lba_hi,
1831 		fis.device);
1832 
1833 	/* check for erase mode support during secure erase.*/
1834 	if ((fis.command == ATA_CMD_SEC_ERASE_UNIT) && outbuf &&
1835 					(outbuf[0] & MTIP_SEC_ERASE_MODE)) {
1836 		erasemode = 1;
1837 	}
1838 
1839 	mtip_set_timeout(dd, &fis, &timeout, erasemode);
1840 
1841 	/* Determine the correct transfer size.*/
1842 	if (force_single_sector)
1843 		transfer_size = ATA_SECT_SIZE;
1844 	else
1845 		transfer_size = ATA_SECT_SIZE * fis.sect_count;
1846 
1847 	/* Execute the command.*/
1848 	if (mtip_exec_internal_command(dd->port,
1849 				 &fis,
1850 				 5,
1851 				 dma_buffer,
1852 				 transfer_size,
1853 				 0,
1854 				 timeout) < 0) {
1855 		err = -EIO;
1856 		goto abort;
1857 	}
1858 
1859 	task_file_data = readl(dd->port->mmio+PORT_TFDATA);
1860 
1861 	if ((req_task->data_phase == TASKFILE_IN) && !(task_file_data & 1)) {
1862 		reply = dd->port->rxfis + RX_FIS_PIO_SETUP;
1863 		req_task->io_ports[7] = reply->control;
1864 	} else {
1865 		reply = dd->port->rxfis + RX_FIS_D2H_REG;
1866 		req_task->io_ports[7] = reply->command;
1867 	}
1868 
1869 	/* reclaim the DMA buffers.*/
1870 	if (inbuf_dma)
1871 		dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
1872 				 DMA_FROM_DEVICE);
1873 	if (outbuf_dma)
1874 		dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
1875 				 DMA_TO_DEVICE);
1876 	inbuf_dma  = 0;
1877 	outbuf_dma = 0;
1878 
1879 	/* return the ATA registers to the caller.*/
1880 	req_task->io_ports[1] = reply->features;
1881 	req_task->io_ports[2] = reply->sect_count;
1882 	req_task->io_ports[3] = reply->lba_low;
1883 	req_task->io_ports[4] = reply->lba_mid;
1884 	req_task->io_ports[5] = reply->lba_hi;
1885 	req_task->io_ports[6] = reply->device;
1886 
1887 	if (req_task->out_flags.all & 1)  {
1888 
1889 		req_task->hob_ports[3] = reply->lba_low_ex;
1890 		req_task->hob_ports[4] = reply->lba_mid_ex;
1891 		req_task->hob_ports[5] = reply->lba_hi_ex;
1892 		req_task->hob_ports[1] = reply->features_ex;
1893 		req_task->hob_ports[2] = reply->sect_cnt_ex;
1894 	}
1895 	dbg_printk(MTIP_DRV_NAME
1896 		" %s: Completion: stat %x,"
1897 		"err %x, sect_cnt %x, lbalo %x,"
1898 		"lbamid %x, lbahi %x, dev %x\n",
1899 		__func__,
1900 		req_task->io_ports[7],
1901 		req_task->io_ports[1],
1902 		req_task->io_ports[2],
1903 		req_task->io_ports[3],
1904 		req_task->io_ports[4],
1905 		req_task->io_ports[5],
1906 		req_task->io_ports[6]);
1907 
1908 	if (taskout) {
1909 		if (copy_to_user(buf + outtotal, outbuf, taskout)) {
1910 			err = -EFAULT;
1911 			goto abort;
1912 		}
1913 	}
1914 	if (taskin) {
1915 		if (copy_to_user(buf + intotal, inbuf, taskin)) {
1916 			err = -EFAULT;
1917 			goto abort;
1918 		}
1919 	}
1920 abort:
1921 	if (inbuf_dma)
1922 		dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin,
1923 				 DMA_FROM_DEVICE);
1924 	if (outbuf_dma)
1925 		dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout,
1926 				 DMA_TO_DEVICE);
1927 	kfree(outbuf);
1928 	kfree(inbuf);
1929 
1930 	return err;
1931 }
1932 
1933 /*
1934  * Handle IOCTL calls from the Block Layer.
1935  *
1936  * This function is called by the Block Layer when it receives an IOCTL
1937  * command that it does not understand. If the IOCTL command is not supported
1938  * this function returns -ENOTTY.
1939  *
1940  * @dd  Pointer to the driver data structure.
1941  * @cmd IOCTL command passed from the Block Layer.
1942  * @arg IOCTL argument passed from the Block Layer.
1943  *
1944  * return value
1945  *	0	The IOCTL completed successfully.
1946  *	-ENOTTY The specified command is not supported.
1947  *	-EFAULT An error occurred copying data to a user space buffer.
1948  *	-EIO	An error occurred while executing the command.
1949  */
1950 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd,
1951 			 unsigned long arg)
1952 {
1953 	switch (cmd) {
1954 	case HDIO_GET_IDENTITY:
1955 	{
1956 		if (copy_to_user((void __user *)arg, dd->port->identify,
1957 						sizeof(u16) * ATA_ID_WORDS))
1958 			return -EFAULT;
1959 		break;
1960 	}
1961 	case HDIO_DRIVE_CMD:
1962 	{
1963 		u8 drive_command[4];
1964 
1965 		/* Copy the user command info to our buffer. */
1966 		if (copy_from_user(drive_command,
1967 					 (void __user *) arg,
1968 					 sizeof(drive_command)))
1969 			return -EFAULT;
1970 
1971 		/* Execute the drive command. */
1972 		if (exec_drive_command(dd->port,
1973 					 drive_command,
1974 					 (void __user *) (arg+4)))
1975 			return -EIO;
1976 
1977 		/* Copy the status back to the users buffer. */
1978 		if (copy_to_user((void __user *) arg,
1979 					 drive_command,
1980 					 sizeof(drive_command)))
1981 			return -EFAULT;
1982 
1983 		break;
1984 	}
1985 	case HDIO_DRIVE_TASK:
1986 	{
1987 		u8 drive_command[7];
1988 
1989 		/* Copy the user command info to our buffer. */
1990 		if (copy_from_user(drive_command,
1991 					 (void __user *) arg,
1992 					 sizeof(drive_command)))
1993 			return -EFAULT;
1994 
1995 		/* Execute the drive command. */
1996 		if (exec_drive_task(dd->port, drive_command))
1997 			return -EIO;
1998 
1999 		/* Copy the status back to the users buffer. */
2000 		if (copy_to_user((void __user *) arg,
2001 					 drive_command,
2002 					 sizeof(drive_command)))
2003 			return -EFAULT;
2004 
2005 		break;
2006 	}
2007 	case HDIO_DRIVE_TASKFILE: {
2008 		ide_task_request_t req_task;
2009 		int ret, outtotal;
2010 
2011 		if (copy_from_user(&req_task, (void __user *) arg,
2012 					sizeof(req_task)))
2013 			return -EFAULT;
2014 
2015 		outtotal = sizeof(req_task);
2016 
2017 		ret = exec_drive_taskfile(dd, (void __user *) arg,
2018 						&req_task, outtotal);
2019 
2020 		if (copy_to_user((void __user *) arg, &req_task,
2021 							sizeof(req_task)))
2022 			return -EFAULT;
2023 
2024 		return ret;
2025 	}
2026 
2027 	default:
2028 		return -EINVAL;
2029 	}
2030 	return 0;
2031 }
2032 
2033 /*
2034  * Submit an IO to the hw
2035  *
2036  * This function is called by the block layer to issue an io
2037  * to the device. Upon completion, the callback function will
2038  * be called with the data parameter passed as the callback data.
2039  *
2040  * @dd       Pointer to the driver data structure.
2041  * @start    First sector to read.
2042  * @nsect    Number of sectors to read.
2043  * @tag      The tag of this read command.
2044  * @callback Pointer to the function that should be called
2045  *	     when the read completes.
2046  * @data     Callback data passed to the callback function
2047  *	     when the read completes.
2048  * @dir      Direction (read or write)
2049  *
2050  * return value
2051  *	None
2052  */
2053 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq,
2054 			      struct mtip_cmd *command,
2055 			      struct blk_mq_hw_ctx *hctx)
2056 {
2057 	struct mtip_cmd_hdr *hdr =
2058 		dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
2059 	struct host_to_dev_fis	*fis;
2060 	struct mtip_port *port = dd->port;
2061 	int dma_dir = rq_data_dir(rq) == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
2062 	u64 start = blk_rq_pos(rq);
2063 	unsigned int nsect = blk_rq_sectors(rq);
2064 	unsigned int nents;
2065 
2066 	/* Map the scatter list for DMA access */
2067 	nents = blk_rq_map_sg(hctx->queue, rq, command->sg);
2068 	nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir);
2069 
2070 	prefetch(&port->flags);
2071 
2072 	command->scatter_ents = nents;
2073 
2074 	/*
2075 	 * The number of retries for this command before it is
2076 	 * reported as a failure to the upper layers.
2077 	 */
2078 	command->retries = MTIP_MAX_RETRIES;
2079 
2080 	/* Fill out fis */
2081 	fis = command->command;
2082 	fis->type        = 0x27;
2083 	fis->opts        = 1 << 7;
2084 	if (dma_dir == DMA_FROM_DEVICE)
2085 		fis->command = ATA_CMD_FPDMA_READ;
2086 	else
2087 		fis->command = ATA_CMD_FPDMA_WRITE;
2088 	fis->lba_low     = start & 0xFF;
2089 	fis->lba_mid     = (start >> 8) & 0xFF;
2090 	fis->lba_hi      = (start >> 16) & 0xFF;
2091 	fis->lba_low_ex  = (start >> 24) & 0xFF;
2092 	fis->lba_mid_ex  = (start >> 32) & 0xFF;
2093 	fis->lba_hi_ex   = (start >> 40) & 0xFF;
2094 	fis->device	 = 1 << 6;
2095 	fis->features    = nsect & 0xFF;
2096 	fis->features_ex = (nsect >> 8) & 0xFF;
2097 	fis->sect_count  = ((rq->tag << 3) | (rq->tag >> 5));
2098 	fis->sect_cnt_ex = 0;
2099 	fis->control     = 0;
2100 	fis->res2        = 0;
2101 	fis->res3        = 0;
2102 	fill_command_sg(dd, command, nents);
2103 
2104 	if (unlikely(command->unaligned))
2105 		fis->device |= 1 << 7;
2106 
2107 	/* Populate the command header */
2108 	hdr->ctba = cpu_to_le32(command->command_dma & 0xFFFFFFFF);
2109 	if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
2110 		hdr->ctbau = cpu_to_le32((command->command_dma >> 16) >> 16);
2111 	hdr->opts = cpu_to_le32((nents << 16) | 5 | AHCI_CMD_PREFETCH);
2112 	hdr->byte_count = 0;
2113 
2114 	command->direction = dma_dir;
2115 
2116 	/*
2117 	 * To prevent this command from being issued
2118 	 * if an internal command is in progress or error handling is active.
2119 	 */
2120 	if (unlikely(port->flags & MTIP_PF_PAUSE_IO)) {
2121 		set_bit(rq->tag, port->cmds_to_issue);
2122 		set_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
2123 		return;
2124 	}
2125 
2126 	/* Issue the command to the hardware */
2127 	mtip_issue_ncq_command(port, rq->tag);
2128 }
2129 
2130 /*
2131  * Sysfs status dump.
2132  *
2133  * @dev  Pointer to the device structure, passed by the kernrel.
2134  * @attr Pointer to the device_attribute structure passed by the kernel.
2135  * @buf  Pointer to the char buffer that will receive the stats info.
2136  *
2137  * return value
2138  *	The size, in bytes, of the data copied into buf.
2139  */
2140 static ssize_t mtip_hw_show_status(struct device *dev,
2141 				struct device_attribute *attr,
2142 				char *buf)
2143 {
2144 	struct driver_data *dd = dev_to_disk(dev)->private_data;
2145 	int size = 0;
2146 
2147 	if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
2148 		size += sprintf(buf, "%s", "thermal_shutdown\n");
2149 	else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag))
2150 		size += sprintf(buf, "%s", "write_protect\n");
2151 	else
2152 		size += sprintf(buf, "%s", "online\n");
2153 
2154 	return size;
2155 }
2156 
2157 static DEVICE_ATTR(status, 0444, mtip_hw_show_status, NULL);
2158 
2159 static struct attribute *mtip_disk_attrs[] = {
2160 	&dev_attr_status.attr,
2161 	NULL,
2162 };
2163 
2164 static const struct attribute_group mtip_disk_attr_group = {
2165 	.attrs = mtip_disk_attrs,
2166 };
2167 
2168 static const struct attribute_group *mtip_disk_attr_groups[] = {
2169 	&mtip_disk_attr_group,
2170 	NULL,
2171 };
2172 
2173 /* debugsfs entries */
2174 
2175 static ssize_t show_device_status(struct device_driver *drv, char *buf)
2176 {
2177 	int size = 0;
2178 	struct driver_data *dd, *tmp;
2179 	unsigned long flags;
2180 	char id_buf[42];
2181 	u16 status = 0;
2182 
2183 	spin_lock_irqsave(&dev_lock, flags);
2184 	size += sprintf(&buf[size], "Devices Present:\n");
2185 	list_for_each_entry_safe(dd, tmp, &online_list, online_list) {
2186 		if (dd->pdev) {
2187 			if (dd->port &&
2188 			    dd->port->identify &&
2189 			    dd->port->identify_valid) {
2190 				strlcpy(id_buf,
2191 					(char *) (dd->port->identify + 10), 21);
2192 				status = *(dd->port->identify + 141);
2193 			} else {
2194 				memset(id_buf, 0, 42);
2195 				status = 0;
2196 			}
2197 
2198 			if (dd->port &&
2199 			    test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
2200 				size += sprintf(&buf[size],
2201 					" device %s %s (ftl rebuild %d %%)\n",
2202 					dev_name(&dd->pdev->dev),
2203 					id_buf,
2204 					status);
2205 			} else {
2206 				size += sprintf(&buf[size],
2207 					" device %s %s\n",
2208 					dev_name(&dd->pdev->dev),
2209 					id_buf);
2210 			}
2211 		}
2212 	}
2213 
2214 	size += sprintf(&buf[size], "Devices Being Removed:\n");
2215 	list_for_each_entry_safe(dd, tmp, &removing_list, remove_list) {
2216 		if (dd->pdev) {
2217 			if (dd->port &&
2218 			    dd->port->identify &&
2219 			    dd->port->identify_valid) {
2220 				strlcpy(id_buf,
2221 					(char *) (dd->port->identify+10), 21);
2222 				status = *(dd->port->identify + 141);
2223 			} else {
2224 				memset(id_buf, 0, 42);
2225 				status = 0;
2226 			}
2227 
2228 			if (dd->port &&
2229 			    test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags)) {
2230 				size += sprintf(&buf[size],
2231 					" device %s %s (ftl rebuild %d %%)\n",
2232 					dev_name(&dd->pdev->dev),
2233 					id_buf,
2234 					status);
2235 			} else {
2236 				size += sprintf(&buf[size],
2237 					" device %s %s\n",
2238 					dev_name(&dd->pdev->dev),
2239 					id_buf);
2240 			}
2241 		}
2242 	}
2243 	spin_unlock_irqrestore(&dev_lock, flags);
2244 
2245 	return size;
2246 }
2247 
2248 static ssize_t mtip_hw_read_device_status(struct file *f, char __user *ubuf,
2249 						size_t len, loff_t *offset)
2250 {
2251 	int size = *offset;
2252 	char *buf;
2253 	int rv = 0;
2254 
2255 	if (!len || *offset)
2256 		return 0;
2257 
2258 	buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2259 	if (!buf)
2260 		return -ENOMEM;
2261 
2262 	size += show_device_status(NULL, buf);
2263 
2264 	*offset = size <= len ? size : len;
2265 	size = copy_to_user(ubuf, buf, *offset);
2266 	if (size)
2267 		rv = -EFAULT;
2268 
2269 	kfree(buf);
2270 	return rv ? rv : *offset;
2271 }
2272 
2273 static ssize_t mtip_hw_read_registers(struct file *f, char __user *ubuf,
2274 				  size_t len, loff_t *offset)
2275 {
2276 	struct driver_data *dd =  (struct driver_data *)f->private_data;
2277 	char *buf;
2278 	u32 group_allocated;
2279 	int size = *offset;
2280 	int n, rv = 0;
2281 
2282 	if (!len || size)
2283 		return 0;
2284 
2285 	buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2286 	if (!buf)
2287 		return -ENOMEM;
2288 
2289 	size += sprintf(&buf[size], "H/ S ACTive      : [ 0x");
2290 
2291 	for (n = dd->slot_groups-1; n >= 0; n--)
2292 		size += sprintf(&buf[size], "%08X ",
2293 					 readl(dd->port->s_active[n]));
2294 
2295 	size += sprintf(&buf[size], "]\n");
2296 	size += sprintf(&buf[size], "H/ Command Issue : [ 0x");
2297 
2298 	for (n = dd->slot_groups-1; n >= 0; n--)
2299 		size += sprintf(&buf[size], "%08X ",
2300 					readl(dd->port->cmd_issue[n]));
2301 
2302 	size += sprintf(&buf[size], "]\n");
2303 	size += sprintf(&buf[size], "H/ Completed     : [ 0x");
2304 
2305 	for (n = dd->slot_groups-1; n >= 0; n--)
2306 		size += sprintf(&buf[size], "%08X ",
2307 				readl(dd->port->completed[n]));
2308 
2309 	size += sprintf(&buf[size], "]\n");
2310 	size += sprintf(&buf[size], "H/ PORT IRQ STAT : [ 0x%08X ]\n",
2311 				readl(dd->port->mmio + PORT_IRQ_STAT));
2312 	size += sprintf(&buf[size], "H/ HOST IRQ STAT : [ 0x%08X ]\n",
2313 				readl(dd->mmio + HOST_IRQ_STAT));
2314 	size += sprintf(&buf[size], "\n");
2315 
2316 	size += sprintf(&buf[size], "L/ Commands in Q : [ 0x");
2317 
2318 	for (n = dd->slot_groups-1; n >= 0; n--) {
2319 		if (sizeof(long) > sizeof(u32))
2320 			group_allocated =
2321 				dd->port->cmds_to_issue[n/2] >> (32*(n&1));
2322 		else
2323 			group_allocated = dd->port->cmds_to_issue[n];
2324 		size += sprintf(&buf[size], "%08X ", group_allocated);
2325 	}
2326 	size += sprintf(&buf[size], "]\n");
2327 
2328 	*offset = size <= len ? size : len;
2329 	size = copy_to_user(ubuf, buf, *offset);
2330 	if (size)
2331 		rv = -EFAULT;
2332 
2333 	kfree(buf);
2334 	return rv ? rv : *offset;
2335 }
2336 
2337 static ssize_t mtip_hw_read_flags(struct file *f, char __user *ubuf,
2338 				  size_t len, loff_t *offset)
2339 {
2340 	struct driver_data *dd =  (struct driver_data *)f->private_data;
2341 	char *buf;
2342 	int size = *offset;
2343 	int rv = 0;
2344 
2345 	if (!len || size)
2346 		return 0;
2347 
2348 	buf = kzalloc(MTIP_DFS_MAX_BUF_SIZE, GFP_KERNEL);
2349 	if (!buf)
2350 		return -ENOMEM;
2351 
2352 	size += sprintf(&buf[size], "Flag-port : [ %08lX ]\n",
2353 							dd->port->flags);
2354 	size += sprintf(&buf[size], "Flag-dd   : [ %08lX ]\n",
2355 							dd->dd_flag);
2356 
2357 	*offset = size <= len ? size : len;
2358 	size = copy_to_user(ubuf, buf, *offset);
2359 	if (size)
2360 		rv = -EFAULT;
2361 
2362 	kfree(buf);
2363 	return rv ? rv : *offset;
2364 }
2365 
2366 static const struct file_operations mtip_device_status_fops = {
2367 	.owner  = THIS_MODULE,
2368 	.open   = simple_open,
2369 	.read   = mtip_hw_read_device_status,
2370 	.llseek = no_llseek,
2371 };
2372 
2373 static const struct file_operations mtip_regs_fops = {
2374 	.owner  = THIS_MODULE,
2375 	.open   = simple_open,
2376 	.read   = mtip_hw_read_registers,
2377 	.llseek = no_llseek,
2378 };
2379 
2380 static const struct file_operations mtip_flags_fops = {
2381 	.owner  = THIS_MODULE,
2382 	.open   = simple_open,
2383 	.read   = mtip_hw_read_flags,
2384 	.llseek = no_llseek,
2385 };
2386 
2387 static int mtip_hw_debugfs_init(struct driver_data *dd)
2388 {
2389 	if (!dfs_parent)
2390 		return -1;
2391 
2392 	dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent);
2393 	if (IS_ERR_OR_NULL(dd->dfs_node)) {
2394 		dev_warn(&dd->pdev->dev,
2395 			"Error creating node %s under debugfs\n",
2396 						dd->disk->disk_name);
2397 		dd->dfs_node = NULL;
2398 		return -1;
2399 	}
2400 
2401 	debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops);
2402 	debugfs_create_file("registers", 0444, dd->dfs_node, dd,
2403 			    &mtip_regs_fops);
2404 
2405 	return 0;
2406 }
2407 
2408 static void mtip_hw_debugfs_exit(struct driver_data *dd)
2409 {
2410 	debugfs_remove_recursive(dd->dfs_node);
2411 }
2412 
2413 /*
2414  * Perform any init/resume time hardware setup
2415  *
2416  * @dd Pointer to the driver data structure.
2417  *
2418  * return value
2419  *	None
2420  */
2421 static inline void hba_setup(struct driver_data *dd)
2422 {
2423 	u32 hwdata;
2424 	hwdata = readl(dd->mmio + HOST_HSORG);
2425 
2426 	/* interrupt bug workaround: use only 1 IS bit.*/
2427 	writel(hwdata |
2428 		HSORG_DISABLE_SLOTGRP_INTR |
2429 		HSORG_DISABLE_SLOTGRP_PXIS,
2430 		dd->mmio + HOST_HSORG);
2431 }
2432 
2433 static int mtip_device_unaligned_constrained(struct driver_data *dd)
2434 {
2435 	return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0);
2436 }
2437 
2438 /*
2439  * Detect the details of the product, and store anything needed
2440  * into the driver data structure.  This includes product type and
2441  * version and number of slot groups.
2442  *
2443  * @dd Pointer to the driver data structure.
2444  *
2445  * return value
2446  *	None
2447  */
2448 static void mtip_detect_product(struct driver_data *dd)
2449 {
2450 	u32 hwdata;
2451 	unsigned int rev, slotgroups;
2452 
2453 	/*
2454 	 * HBA base + 0xFC [15:0] - vendor-specific hardware interface
2455 	 * info register:
2456 	 * [15:8] hardware/software interface rev#
2457 	 * [   3] asic-style interface
2458 	 * [ 2:0] number of slot groups, minus 1 (only valid for asic-style).
2459 	 */
2460 	hwdata = readl(dd->mmio + HOST_HSORG);
2461 
2462 	dd->product_type = MTIP_PRODUCT_UNKNOWN;
2463 	dd->slot_groups = 1;
2464 
2465 	if (hwdata & 0x8) {
2466 		dd->product_type = MTIP_PRODUCT_ASICFPGA;
2467 		rev = (hwdata & HSORG_HWREV) >> 8;
2468 		slotgroups = (hwdata & HSORG_SLOTGROUPS) + 1;
2469 		dev_info(&dd->pdev->dev,
2470 			"ASIC-FPGA design, HS rev 0x%x, "
2471 			"%i slot groups [%i slots]\n",
2472 			 rev,
2473 			 slotgroups,
2474 			 slotgroups * 32);
2475 
2476 		if (slotgroups > MTIP_MAX_SLOT_GROUPS) {
2477 			dev_warn(&dd->pdev->dev,
2478 				"Warning: driver only supports "
2479 				"%i slot groups.\n", MTIP_MAX_SLOT_GROUPS);
2480 			slotgroups = MTIP_MAX_SLOT_GROUPS;
2481 		}
2482 		dd->slot_groups = slotgroups;
2483 		return;
2484 	}
2485 
2486 	dev_warn(&dd->pdev->dev, "Unrecognized product id\n");
2487 }
2488 
2489 /*
2490  * Blocking wait for FTL rebuild to complete
2491  *
2492  * @dd Pointer to the DRIVER_DATA structure.
2493  *
2494  * return value
2495  *	0	FTL rebuild completed successfully
2496  *	-EFAULT FTL rebuild error/timeout/interruption
2497  */
2498 static int mtip_ftl_rebuild_poll(struct driver_data *dd)
2499 {
2500 	unsigned long timeout, cnt = 0, start;
2501 
2502 	dev_warn(&dd->pdev->dev,
2503 		"FTL rebuild in progress. Polling for completion.\n");
2504 
2505 	start = jiffies;
2506 	timeout = jiffies + msecs_to_jiffies(MTIP_FTL_REBUILD_TIMEOUT_MS);
2507 
2508 	do {
2509 		if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
2510 				&dd->dd_flag)))
2511 			return -EFAULT;
2512 		if (mtip_check_surprise_removal(dd))
2513 			return -EFAULT;
2514 
2515 		if (mtip_get_identify(dd->port, NULL) < 0)
2516 			return -EFAULT;
2517 
2518 		if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2519 			MTIP_FTL_REBUILD_MAGIC) {
2520 			ssleep(1);
2521 			/* Print message every 3 minutes */
2522 			if (cnt++ >= 180) {
2523 				dev_warn(&dd->pdev->dev,
2524 				"FTL rebuild in progress (%d secs).\n",
2525 				jiffies_to_msecs(jiffies - start) / 1000);
2526 				cnt = 0;
2527 			}
2528 		} else {
2529 			dev_warn(&dd->pdev->dev,
2530 				"FTL rebuild complete (%d secs).\n",
2531 			jiffies_to_msecs(jiffies - start) / 1000);
2532 			mtip_block_initialize(dd);
2533 			return 0;
2534 		}
2535 	} while (time_before(jiffies, timeout));
2536 
2537 	/* Check for timeout */
2538 	dev_err(&dd->pdev->dev,
2539 		"Timed out waiting for FTL rebuild to complete (%d secs).\n",
2540 		jiffies_to_msecs(jiffies - start) / 1000);
2541 	return -EFAULT;
2542 }
2543 
2544 static void mtip_softirq_done_fn(struct request *rq)
2545 {
2546 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
2547 	struct driver_data *dd = rq->q->queuedata;
2548 
2549 	/* Unmap the DMA scatter list entries */
2550 	dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents,
2551 							cmd->direction);
2552 
2553 	if (unlikely(cmd->unaligned))
2554 		atomic_inc(&dd->port->cmd_slot_unal);
2555 
2556 	blk_mq_end_request(rq, cmd->status);
2557 }
2558 
2559 static bool mtip_abort_cmd(struct request *req, void *data, bool reserved)
2560 {
2561 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
2562 	struct driver_data *dd = data;
2563 
2564 	dbg_printk(MTIP_DRV_NAME " Aborting request, tag = %d\n", req->tag);
2565 
2566 	clear_bit(req->tag, dd->port->cmds_to_issue);
2567 	cmd->status = BLK_STS_IOERR;
2568 	mtip_softirq_done_fn(req);
2569 	return true;
2570 }
2571 
2572 static bool mtip_queue_cmd(struct request *req, void *data, bool reserved)
2573 {
2574 	struct driver_data *dd = data;
2575 
2576 	set_bit(req->tag, dd->port->cmds_to_issue);
2577 	blk_abort_request(req);
2578 	return true;
2579 }
2580 
2581 /*
2582  * service thread to issue queued commands
2583  *
2584  * @data Pointer to the driver data structure.
2585  *
2586  * return value
2587  *	0
2588  */
2589 
2590 static int mtip_service_thread(void *data)
2591 {
2592 	struct driver_data *dd = (struct driver_data *)data;
2593 	unsigned long slot, slot_start, slot_wrap, to;
2594 	unsigned int num_cmd_slots = dd->slot_groups * 32;
2595 	struct mtip_port *port = dd->port;
2596 
2597 	while (1) {
2598 		if (kthread_should_stop() ||
2599 			test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
2600 			goto st_out;
2601 		clear_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
2602 
2603 		/*
2604 		 * the condition is to check neither an internal command is
2605 		 * is in progress nor error handling is active
2606 		 */
2607 		wait_event_interruptible(port->svc_wait, (port->flags) &&
2608 			(port->flags & MTIP_PF_SVC_THD_WORK));
2609 
2610 		if (kthread_should_stop() ||
2611 			test_bit(MTIP_PF_SVC_THD_STOP_BIT, &port->flags))
2612 			goto st_out;
2613 
2614 		if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT,
2615 				&dd->dd_flag)))
2616 			goto st_out;
2617 
2618 		set_bit(MTIP_PF_SVC_THD_ACTIVE_BIT, &port->flags);
2619 
2620 restart_eh:
2621 		/* Demux bits: start with error handling */
2622 		if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags)) {
2623 			mtip_handle_tfe(dd);
2624 			clear_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags);
2625 		}
2626 
2627 		if (test_bit(MTIP_PF_EH_ACTIVE_BIT, &port->flags))
2628 			goto restart_eh;
2629 
2630 		if (test_bit(MTIP_PF_TO_ACTIVE_BIT, &port->flags)) {
2631 			to = jiffies + msecs_to_jiffies(5000);
2632 
2633 			do {
2634 				mdelay(100);
2635 			} while (atomic_read(&dd->irq_workers_active) != 0 &&
2636 				time_before(jiffies, to));
2637 
2638 			if (atomic_read(&dd->irq_workers_active) != 0)
2639 				dev_warn(&dd->pdev->dev,
2640 					"Completion workers still active!");
2641 
2642 			blk_mq_quiesce_queue(dd->queue);
2643 
2644 			blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd);
2645 
2646 			set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags);
2647 
2648 			if (mtip_device_reset(dd))
2649 				blk_mq_tagset_busy_iter(&dd->tags,
2650 							mtip_abort_cmd, dd);
2651 
2652 			clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags);
2653 
2654 			blk_mq_unquiesce_queue(dd->queue);
2655 		}
2656 
2657 		if (test_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags)) {
2658 			slot = 1;
2659 			/* used to restrict the loop to one iteration */
2660 			slot_start = num_cmd_slots;
2661 			slot_wrap = 0;
2662 			while (1) {
2663 				slot = find_next_bit(port->cmds_to_issue,
2664 						num_cmd_slots, slot);
2665 				if (slot_wrap == 1) {
2666 					if ((slot_start >= slot) ||
2667 						(slot >= num_cmd_slots))
2668 						break;
2669 				}
2670 				if (unlikely(slot_start == num_cmd_slots))
2671 					slot_start = slot;
2672 
2673 				if (unlikely(slot == num_cmd_slots)) {
2674 					slot = 1;
2675 					slot_wrap = 1;
2676 					continue;
2677 				}
2678 
2679 				/* Issue the command to the hardware */
2680 				mtip_issue_ncq_command(port, slot);
2681 
2682 				clear_bit(slot, port->cmds_to_issue);
2683 			}
2684 
2685 			clear_bit(MTIP_PF_ISSUE_CMDS_BIT, &port->flags);
2686 		}
2687 
2688 		if (test_bit(MTIP_PF_REBUILD_BIT, &port->flags)) {
2689 			if (mtip_ftl_rebuild_poll(dd) == 0)
2690 				clear_bit(MTIP_PF_REBUILD_BIT, &port->flags);
2691 		}
2692 	}
2693 
2694 st_out:
2695 	return 0;
2696 }
2697 
2698 /*
2699  * DMA region teardown
2700  *
2701  * @dd Pointer to driver_data structure
2702  *
2703  * return value
2704  *      None
2705  */
2706 static void mtip_dma_free(struct driver_data *dd)
2707 {
2708 	struct mtip_port *port = dd->port;
2709 
2710 	if (port->block1)
2711 		dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2712 					port->block1, port->block1_dma);
2713 
2714 	if (port->command_list) {
2715 		dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
2716 				port->command_list, port->command_list_dma);
2717 	}
2718 }
2719 
2720 /*
2721  * DMA region setup
2722  *
2723  * @dd Pointer to driver_data structure
2724  *
2725  * return value
2726  *      -ENOMEM Not enough free DMA region space to initialize driver
2727  */
2728 static int mtip_dma_alloc(struct driver_data *dd)
2729 {
2730 	struct mtip_port *port = dd->port;
2731 
2732 	/* Allocate dma memory for RX Fis, Identify, and Sector Buffer */
2733 	port->block1 =
2734 		dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2735 					&port->block1_dma, GFP_KERNEL);
2736 	if (!port->block1)
2737 		return -ENOMEM;
2738 
2739 	/* Allocate dma memory for command list */
2740 	port->command_list =
2741 		dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ,
2742 					&port->command_list_dma, GFP_KERNEL);
2743 	if (!port->command_list) {
2744 		dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ,
2745 					port->block1, port->block1_dma);
2746 		port->block1 = NULL;
2747 		port->block1_dma = 0;
2748 		return -ENOMEM;
2749 	}
2750 
2751 	/* Setup all pointers into first DMA region */
2752 	port->rxfis         = port->block1 + AHCI_RX_FIS_OFFSET;
2753 	port->rxfis_dma     = port->block1_dma + AHCI_RX_FIS_OFFSET;
2754 	port->identify      = port->block1 + AHCI_IDFY_OFFSET;
2755 	port->identify_dma  = port->block1_dma + AHCI_IDFY_OFFSET;
2756 	port->log_buf       = port->block1 + AHCI_SECTBUF_OFFSET;
2757 	port->log_buf_dma   = port->block1_dma + AHCI_SECTBUF_OFFSET;
2758 	port->smart_buf     = port->block1 + AHCI_SMARTBUF_OFFSET;
2759 	port->smart_buf_dma = port->block1_dma + AHCI_SMARTBUF_OFFSET;
2760 
2761 	return 0;
2762 }
2763 
2764 static int mtip_hw_get_identify(struct driver_data *dd)
2765 {
2766 	struct smart_attr attr242;
2767 	unsigned char *buf;
2768 	int rv;
2769 
2770 	if (mtip_get_identify(dd->port, NULL) < 0)
2771 		return -EFAULT;
2772 
2773 	if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) ==
2774 		MTIP_FTL_REBUILD_MAGIC) {
2775 		set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags);
2776 		return MTIP_FTL_REBUILD_MAGIC;
2777 	}
2778 	mtip_dump_identify(dd->port);
2779 
2780 	/* check write protect, over temp and rebuild statuses */
2781 	rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ,
2782 				dd->port->log_buf,
2783 				dd->port->log_buf_dma, 1);
2784 	if (rv) {
2785 		dev_warn(&dd->pdev->dev,
2786 			"Error in READ LOG EXT (10h) command\n");
2787 		/* non-critical error, don't fail the load */
2788 	} else {
2789 		buf = (unsigned char *)dd->port->log_buf;
2790 		if (buf[259] & 0x1) {
2791 			dev_info(&dd->pdev->dev,
2792 				"Write protect bit is set.\n");
2793 			set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag);
2794 		}
2795 		if (buf[288] == 0xF7) {
2796 			dev_info(&dd->pdev->dev,
2797 				"Exceeded Tmax, drive in thermal shutdown.\n");
2798 			set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag);
2799 		}
2800 		if (buf[288] == 0xBF) {
2801 			dev_info(&dd->pdev->dev,
2802 				"Drive indicates rebuild has failed.\n");
2803 			set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag);
2804 		}
2805 	}
2806 
2807 	/* get write protect progess */
2808 	memset(&attr242, 0, sizeof(struct smart_attr));
2809 	if (mtip_get_smart_attr(dd->port, 242, &attr242))
2810 		dev_warn(&dd->pdev->dev,
2811 				"Unable to check write protect progress\n");
2812 	else
2813 		dev_info(&dd->pdev->dev,
2814 				"Write protect progress: %u%% (%u blocks)\n",
2815 				attr242.cur, le32_to_cpu(attr242.data));
2816 
2817 	return rv;
2818 }
2819 
2820 /*
2821  * Called once for each card.
2822  *
2823  * @dd Pointer to the driver data structure.
2824  *
2825  * return value
2826  *	0 on success, else an error code.
2827  */
2828 static int mtip_hw_init(struct driver_data *dd)
2829 {
2830 	int i;
2831 	int rv;
2832 	unsigned long timeout, timetaken;
2833 
2834 	dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR];
2835 
2836 	mtip_detect_product(dd);
2837 	if (dd->product_type == MTIP_PRODUCT_UNKNOWN) {
2838 		rv = -EIO;
2839 		goto out1;
2840 	}
2841 
2842 	hba_setup(dd);
2843 
2844 	dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL,
2845 				dd->numa_node);
2846 	if (!dd->port)
2847 		return -ENOMEM;
2848 
2849 	/* Continue workqueue setup */
2850 	for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
2851 		dd->work[i].port = dd->port;
2852 
2853 	/* Enable unaligned IO constraints for some devices */
2854 	if (mtip_device_unaligned_constrained(dd))
2855 		dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS;
2856 	else
2857 		dd->unal_qdepth = 0;
2858 
2859 	atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth);
2860 
2861 	/* Spinlock to prevent concurrent issue */
2862 	for (i = 0; i < MTIP_MAX_SLOT_GROUPS; i++)
2863 		spin_lock_init(&dd->port->cmd_issue_lock[i]);
2864 
2865 	/* Set the port mmio base address. */
2866 	dd->port->mmio	= dd->mmio + PORT_OFFSET;
2867 	dd->port->dd	= dd;
2868 
2869 	/* DMA allocations */
2870 	rv = mtip_dma_alloc(dd);
2871 	if (rv < 0)
2872 		goto out1;
2873 
2874 	/* Setup the pointers to the extended s_active and CI registers. */
2875 	for (i = 0; i < dd->slot_groups; i++) {
2876 		dd->port->s_active[i] =
2877 			dd->port->mmio + i*0x80 + PORT_SCR_ACT;
2878 		dd->port->cmd_issue[i] =
2879 			dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE;
2880 		dd->port->completed[i] =
2881 			dd->port->mmio + i*0x80 + PORT_SDBV;
2882 	}
2883 
2884 	timetaken = jiffies;
2885 	timeout = jiffies + msecs_to_jiffies(30000);
2886 	while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) &&
2887 		 time_before(jiffies, timeout)) {
2888 		mdelay(100);
2889 	}
2890 	if (unlikely(mtip_check_surprise_removal(dd))) {
2891 		timetaken = jiffies - timetaken;
2892 		dev_warn(&dd->pdev->dev,
2893 			"Surprise removal detected at %u ms\n",
2894 			jiffies_to_msecs(timetaken));
2895 		rv = -ENODEV;
2896 		goto out2 ;
2897 	}
2898 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) {
2899 		timetaken = jiffies - timetaken;
2900 		dev_warn(&dd->pdev->dev,
2901 			"Removal detected at %u ms\n",
2902 			jiffies_to_msecs(timetaken));
2903 		rv = -EFAULT;
2904 		goto out2;
2905 	}
2906 
2907 	/* Conditionally reset the HBA. */
2908 	if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) {
2909 		if (mtip_hba_reset(dd) < 0) {
2910 			dev_err(&dd->pdev->dev,
2911 				"Card did not reset within timeout\n");
2912 			rv = -EIO;
2913 			goto out2;
2914 		}
2915 	} else {
2916 		/* Clear any pending interrupts on the HBA */
2917 		writel(readl(dd->mmio + HOST_IRQ_STAT),
2918 			dd->mmio + HOST_IRQ_STAT);
2919 	}
2920 
2921 	mtip_init_port(dd->port);
2922 	mtip_start_port(dd->port);
2923 
2924 	/* Setup the ISR and enable interrupts. */
2925 	rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED,
2926 			 dev_driver_string(&dd->pdev->dev), dd);
2927 	if (rv) {
2928 		dev_err(&dd->pdev->dev,
2929 			"Unable to allocate IRQ %d\n", dd->pdev->irq);
2930 		goto out2;
2931 	}
2932 	irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding));
2933 
2934 	/* Enable interrupts on the HBA. */
2935 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
2936 					dd->mmio + HOST_CTL);
2937 
2938 	init_waitqueue_head(&dd->port->svc_wait);
2939 
2940 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) {
2941 		rv = -EFAULT;
2942 		goto out3;
2943 	}
2944 
2945 	return rv;
2946 
2947 out3:
2948 	/* Disable interrupts on the HBA. */
2949 	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
2950 			dd->mmio + HOST_CTL);
2951 
2952 	/* Release the IRQ. */
2953 	irq_set_affinity_hint(dd->pdev->irq, NULL);
2954 	free_irq(dd->pdev->irq, dd);
2955 
2956 out2:
2957 	mtip_deinit_port(dd->port);
2958 	mtip_dma_free(dd);
2959 
2960 out1:
2961 	/* Free the memory allocated for the for structure. */
2962 	kfree(dd->port);
2963 
2964 	return rv;
2965 }
2966 
2967 static int mtip_standby_drive(struct driver_data *dd)
2968 {
2969 	int rv = 0;
2970 
2971 	if (dd->sr || !dd->port)
2972 		return -ENODEV;
2973 	/*
2974 	 * Send standby immediate (E0h) to the drive so that it
2975 	 * saves its state.
2976 	 */
2977 	if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) &&
2978 	    !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) &&
2979 	    !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) {
2980 		rv = mtip_standby_immediate(dd->port);
2981 		if (rv)
2982 			dev_warn(&dd->pdev->dev,
2983 				"STANDBY IMMEDIATE failed\n");
2984 	}
2985 	return rv;
2986 }
2987 
2988 /*
2989  * Called to deinitialize an interface.
2990  *
2991  * @dd Pointer to the driver data structure.
2992  *
2993  * return value
2994  *	0
2995  */
2996 static int mtip_hw_exit(struct driver_data *dd)
2997 {
2998 	if (!dd->sr) {
2999 		/* de-initialize the port. */
3000 		mtip_deinit_port(dd->port);
3001 
3002 		/* Disable interrupts on the HBA. */
3003 		writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
3004 				dd->mmio + HOST_CTL);
3005 	}
3006 
3007 	/* Release the IRQ. */
3008 	irq_set_affinity_hint(dd->pdev->irq, NULL);
3009 	free_irq(dd->pdev->irq, dd);
3010 	msleep(1000);
3011 
3012 	/* Free dma regions */
3013 	mtip_dma_free(dd);
3014 
3015 	/* Free the memory allocated for the for structure. */
3016 	kfree(dd->port);
3017 	dd->port = NULL;
3018 
3019 	return 0;
3020 }
3021 
3022 /*
3023  * Issue a Standby Immediate command to the device.
3024  *
3025  * This function is called by the Block Layer just before the
3026  * system powers off during a shutdown.
3027  *
3028  * @dd Pointer to the driver data structure.
3029  *
3030  * return value
3031  *	0
3032  */
3033 static int mtip_hw_shutdown(struct driver_data *dd)
3034 {
3035 	/*
3036 	 * Send standby immediate (E0h) to the drive so that it
3037 	 * saves its state.
3038 	 */
3039 	mtip_standby_drive(dd);
3040 
3041 	return 0;
3042 }
3043 
3044 /*
3045  * Suspend function
3046  *
3047  * This function is called by the Block Layer just before the
3048  * system hibernates.
3049  *
3050  * @dd Pointer to the driver data structure.
3051  *
3052  * return value
3053  *	0	Suspend was successful
3054  *	-EFAULT Suspend was not successful
3055  */
3056 static int mtip_hw_suspend(struct driver_data *dd)
3057 {
3058 	/*
3059 	 * Send standby immediate (E0h) to the drive
3060 	 * so that it saves its state.
3061 	 */
3062 	if (mtip_standby_drive(dd) != 0) {
3063 		dev_err(&dd->pdev->dev,
3064 			"Failed standby-immediate command\n");
3065 		return -EFAULT;
3066 	}
3067 
3068 	/* Disable interrupts on the HBA.*/
3069 	writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN,
3070 			dd->mmio + HOST_CTL);
3071 	mtip_deinit_port(dd->port);
3072 
3073 	return 0;
3074 }
3075 
3076 /*
3077  * Resume function
3078  *
3079  * This function is called by the Block Layer as the
3080  * system resumes.
3081  *
3082  * @dd Pointer to the driver data structure.
3083  *
3084  * return value
3085  *	0	Resume was successful
3086  *      -EFAULT Resume was not successful
3087  */
3088 static int mtip_hw_resume(struct driver_data *dd)
3089 {
3090 	/* Perform any needed hardware setup steps */
3091 	hba_setup(dd);
3092 
3093 	/* Reset the HBA */
3094 	if (mtip_hba_reset(dd) != 0) {
3095 		dev_err(&dd->pdev->dev,
3096 			"Unable to reset the HBA\n");
3097 		return -EFAULT;
3098 	}
3099 
3100 	/*
3101 	 * Enable the port, DMA engine, and FIS reception specific
3102 	 * h/w in controller.
3103 	 */
3104 	mtip_init_port(dd->port);
3105 	mtip_start_port(dd->port);
3106 
3107 	/* Enable interrupts on the HBA.*/
3108 	writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN,
3109 			dd->mmio + HOST_CTL);
3110 
3111 	return 0;
3112 }
3113 
3114 /*
3115  * Helper function for reusing disk name
3116  * upon hot insertion.
3117  */
3118 static int rssd_disk_name_format(char *prefix,
3119 				 int index,
3120 				 char *buf,
3121 				 int buflen)
3122 {
3123 	const int base = 'z' - 'a' + 1;
3124 	char *begin = buf + strlen(prefix);
3125 	char *end = buf + buflen;
3126 	char *p;
3127 	int unit;
3128 
3129 	p = end - 1;
3130 	*p = '\0';
3131 	unit = base;
3132 	do {
3133 		if (p == begin)
3134 			return -EINVAL;
3135 		*--p = 'a' + (index % unit);
3136 		index = (index / unit) - 1;
3137 	} while (index >= 0);
3138 
3139 	memmove(begin, p, end - p);
3140 	memcpy(buf, prefix, strlen(prefix));
3141 
3142 	return 0;
3143 }
3144 
3145 /*
3146  * Block layer IOCTL handler.
3147  *
3148  * @dev Pointer to the block_device structure.
3149  * @mode ignored
3150  * @cmd IOCTL command passed from the user application.
3151  * @arg Argument passed from the user application.
3152  *
3153  * return value
3154  *	0        IOCTL completed successfully.
3155  *	-ENOTTY  IOCTL not supported or invalid driver data
3156  *                 structure pointer.
3157  */
3158 static int mtip_block_ioctl(struct block_device *dev,
3159 			    fmode_t mode,
3160 			    unsigned cmd,
3161 			    unsigned long arg)
3162 {
3163 	struct driver_data *dd = dev->bd_disk->private_data;
3164 
3165 	if (!capable(CAP_SYS_ADMIN))
3166 		return -EACCES;
3167 
3168 	if (!dd)
3169 		return -ENOTTY;
3170 
3171 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
3172 		return -ENOTTY;
3173 
3174 	switch (cmd) {
3175 	case BLKFLSBUF:
3176 		return -ENOTTY;
3177 	default:
3178 		return mtip_hw_ioctl(dd, cmd, arg);
3179 	}
3180 }
3181 
3182 #ifdef CONFIG_COMPAT
3183 /*
3184  * Block layer compat IOCTL handler.
3185  *
3186  * @dev Pointer to the block_device structure.
3187  * @mode ignored
3188  * @cmd IOCTL command passed from the user application.
3189  * @arg Argument passed from the user application.
3190  *
3191  * return value
3192  *	0        IOCTL completed successfully.
3193  *	-ENOTTY  IOCTL not supported or invalid driver data
3194  *                 structure pointer.
3195  */
3196 static int mtip_block_compat_ioctl(struct block_device *dev,
3197 			    fmode_t mode,
3198 			    unsigned cmd,
3199 			    unsigned long arg)
3200 {
3201 	struct driver_data *dd = dev->bd_disk->private_data;
3202 
3203 	if (!capable(CAP_SYS_ADMIN))
3204 		return -EACCES;
3205 
3206 	if (!dd)
3207 		return -ENOTTY;
3208 
3209 	if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)))
3210 		return -ENOTTY;
3211 
3212 	switch (cmd) {
3213 	case BLKFLSBUF:
3214 		return -ENOTTY;
3215 	case HDIO_DRIVE_TASKFILE: {
3216 		struct mtip_compat_ide_task_request_s __user *compat_req_task;
3217 		ide_task_request_t req_task;
3218 		int compat_tasksize, outtotal, ret;
3219 
3220 		compat_tasksize =
3221 			sizeof(struct mtip_compat_ide_task_request_s);
3222 
3223 		compat_req_task =
3224 			(struct mtip_compat_ide_task_request_s __user *) arg;
3225 
3226 		if (copy_from_user(&req_task, (void __user *) arg,
3227 			compat_tasksize - (2 * sizeof(compat_long_t))))
3228 			return -EFAULT;
3229 
3230 		if (get_user(req_task.out_size, &compat_req_task->out_size))
3231 			return -EFAULT;
3232 
3233 		if (get_user(req_task.in_size, &compat_req_task->in_size))
3234 			return -EFAULT;
3235 
3236 		outtotal = sizeof(struct mtip_compat_ide_task_request_s);
3237 
3238 		ret = exec_drive_taskfile(dd, (void __user *) arg,
3239 						&req_task, outtotal);
3240 
3241 		if (copy_to_user((void __user *) arg, &req_task,
3242 				compat_tasksize -
3243 				(2 * sizeof(compat_long_t))))
3244 			return -EFAULT;
3245 
3246 		if (put_user(req_task.out_size, &compat_req_task->out_size))
3247 			return -EFAULT;
3248 
3249 		if (put_user(req_task.in_size, &compat_req_task->in_size))
3250 			return -EFAULT;
3251 
3252 		return ret;
3253 	}
3254 	default:
3255 		return mtip_hw_ioctl(dd, cmd, arg);
3256 	}
3257 }
3258 #endif
3259 
3260 /*
3261  * Obtain the geometry of the device.
3262  *
3263  * You may think that this function is obsolete, but some applications,
3264  * fdisk for example still used CHS values. This function describes the
3265  * device as having 224 heads and 56 sectors per cylinder. These values are
3266  * chosen so that each cylinder is aligned on a 4KB boundary. Since a
3267  * partition is described in terms of a start and end cylinder this means
3268  * that each partition is also 4KB aligned. Non-aligned partitions adversely
3269  * affects performance.
3270  *
3271  * @dev Pointer to the block_device strucutre.
3272  * @geo Pointer to a hd_geometry structure.
3273  *
3274  * return value
3275  *	0       Operation completed successfully.
3276  *	-ENOTTY An error occurred while reading the drive capacity.
3277  */
3278 static int mtip_block_getgeo(struct block_device *dev,
3279 				struct hd_geometry *geo)
3280 {
3281 	struct driver_data *dd = dev->bd_disk->private_data;
3282 	sector_t capacity;
3283 
3284 	if (!dd)
3285 		return -ENOTTY;
3286 
3287 	if (!(mtip_hw_get_capacity(dd, &capacity))) {
3288 		dev_warn(&dd->pdev->dev,
3289 			"Could not get drive capacity.\n");
3290 		return -ENOTTY;
3291 	}
3292 
3293 	geo->heads = 224;
3294 	geo->sectors = 56;
3295 	sector_div(capacity, (geo->heads * geo->sectors));
3296 	geo->cylinders = capacity;
3297 	return 0;
3298 }
3299 
3300 static int mtip_block_open(struct block_device *dev, fmode_t mode)
3301 {
3302 	struct driver_data *dd;
3303 
3304 	if (dev && dev->bd_disk) {
3305 		dd = (struct driver_data *) dev->bd_disk->private_data;
3306 
3307 		if (dd) {
3308 			if (test_bit(MTIP_DDF_REMOVAL_BIT,
3309 							&dd->dd_flag)) {
3310 				return -ENODEV;
3311 			}
3312 			return 0;
3313 		}
3314 	}
3315 	return -ENODEV;
3316 }
3317 
3318 static void mtip_block_release(struct gendisk *disk, fmode_t mode)
3319 {
3320 }
3321 
3322 /*
3323  * Block device operation function.
3324  *
3325  * This structure contains pointers to the functions required by the block
3326  * layer.
3327  */
3328 static const struct block_device_operations mtip_block_ops = {
3329 	.open		= mtip_block_open,
3330 	.release	= mtip_block_release,
3331 	.ioctl		= mtip_block_ioctl,
3332 #ifdef CONFIG_COMPAT
3333 	.compat_ioctl	= mtip_block_compat_ioctl,
3334 #endif
3335 	.getgeo		= mtip_block_getgeo,
3336 	.owner		= THIS_MODULE
3337 };
3338 
3339 static inline bool is_se_active(struct driver_data *dd)
3340 {
3341 	if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) {
3342 		if (dd->port->ic_pause_timer) {
3343 			unsigned long to = dd->port->ic_pause_timer +
3344 							msecs_to_jiffies(1000);
3345 			if (time_after(jiffies, to)) {
3346 				clear_bit(MTIP_PF_SE_ACTIVE_BIT,
3347 							&dd->port->flags);
3348 				clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag);
3349 				dd->port->ic_pause_timer = 0;
3350 				wake_up_interruptible(&dd->port->svc_wait);
3351 				return false;
3352 			}
3353 		}
3354 		return true;
3355 	}
3356 	return false;
3357 }
3358 
3359 static inline bool is_stopped(struct driver_data *dd, struct request *rq)
3360 {
3361 	if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO)))
3362 		return false;
3363 
3364 	if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))
3365 		return true;
3366 	if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag))
3367 		return true;
3368 	if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) &&
3369 	    rq_data_dir(rq))
3370 		return true;
3371 	if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag))
3372 		return true;
3373 	if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag))
3374 		return true;
3375 
3376 	return false;
3377 }
3378 
3379 static bool mtip_check_unal_depth(struct blk_mq_hw_ctx *hctx,
3380 				  struct request *rq)
3381 {
3382 	struct driver_data *dd = hctx->queue->queuedata;
3383 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3384 
3385 	if (rq_data_dir(rq) == READ || !dd->unal_qdepth)
3386 		return false;
3387 
3388 	/*
3389 	 * If unaligned depth must be limited on this controller, mark it
3390 	 * as unaligned if the IO isn't on a 4k boundary (start of length).
3391 	 */
3392 	if (blk_rq_sectors(rq) <= 64) {
3393 		if ((blk_rq_pos(rq) & 7) || (blk_rq_sectors(rq) & 7))
3394 			cmd->unaligned = 1;
3395 	}
3396 
3397 	if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0)
3398 		return true;
3399 
3400 	return false;
3401 }
3402 
3403 static blk_status_t mtip_issue_reserved_cmd(struct blk_mq_hw_ctx *hctx,
3404 		struct request *rq)
3405 {
3406 	struct driver_data *dd = hctx->queue->queuedata;
3407 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3408 	struct mtip_int_cmd *icmd = cmd->icmd;
3409 	struct mtip_cmd_hdr *hdr =
3410 		dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag;
3411 	struct mtip_cmd_sg *command_sg;
3412 
3413 	if (mtip_commands_active(dd->port))
3414 		return BLK_STS_DEV_RESOURCE;
3415 
3416 	hdr->ctba = cpu_to_le32(cmd->command_dma & 0xFFFFFFFF);
3417 	if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags))
3418 		hdr->ctbau = cpu_to_le32((cmd->command_dma >> 16) >> 16);
3419 	/* Populate the SG list */
3420 	hdr->opts = cpu_to_le32(icmd->opts | icmd->fis_len);
3421 	if (icmd->buf_len) {
3422 		command_sg = cmd->command + AHCI_CMD_TBL_HDR_SZ;
3423 
3424 		command_sg->info = cpu_to_le32((icmd->buf_len-1) & 0x3FFFFF);
3425 		command_sg->dba	= cpu_to_le32(icmd->buffer & 0xFFFFFFFF);
3426 		command_sg->dba_upper =
3427 			cpu_to_le32((icmd->buffer >> 16) >> 16);
3428 
3429 		hdr->opts |= cpu_to_le32((1 << 16));
3430 	}
3431 
3432 	/* Populate the command header */
3433 	hdr->byte_count = 0;
3434 
3435 	blk_mq_start_request(rq);
3436 	mtip_issue_non_ncq_command(dd->port, rq->tag);
3437 	return 0;
3438 }
3439 
3440 static blk_status_t mtip_queue_rq(struct blk_mq_hw_ctx *hctx,
3441 			 const struct blk_mq_queue_data *bd)
3442 {
3443 	struct driver_data *dd = hctx->queue->queuedata;
3444 	struct request *rq = bd->rq;
3445 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3446 
3447 	if (blk_rq_is_passthrough(rq))
3448 		return mtip_issue_reserved_cmd(hctx, rq);
3449 
3450 	if (unlikely(mtip_check_unal_depth(hctx, rq)))
3451 		return BLK_STS_DEV_RESOURCE;
3452 
3453 	if (is_se_active(dd) || is_stopped(dd, rq))
3454 		return BLK_STS_IOERR;
3455 
3456 	blk_mq_start_request(rq);
3457 
3458 	mtip_hw_submit_io(dd, rq, cmd, hctx);
3459 	return BLK_STS_OK;
3460 }
3461 
3462 static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq,
3463 			  unsigned int hctx_idx)
3464 {
3465 	struct driver_data *dd = set->driver_data;
3466 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3467 
3468 	if (!cmd->command)
3469 		return;
3470 
3471 	dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command,
3472 			  cmd->command_dma);
3473 }
3474 
3475 static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq,
3476 			 unsigned int hctx_idx, unsigned int numa_node)
3477 {
3478 	struct driver_data *dd = set->driver_data;
3479 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3480 
3481 	cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ,
3482 			&cmd->command_dma, GFP_KERNEL);
3483 	if (!cmd->command)
3484 		return -ENOMEM;
3485 
3486 	sg_init_table(cmd->sg, MTIP_MAX_SG);
3487 	return 0;
3488 }
3489 
3490 static enum blk_eh_timer_return mtip_cmd_timeout(struct request *req,
3491 								bool reserved)
3492 {
3493 	struct driver_data *dd = req->q->queuedata;
3494 
3495 	if (reserved) {
3496 		struct mtip_cmd *cmd = blk_mq_rq_to_pdu(req);
3497 
3498 		cmd->status = BLK_STS_TIMEOUT;
3499 		blk_mq_complete_request(req);
3500 		return BLK_EH_DONE;
3501 	}
3502 
3503 	if (test_bit(req->tag, dd->port->cmds_to_issue))
3504 		goto exit_handler;
3505 
3506 	if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags))
3507 		goto exit_handler;
3508 
3509 	wake_up_interruptible(&dd->port->svc_wait);
3510 exit_handler:
3511 	return BLK_EH_RESET_TIMER;
3512 }
3513 
3514 static const struct blk_mq_ops mtip_mq_ops = {
3515 	.queue_rq	= mtip_queue_rq,
3516 	.init_request	= mtip_init_cmd,
3517 	.exit_request	= mtip_free_cmd,
3518 	.complete	= mtip_softirq_done_fn,
3519 	.timeout        = mtip_cmd_timeout,
3520 };
3521 
3522 /*
3523  * Block layer initialization function.
3524  *
3525  * This function is called once by the PCI layer for each P320
3526  * device that is connected to the system.
3527  *
3528  * @dd Pointer to the driver data structure.
3529  *
3530  * return value
3531  *	0 on success else an error code.
3532  */
3533 static int mtip_block_initialize(struct driver_data *dd)
3534 {
3535 	int rv = 0, wait_for_rebuild = 0;
3536 	sector_t capacity;
3537 	unsigned int index = 0;
3538 
3539 	if (dd->disk)
3540 		goto skip_create_disk; /* hw init done, before rebuild */
3541 
3542 	if (mtip_hw_init(dd)) {
3543 		rv = -EINVAL;
3544 		goto protocol_init_error;
3545 	}
3546 
3547 	memset(&dd->tags, 0, sizeof(dd->tags));
3548 	dd->tags.ops = &mtip_mq_ops;
3549 	dd->tags.nr_hw_queues = 1;
3550 	dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS;
3551 	dd->tags.reserved_tags = 1;
3552 	dd->tags.cmd_size = sizeof(struct mtip_cmd);
3553 	dd->tags.numa_node = dd->numa_node;
3554 	dd->tags.flags = BLK_MQ_F_SHOULD_MERGE;
3555 	dd->tags.driver_data = dd;
3556 	dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS;
3557 
3558 	rv = blk_mq_alloc_tag_set(&dd->tags);
3559 	if (rv) {
3560 		dev_err(&dd->pdev->dev,
3561 			"Unable to allocate request queue\n");
3562 		goto block_queue_alloc_tag_error;
3563 	}
3564 
3565 	dd->disk = blk_mq_alloc_disk(&dd->tags, dd);
3566 	if (IS_ERR(dd->disk)) {
3567 		dev_err(&dd->pdev->dev,
3568 			"Unable to allocate request queue\n");
3569 		rv = -ENOMEM;
3570 		goto block_queue_alloc_init_error;
3571 	}
3572 	dd->queue		= dd->disk->queue;
3573 
3574 	rv = ida_alloc(&rssd_index_ida, GFP_KERNEL);
3575 	if (rv < 0)
3576 		goto ida_get_error;
3577 	index = rv;
3578 
3579 	rv = rssd_disk_name_format("rssd",
3580 				index,
3581 				dd->disk->disk_name,
3582 				DISK_NAME_LEN);
3583 	if (rv)
3584 		goto disk_index_error;
3585 
3586 	dd->disk->major		= dd->major;
3587 	dd->disk->first_minor	= index * MTIP_MAX_MINORS;
3588 	dd->disk->minors 	= MTIP_MAX_MINORS;
3589 	dd->disk->fops		= &mtip_block_ops;
3590 	dd->disk->private_data	= dd;
3591 	dd->index		= index;
3592 
3593 	mtip_hw_debugfs_init(dd);
3594 
3595 skip_create_disk:
3596 	/* Initialize the protocol layer. */
3597 	wait_for_rebuild = mtip_hw_get_identify(dd);
3598 	if (wait_for_rebuild < 0) {
3599 		dev_err(&dd->pdev->dev,
3600 			"Protocol layer initialization failed\n");
3601 		rv = -EINVAL;
3602 		goto init_hw_cmds_error;
3603 	}
3604 
3605 	/*
3606 	 * if rebuild pending, start the service thread, and delay the block
3607 	 * queue creation and device_add_disk()
3608 	 */
3609 	if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
3610 		goto start_service_thread;
3611 
3612 	/* Set device limits. */
3613 	blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue);
3614 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue);
3615 	blk_queue_max_segments(dd->queue, MTIP_MAX_SG);
3616 	blk_queue_physical_block_size(dd->queue, 4096);
3617 	blk_queue_max_hw_sectors(dd->queue, 0xffff);
3618 	blk_queue_max_segment_size(dd->queue, 0x400000);
3619 	dma_set_max_seg_size(&dd->pdev->dev, 0x400000);
3620 	blk_queue_io_min(dd->queue, 4096);
3621 
3622 	/* Set the capacity of the device in 512 byte sectors. */
3623 	if (!(mtip_hw_get_capacity(dd, &capacity))) {
3624 		dev_warn(&dd->pdev->dev,
3625 			"Could not read drive capacity\n");
3626 		rv = -EIO;
3627 		goto read_capacity_error;
3628 	}
3629 	set_capacity(dd->disk, capacity);
3630 
3631 	/* Enable the block device and add it to /dev */
3632 	rv = device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups);
3633 	if (rv)
3634 		goto read_capacity_error;
3635 
3636 	if (dd->mtip_svc_handler) {
3637 		set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
3638 		return rv; /* service thread created for handling rebuild */
3639 	}
3640 
3641 start_service_thread:
3642 	dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread,
3643 						dd, dd->numa_node,
3644 						"mtip_svc_thd_%02d", index);
3645 
3646 	if (IS_ERR(dd->mtip_svc_handler)) {
3647 		dev_err(&dd->pdev->dev, "service thread failed to start\n");
3648 		dd->mtip_svc_handler = NULL;
3649 		rv = -EFAULT;
3650 		goto kthread_run_error;
3651 	}
3652 	wake_up_process(dd->mtip_svc_handler);
3653 	if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC)
3654 		rv = wait_for_rebuild;
3655 
3656 	return rv;
3657 
3658 kthread_run_error:
3659 	/* Delete our gendisk. This also removes the device from /dev */
3660 	del_gendisk(dd->disk);
3661 read_capacity_error:
3662 init_hw_cmds_error:
3663 	mtip_hw_debugfs_exit(dd);
3664 disk_index_error:
3665 	ida_free(&rssd_index_ida, index);
3666 ida_get_error:
3667 	blk_cleanup_disk(dd->disk);
3668 block_queue_alloc_init_error:
3669 	blk_mq_free_tag_set(&dd->tags);
3670 block_queue_alloc_tag_error:
3671 	mtip_hw_exit(dd); /* De-initialize the protocol layer. */
3672 protocol_init_error:
3673 	return rv;
3674 }
3675 
3676 static bool mtip_no_dev_cleanup(struct request *rq, void *data, bool reserv)
3677 {
3678 	struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq);
3679 
3680 	cmd->status = BLK_STS_IOERR;
3681 	blk_mq_complete_request(rq);
3682 	return true;
3683 }
3684 
3685 /*
3686  * Block layer deinitialization function.
3687  *
3688  * Called by the PCI layer as each P320 device is removed.
3689  *
3690  * @dd Pointer to the driver data structure.
3691  *
3692  * return value
3693  *	0
3694  */
3695 static int mtip_block_remove(struct driver_data *dd)
3696 {
3697 	mtip_hw_debugfs_exit(dd);
3698 
3699 	if (dd->mtip_svc_handler) {
3700 		set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags);
3701 		wake_up_interruptible(&dd->port->svc_wait);
3702 		kthread_stop(dd->mtip_svc_handler);
3703 	}
3704 
3705 	if (!dd->sr) {
3706 		/*
3707 		 * Explicitly wait here for IOs to quiesce,
3708 		 * as mtip_standby_drive usually won't wait for IOs.
3709 		 */
3710 		if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS))
3711 			mtip_standby_drive(dd);
3712 	}
3713 	else
3714 		dev_info(&dd->pdev->dev, "device %s surprise removal\n",
3715 						dd->disk->disk_name);
3716 
3717 	blk_freeze_queue_start(dd->queue);
3718 	blk_mq_quiesce_queue(dd->queue);
3719 	blk_mq_tagset_busy_iter(&dd->tags, mtip_no_dev_cleanup, dd);
3720 	blk_mq_unquiesce_queue(dd->queue);
3721 
3722 	if (dd->disk) {
3723 		if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3724 			del_gendisk(dd->disk);
3725 		if (dd->disk->queue) {
3726 			blk_cleanup_queue(dd->queue);
3727 			blk_mq_free_tag_set(&dd->tags);
3728 			dd->queue = NULL;
3729 		}
3730 		put_disk(dd->disk);
3731 	}
3732 	dd->disk  = NULL;
3733 
3734 	ida_free(&rssd_index_ida, dd->index);
3735 
3736 	/* De-initialize the protocol layer. */
3737 	mtip_hw_exit(dd);
3738 
3739 	return 0;
3740 }
3741 
3742 /*
3743  * Function called by the PCI layer when just before the
3744  * machine shuts down.
3745  *
3746  * If a protocol layer shutdown function is present it will be called
3747  * by this function.
3748  *
3749  * @dd Pointer to the driver data structure.
3750  *
3751  * return value
3752  *	0
3753  */
3754 static int mtip_block_shutdown(struct driver_data *dd)
3755 {
3756 	mtip_hw_shutdown(dd);
3757 
3758 	/* Delete our gendisk structure, and cleanup the blk queue. */
3759 	if (dd->disk) {
3760 		dev_info(&dd->pdev->dev,
3761 			"Shutting down %s ...\n", dd->disk->disk_name);
3762 
3763 		if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag))
3764 			del_gendisk(dd->disk);
3765 		if (dd->disk->queue) {
3766 			blk_cleanup_queue(dd->queue);
3767 			blk_mq_free_tag_set(&dd->tags);
3768 		}
3769 		put_disk(dd->disk);
3770 		dd->disk  = NULL;
3771 		dd->queue = NULL;
3772 	}
3773 
3774 	ida_free(&rssd_index_ida, dd->index);
3775 	return 0;
3776 }
3777 
3778 static int mtip_block_suspend(struct driver_data *dd)
3779 {
3780 	dev_info(&dd->pdev->dev,
3781 		"Suspending %s ...\n", dd->disk->disk_name);
3782 	mtip_hw_suspend(dd);
3783 	return 0;
3784 }
3785 
3786 static int mtip_block_resume(struct driver_data *dd)
3787 {
3788 	dev_info(&dd->pdev->dev, "Resuming %s ...\n",
3789 		dd->disk->disk_name);
3790 	mtip_hw_resume(dd);
3791 	return 0;
3792 }
3793 
3794 static void drop_cpu(int cpu)
3795 {
3796 	cpu_use[cpu]--;
3797 }
3798 
3799 static int get_least_used_cpu_on_node(int node)
3800 {
3801 	int cpu, least_used_cpu, least_cnt;
3802 	const struct cpumask *node_mask;
3803 
3804 	node_mask = cpumask_of_node(node);
3805 	least_used_cpu = cpumask_first(node_mask);
3806 	least_cnt = cpu_use[least_used_cpu];
3807 	cpu = least_used_cpu;
3808 
3809 	for_each_cpu(cpu, node_mask) {
3810 		if (cpu_use[cpu] < least_cnt) {
3811 			least_used_cpu = cpu;
3812 			least_cnt = cpu_use[cpu];
3813 		}
3814 	}
3815 	cpu_use[least_used_cpu]++;
3816 	return least_used_cpu;
3817 }
3818 
3819 /* Helper for selecting a node in round robin mode */
3820 static inline int mtip_get_next_rr_node(void)
3821 {
3822 	static int next_node = NUMA_NO_NODE;
3823 
3824 	if (next_node == NUMA_NO_NODE) {
3825 		next_node = first_online_node;
3826 		return next_node;
3827 	}
3828 
3829 	next_node = next_online_node(next_node);
3830 	if (next_node == MAX_NUMNODES)
3831 		next_node = first_online_node;
3832 	return next_node;
3833 }
3834 
3835 static DEFINE_HANDLER(0);
3836 static DEFINE_HANDLER(1);
3837 static DEFINE_HANDLER(2);
3838 static DEFINE_HANDLER(3);
3839 static DEFINE_HANDLER(4);
3840 static DEFINE_HANDLER(5);
3841 static DEFINE_HANDLER(6);
3842 static DEFINE_HANDLER(7);
3843 
3844 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev)
3845 {
3846 	unsigned short pcie_dev_ctrl;
3847 
3848 	if (pci_is_pcie(pdev)) {
3849 		pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &pcie_dev_ctrl);
3850 		if (pcie_dev_ctrl & PCI_EXP_DEVCTL_NOSNOOP_EN ||
3851 		    pcie_dev_ctrl & PCI_EXP_DEVCTL_RELAX_EN) {
3852 			dev_info(&dd->pdev->dev,
3853 				"Disabling ERO/No-Snoop on bridge device %04x:%04x\n",
3854 					pdev->vendor, pdev->device);
3855 			pcie_dev_ctrl &= ~(PCI_EXP_DEVCTL_NOSNOOP_EN |
3856 						PCI_EXP_DEVCTL_RELAX_EN);
3857 			pcie_capability_write_word(pdev, PCI_EXP_DEVCTL,
3858 				pcie_dev_ctrl);
3859 		}
3860 	}
3861 }
3862 
3863 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev)
3864 {
3865 	/*
3866 	 * This workaround is specific to AMD/ATI chipset with a PCI upstream
3867 	 * device with device id 0x5aXX
3868 	 */
3869 	if (pdev->bus && pdev->bus->self) {
3870 		if (pdev->bus->self->vendor == PCI_VENDOR_ID_ATI &&
3871 		    ((pdev->bus->self->device & 0xff00) == 0x5a00)) {
3872 			mtip_disable_link_opts(dd, pdev->bus->self);
3873 		} else {
3874 			/* Check further up the topology */
3875 			struct pci_dev *parent_dev = pdev->bus->self;
3876 			if (parent_dev->bus &&
3877 				parent_dev->bus->parent &&
3878 				parent_dev->bus->parent->self &&
3879 				parent_dev->bus->parent->self->vendor ==
3880 					 PCI_VENDOR_ID_ATI &&
3881 				(parent_dev->bus->parent->self->device &
3882 					0xff00) == 0x5a00) {
3883 				mtip_disable_link_opts(dd,
3884 					parent_dev->bus->parent->self);
3885 			}
3886 		}
3887 	}
3888 }
3889 
3890 /*
3891  * Called for each supported PCI device detected.
3892  *
3893  * This function allocates the private data structure, enables the
3894  * PCI device and then calls the block layer initialization function.
3895  *
3896  * return value
3897  *	0 on success else an error code.
3898  */
3899 static int mtip_pci_probe(struct pci_dev *pdev,
3900 			const struct pci_device_id *ent)
3901 {
3902 	int rv = 0;
3903 	struct driver_data *dd = NULL;
3904 	char cpu_list[256];
3905 	const struct cpumask *node_mask;
3906 	int cpu, i = 0, j = 0;
3907 	int my_node = NUMA_NO_NODE;
3908 	unsigned long flags;
3909 
3910 	/* Allocate memory for this devices private data. */
3911 	my_node = pcibus_to_node(pdev->bus);
3912 	if (my_node != NUMA_NO_NODE) {
3913 		if (!node_online(my_node))
3914 			my_node = mtip_get_next_rr_node();
3915 	} else {
3916 		dev_info(&pdev->dev, "Kernel not reporting proximity, choosing a node\n");
3917 		my_node = mtip_get_next_rr_node();
3918 	}
3919 	dev_info(&pdev->dev, "NUMA node %d (closest: %d,%d, probe on %d:%d)\n",
3920 		my_node, pcibus_to_node(pdev->bus), dev_to_node(&pdev->dev),
3921 		cpu_to_node(raw_smp_processor_id()), raw_smp_processor_id());
3922 
3923 	dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node);
3924 	if (!dd)
3925 		return -ENOMEM;
3926 
3927 	/* Attach the private data to this PCI device.  */
3928 	pci_set_drvdata(pdev, dd);
3929 
3930 	rv = pcim_enable_device(pdev);
3931 	if (rv < 0) {
3932 		dev_err(&pdev->dev, "Unable to enable device\n");
3933 		goto iomap_err;
3934 	}
3935 
3936 	/* Map BAR5 to memory. */
3937 	rv = pcim_iomap_regions(pdev, 1 << MTIP_ABAR, MTIP_DRV_NAME);
3938 	if (rv < 0) {
3939 		dev_err(&pdev->dev, "Unable to map regions\n");
3940 		goto iomap_err;
3941 	}
3942 
3943 	rv = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3944 	if (rv) {
3945 		dev_warn(&pdev->dev, "64-bit DMA enable failed\n");
3946 		goto setmask_err;
3947 	}
3948 
3949 	/* Copy the info we may need later into the private data structure. */
3950 	dd->major	= mtip_major;
3951 	dd->instance	= instance;
3952 	dd->pdev	= pdev;
3953 	dd->numa_node	= my_node;
3954 
3955 	INIT_LIST_HEAD(&dd->online_list);
3956 	INIT_LIST_HEAD(&dd->remove_list);
3957 
3958 	memset(dd->workq_name, 0, 32);
3959 	snprintf(dd->workq_name, 31, "mtipq%d", dd->instance);
3960 
3961 	dd->isr_workq = create_workqueue(dd->workq_name);
3962 	if (!dd->isr_workq) {
3963 		dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance);
3964 		rv = -ENOMEM;
3965 		goto setmask_err;
3966 	}
3967 
3968 	memset(cpu_list, 0, sizeof(cpu_list));
3969 
3970 	node_mask = cpumask_of_node(dd->numa_node);
3971 	if (!cpumask_empty(node_mask)) {
3972 		for_each_cpu(cpu, node_mask)
3973 		{
3974 			snprintf(&cpu_list[j], 256 - j, "%d ", cpu);
3975 			j = strlen(cpu_list);
3976 		}
3977 
3978 		dev_info(&pdev->dev, "Node %d on package %d has %d cpu(s): %s\n",
3979 			dd->numa_node,
3980 			topology_physical_package_id(cpumask_first(node_mask)),
3981 			nr_cpus_node(dd->numa_node),
3982 			cpu_list);
3983 	} else
3984 		dev_dbg(&pdev->dev, "mtip32xx: node_mask empty\n");
3985 
3986 	dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node);
3987 	dev_info(&pdev->dev, "Initial IRQ binding node:cpu %d:%d\n",
3988 		cpu_to_node(dd->isr_binding), dd->isr_binding);
3989 
3990 	/* first worker context always runs in ISR */
3991 	dd->work[0].cpu_binding = dd->isr_binding;
3992 	dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
3993 	dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node);
3994 	dd->work[3].cpu_binding = dd->work[0].cpu_binding;
3995 	dd->work[4].cpu_binding = dd->work[1].cpu_binding;
3996 	dd->work[5].cpu_binding = dd->work[2].cpu_binding;
3997 	dd->work[6].cpu_binding = dd->work[2].cpu_binding;
3998 	dd->work[7].cpu_binding = dd->work[1].cpu_binding;
3999 
4000 	/* Log the bindings */
4001 	for_each_present_cpu(cpu) {
4002 		memset(cpu_list, 0, sizeof(cpu_list));
4003 		for (i = 0, j = 0; i < MTIP_MAX_SLOT_GROUPS; i++) {
4004 			if (dd->work[i].cpu_binding == cpu) {
4005 				snprintf(&cpu_list[j], 256 - j, "%d ", i);
4006 				j = strlen(cpu_list);
4007 			}
4008 		}
4009 		if (j)
4010 			dev_info(&pdev->dev, "CPU %d: WQs %s\n", cpu, cpu_list);
4011 	}
4012 
4013 	INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0);
4014 	INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1);
4015 	INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2);
4016 	INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3);
4017 	INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4);
4018 	INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5);
4019 	INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6);
4020 	INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7);
4021 
4022 	pci_set_master(pdev);
4023 	rv = pci_enable_msi(pdev);
4024 	if (rv) {
4025 		dev_warn(&pdev->dev,
4026 			"Unable to enable MSI interrupt.\n");
4027 		goto msi_initialize_err;
4028 	}
4029 
4030 	mtip_fix_ero_nosnoop(dd, pdev);
4031 
4032 	/* Initialize the block layer. */
4033 	rv = mtip_block_initialize(dd);
4034 	if (rv < 0) {
4035 		dev_err(&pdev->dev,
4036 			"Unable to initialize block layer\n");
4037 		goto block_initialize_err;
4038 	}
4039 
4040 	/*
4041 	 * Increment the instance count so that each device has a unique
4042 	 * instance number.
4043 	 */
4044 	instance++;
4045 	if (rv != MTIP_FTL_REBUILD_MAGIC)
4046 		set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag);
4047 	else
4048 		rv = 0; /* device in rebuild state, return 0 from probe */
4049 
4050 	/* Add to online list even if in ftl rebuild */
4051 	spin_lock_irqsave(&dev_lock, flags);
4052 	list_add(&dd->online_list, &online_list);
4053 	spin_unlock_irqrestore(&dev_lock, flags);
4054 
4055 	goto done;
4056 
4057 block_initialize_err:
4058 	pci_disable_msi(pdev);
4059 
4060 msi_initialize_err:
4061 	if (dd->isr_workq) {
4062 		destroy_workqueue(dd->isr_workq);
4063 		drop_cpu(dd->work[0].cpu_binding);
4064 		drop_cpu(dd->work[1].cpu_binding);
4065 		drop_cpu(dd->work[2].cpu_binding);
4066 	}
4067 setmask_err:
4068 	pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
4069 
4070 iomap_err:
4071 	kfree(dd);
4072 	pci_set_drvdata(pdev, NULL);
4073 	return rv;
4074 done:
4075 	return rv;
4076 }
4077 
4078 /*
4079  * Called for each probed device when the device is removed or the
4080  * driver is unloaded.
4081  *
4082  * return value
4083  *	None
4084  */
4085 static void mtip_pci_remove(struct pci_dev *pdev)
4086 {
4087 	struct driver_data *dd = pci_get_drvdata(pdev);
4088 	unsigned long flags, to;
4089 
4090 	set_bit(MTIP_DDF_REMOVAL_BIT, &dd->dd_flag);
4091 
4092 	spin_lock_irqsave(&dev_lock, flags);
4093 	list_del_init(&dd->online_list);
4094 	list_add(&dd->remove_list, &removing_list);
4095 	spin_unlock_irqrestore(&dev_lock, flags);
4096 
4097 	mtip_check_surprise_removal(dd);
4098 	synchronize_irq(dd->pdev->irq);
4099 
4100 	/* Spin until workers are done */
4101 	to = jiffies + msecs_to_jiffies(4000);
4102 	do {
4103 		msleep(20);
4104 	} while (atomic_read(&dd->irq_workers_active) != 0 &&
4105 		time_before(jiffies, to));
4106 
4107 	if (atomic_read(&dd->irq_workers_active) != 0) {
4108 		dev_warn(&dd->pdev->dev,
4109 			"Completion workers still active!\n");
4110 	}
4111 
4112 	blk_mark_disk_dead(dd->disk);
4113 	set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag);
4114 
4115 	/* Clean up the block layer. */
4116 	mtip_block_remove(dd);
4117 
4118 	if (dd->isr_workq) {
4119 		destroy_workqueue(dd->isr_workq);
4120 		drop_cpu(dd->work[0].cpu_binding);
4121 		drop_cpu(dd->work[1].cpu_binding);
4122 		drop_cpu(dd->work[2].cpu_binding);
4123 	}
4124 
4125 	pci_disable_msi(pdev);
4126 
4127 	spin_lock_irqsave(&dev_lock, flags);
4128 	list_del_init(&dd->remove_list);
4129 	spin_unlock_irqrestore(&dev_lock, flags);
4130 
4131 	kfree(dd);
4132 
4133 	pcim_iounmap_regions(pdev, 1 << MTIP_ABAR);
4134 	pci_set_drvdata(pdev, NULL);
4135 }
4136 
4137 /*
4138  * Called for each probed device when the device is suspended.
4139  *
4140  * return value
4141  *	0  Success
4142  *	<0 Error
4143  */
4144 static int __maybe_unused mtip_pci_suspend(struct device *dev)
4145 {
4146 	int rv = 0;
4147 	struct driver_data *dd = dev_get_drvdata(dev);
4148 
4149 	set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
4150 
4151 	/* Disable ports & interrupts then send standby immediate */
4152 	rv = mtip_block_suspend(dd);
4153 	if (rv < 0)
4154 		dev_err(dev, "Failed to suspend controller\n");
4155 
4156 	return rv;
4157 }
4158 
4159 /*
4160  * Called for each probed device when the device is resumed.
4161  *
4162  * return value
4163  *      0  Success
4164  *      <0 Error
4165  */
4166 static int __maybe_unused mtip_pci_resume(struct device *dev)
4167 {
4168 	int rv = 0;
4169 	struct driver_data *dd = dev_get_drvdata(dev);
4170 
4171 	/*
4172 	 * Calls hbaReset, initPort, & startPort function
4173 	 * then enables interrupts
4174 	 */
4175 	rv = mtip_block_resume(dd);
4176 	if (rv < 0)
4177 		dev_err(dev, "Unable to resume\n");
4178 
4179 	clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag);
4180 
4181 	return rv;
4182 }
4183 
4184 /*
4185  * Shutdown routine
4186  *
4187  * return value
4188  *      None
4189  */
4190 static void mtip_pci_shutdown(struct pci_dev *pdev)
4191 {
4192 	struct driver_data *dd = pci_get_drvdata(pdev);
4193 	if (dd)
4194 		mtip_block_shutdown(dd);
4195 }
4196 
4197 /* Table of device ids supported by this driver. */
4198 static const struct pci_device_id mtip_pci_tbl[] = {
4199 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320H_DEVICE_ID) },
4200 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320M_DEVICE_ID) },
4201 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P320S_DEVICE_ID) },
4202 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P325M_DEVICE_ID) },
4203 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420H_DEVICE_ID) },
4204 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P420M_DEVICE_ID) },
4205 	{ PCI_DEVICE(PCI_VENDOR_ID_MICRON, P425M_DEVICE_ID) },
4206 	{ 0 }
4207 };
4208 
4209 static SIMPLE_DEV_PM_OPS(mtip_pci_pm_ops, mtip_pci_suspend, mtip_pci_resume);
4210 
4211 /* Structure that describes the PCI driver functions. */
4212 static struct pci_driver mtip_pci_driver = {
4213 	.name			= MTIP_DRV_NAME,
4214 	.id_table		= mtip_pci_tbl,
4215 	.probe			= mtip_pci_probe,
4216 	.remove			= mtip_pci_remove,
4217 	.driver.pm		= &mtip_pci_pm_ops,
4218 	.shutdown		= mtip_pci_shutdown,
4219 };
4220 
4221 MODULE_DEVICE_TABLE(pci, mtip_pci_tbl);
4222 
4223 /*
4224  * Module initialization function.
4225  *
4226  * Called once when the module is loaded. This function allocates a major
4227  * block device number to the Cyclone devices and registers the PCI layer
4228  * of the driver.
4229  *
4230  * Return value
4231  *      0 on success else error code.
4232  */
4233 static int __init mtip_init(void)
4234 {
4235 	int error;
4236 
4237 	pr_info(MTIP_DRV_NAME " Version " MTIP_DRV_VERSION "\n");
4238 
4239 	/* Allocate a major block device number to use with this driver. */
4240 	error = register_blkdev(0, MTIP_DRV_NAME);
4241 	if (error <= 0) {
4242 		pr_err("Unable to register block device (%d)\n",
4243 		error);
4244 		return -EBUSY;
4245 	}
4246 	mtip_major = error;
4247 
4248 	dfs_parent = debugfs_create_dir("rssd", NULL);
4249 	if (IS_ERR_OR_NULL(dfs_parent)) {
4250 		pr_warn("Error creating debugfs parent\n");
4251 		dfs_parent = NULL;
4252 	}
4253 	if (dfs_parent) {
4254 		dfs_device_status = debugfs_create_file("device_status",
4255 					0444, dfs_parent, NULL,
4256 					&mtip_device_status_fops);
4257 		if (IS_ERR_OR_NULL(dfs_device_status)) {
4258 			pr_err("Error creating device_status node\n");
4259 			dfs_device_status = NULL;
4260 		}
4261 	}
4262 
4263 	/* Register our PCI operations. */
4264 	error = pci_register_driver(&mtip_pci_driver);
4265 	if (error) {
4266 		debugfs_remove(dfs_parent);
4267 		unregister_blkdev(mtip_major, MTIP_DRV_NAME);
4268 	}
4269 
4270 	return error;
4271 }
4272 
4273 /*
4274  * Module de-initialization function.
4275  *
4276  * Called once when the module is unloaded. This function deallocates
4277  * the major block device number allocated by mtip_init() and
4278  * unregisters the PCI layer of the driver.
4279  *
4280  * Return value
4281  *      none
4282  */
4283 static void __exit mtip_exit(void)
4284 {
4285 	/* Release the allocated major block device number. */
4286 	unregister_blkdev(mtip_major, MTIP_DRV_NAME);
4287 
4288 	/* Unregister the PCI driver. */
4289 	pci_unregister_driver(&mtip_pci_driver);
4290 
4291 	debugfs_remove_recursive(dfs_parent);
4292 }
4293 
4294 MODULE_AUTHOR("Micron Technology, Inc");
4295 MODULE_DESCRIPTION("Micron RealSSD PCIe Block Driver");
4296 MODULE_LICENSE("GPL");
4297 MODULE_VERSION(MTIP_DRV_VERSION);
4298 
4299 module_init(mtip_init);
4300 module_exit(mtip_exit);
4301