xref: /openbmc/linux/drivers/scsi/vmw_pvscsi.c (revision 5b448065)
1 /*
2  * Linux driver for VMware's para-virtualized SCSI HBA.
3  *
4  * Copyright (C) 2008-2014, VMware, Inc. All Rights Reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License as published by the
8  * Free Software Foundation; version 2 of the License and no later version.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13  * NON INFRINGEMENT.  See the GNU General Public License for more
14  * details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19  *
20  */
21 
22 #include <linux/kernel.h>
23 #include <linux/module.h>
24 #include <linux/interrupt.h>
25 #include <linux/slab.h>
26 #include <linux/workqueue.h>
27 #include <linux/pci.h>
28 
29 #include <scsi/scsi.h>
30 #include <scsi/scsi_host.h>
31 #include <scsi/scsi_cmnd.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_tcq.h>
34 
35 #include "vmw_pvscsi.h"
36 
37 #define PVSCSI_LINUX_DRIVER_DESC "VMware PVSCSI driver"
38 
39 MODULE_DESCRIPTION(PVSCSI_LINUX_DRIVER_DESC);
40 MODULE_AUTHOR("VMware, Inc.");
41 MODULE_LICENSE("GPL");
42 MODULE_VERSION(PVSCSI_DRIVER_VERSION_STRING);
43 
44 #define PVSCSI_DEFAULT_NUM_PAGES_PER_RING	8
45 #define PVSCSI_DEFAULT_NUM_PAGES_MSG_RING	1
46 #define PVSCSI_DEFAULT_QUEUE_DEPTH		254
47 #define SGL_SIZE				PAGE_SIZE
48 
49 struct pvscsi_sg_list {
50 	struct PVSCSISGElement sge[PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT];
51 };
52 
53 struct pvscsi_ctx {
54 	/*
55 	 * The index of the context in cmd_map serves as the context ID for a
56 	 * 1-to-1 mapping completions back to requests.
57 	 */
58 	struct scsi_cmnd	*cmd;
59 	struct pvscsi_sg_list	*sgl;
60 	struct list_head	list;
61 	dma_addr_t		dataPA;
62 	dma_addr_t		sensePA;
63 	dma_addr_t		sglPA;
64 	struct completion	*abort_cmp;
65 };
66 
67 struct pvscsi_adapter {
68 	char				*mmioBase;
69 	u8				rev;
70 	bool				use_msg;
71 	bool				use_req_threshold;
72 
73 	spinlock_t			hw_lock;
74 
75 	struct workqueue_struct		*workqueue;
76 	struct work_struct		work;
77 
78 	struct PVSCSIRingReqDesc	*req_ring;
79 	unsigned			req_pages;
80 	unsigned			req_depth;
81 	dma_addr_t			reqRingPA;
82 
83 	struct PVSCSIRingCmpDesc	*cmp_ring;
84 	unsigned			cmp_pages;
85 	dma_addr_t			cmpRingPA;
86 
87 	struct PVSCSIRingMsgDesc	*msg_ring;
88 	unsigned			msg_pages;
89 	dma_addr_t			msgRingPA;
90 
91 	struct PVSCSIRingsState		*rings_state;
92 	dma_addr_t			ringStatePA;
93 
94 	struct pci_dev			*dev;
95 	struct Scsi_Host		*host;
96 
97 	struct list_head		cmd_pool;
98 	struct pvscsi_ctx		*cmd_map;
99 };
100 
101 
102 /* Command line parameters */
103 static int pvscsi_ring_pages;
104 static int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
105 static int pvscsi_cmd_per_lun    = PVSCSI_DEFAULT_QUEUE_DEPTH;
106 static bool pvscsi_disable_msi;
107 static bool pvscsi_disable_msix;
108 static bool pvscsi_use_msg       = true;
109 static bool pvscsi_use_req_threshold = true;
110 
111 #define PVSCSI_RW (S_IRUSR | S_IWUSR)
112 
113 module_param_named(ring_pages, pvscsi_ring_pages, int, PVSCSI_RW);
114 MODULE_PARM_DESC(ring_pages, "Number of pages per req/cmp ring - (default="
115 		 __stringify(PVSCSI_DEFAULT_NUM_PAGES_PER_RING)
116 		 "[up to 16 targets],"
117 		 __stringify(PVSCSI_SETUP_RINGS_MAX_NUM_PAGES)
118 		 "[for 16+ targets])");
119 
120 module_param_named(msg_ring_pages, pvscsi_msg_ring_pages, int, PVSCSI_RW);
121 MODULE_PARM_DESC(msg_ring_pages, "Number of pages for the msg ring - (default="
122 		 __stringify(PVSCSI_DEFAULT_NUM_PAGES_MSG_RING) ")");
123 
124 module_param_named(cmd_per_lun, pvscsi_cmd_per_lun, int, PVSCSI_RW);
125 MODULE_PARM_DESC(cmd_per_lun, "Maximum commands per lun - (default="
126 		 __stringify(PVSCSI_DEFAULT_QUEUE_DEPTH) ")");
127 
128 module_param_named(disable_msi, pvscsi_disable_msi, bool, PVSCSI_RW);
129 MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)");
130 
131 module_param_named(disable_msix, pvscsi_disable_msix, bool, PVSCSI_RW);
132 MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)");
133 
134 module_param_named(use_msg, pvscsi_use_msg, bool, PVSCSI_RW);
135 MODULE_PARM_DESC(use_msg, "Use msg ring when available - (default=1)");
136 
137 module_param_named(use_req_threshold, pvscsi_use_req_threshold,
138 		   bool, PVSCSI_RW);
139 MODULE_PARM_DESC(use_req_threshold, "Use driver-based request coalescing if configured - (default=1)");
140 
141 static const struct pci_device_id pvscsi_pci_tbl[] = {
142 	{ PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_PVSCSI) },
143 	{ 0 }
144 };
145 
146 MODULE_DEVICE_TABLE(pci, pvscsi_pci_tbl);
147 
148 static struct device *
149 pvscsi_dev(const struct pvscsi_adapter *adapter)
150 {
151 	return &(adapter->dev->dev);
152 }
153 
154 static struct pvscsi_ctx *
155 pvscsi_find_context(const struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)
156 {
157 	struct pvscsi_ctx *ctx, *end;
158 
159 	end = &adapter->cmd_map[adapter->req_depth];
160 	for (ctx = adapter->cmd_map; ctx < end; ctx++)
161 		if (ctx->cmd == cmd)
162 			return ctx;
163 
164 	return NULL;
165 }
166 
167 static struct pvscsi_ctx *
168 pvscsi_acquire_context(struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd)
169 {
170 	struct pvscsi_ctx *ctx;
171 
172 	if (list_empty(&adapter->cmd_pool))
173 		return NULL;
174 
175 	ctx = list_first_entry(&adapter->cmd_pool, struct pvscsi_ctx, list);
176 	ctx->cmd = cmd;
177 	list_del(&ctx->list);
178 
179 	return ctx;
180 }
181 
182 static void pvscsi_release_context(struct pvscsi_adapter *adapter,
183 				   struct pvscsi_ctx *ctx)
184 {
185 	ctx->cmd = NULL;
186 	ctx->abort_cmp = NULL;
187 	list_add(&ctx->list, &adapter->cmd_pool);
188 }
189 
190 /*
191  * Map a pvscsi_ctx struct to a context ID field value; we map to a simple
192  * non-zero integer. ctx always points to an entry in cmd_map array, hence
193  * the return value is always >=1.
194  */
195 static u64 pvscsi_map_context(const struct pvscsi_adapter *adapter,
196 			      const struct pvscsi_ctx *ctx)
197 {
198 	return ctx - adapter->cmd_map + 1;
199 }
200 
201 static struct pvscsi_ctx *
202 pvscsi_get_context(const struct pvscsi_adapter *adapter, u64 context)
203 {
204 	return &adapter->cmd_map[context - 1];
205 }
206 
207 static void pvscsi_reg_write(const struct pvscsi_adapter *adapter,
208 			     u32 offset, u32 val)
209 {
210 	writel(val, adapter->mmioBase + offset);
211 }
212 
213 static u32 pvscsi_reg_read(const struct pvscsi_adapter *adapter, u32 offset)
214 {
215 	return readl(adapter->mmioBase + offset);
216 }
217 
218 static u32 pvscsi_read_intr_status(const struct pvscsi_adapter *adapter)
219 {
220 	return pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_INTR_STATUS);
221 }
222 
223 static void pvscsi_write_intr_status(const struct pvscsi_adapter *adapter,
224 				     u32 val)
225 {
226 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_STATUS, val);
227 }
228 
229 static void pvscsi_unmask_intr(const struct pvscsi_adapter *adapter)
230 {
231 	u32 intr_bits;
232 
233 	intr_bits = PVSCSI_INTR_CMPL_MASK;
234 	if (adapter->use_msg)
235 		intr_bits |= PVSCSI_INTR_MSG_MASK;
236 
237 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, intr_bits);
238 }
239 
240 static void pvscsi_mask_intr(const struct pvscsi_adapter *adapter)
241 {
242 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, 0);
243 }
244 
245 static void pvscsi_write_cmd_desc(const struct pvscsi_adapter *adapter,
246 				  u32 cmd, const void *desc, size_t len)
247 {
248 	const u32 *ptr = desc;
249 	size_t i;
250 
251 	len /= sizeof(*ptr);
252 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, cmd);
253 	for (i = 0; i < len; i++)
254 		pvscsi_reg_write(adapter,
255 				 PVSCSI_REG_OFFSET_COMMAND_DATA, ptr[i]);
256 }
257 
258 static void pvscsi_abort_cmd(const struct pvscsi_adapter *adapter,
259 			     const struct pvscsi_ctx *ctx)
260 {
261 	struct PVSCSICmdDescAbortCmd cmd = { 0 };
262 
263 	cmd.target = ctx->cmd->device->id;
264 	cmd.context = pvscsi_map_context(adapter, ctx);
265 
266 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd));
267 }
268 
269 static void pvscsi_kick_rw_io(const struct pvscsi_adapter *adapter)
270 {
271 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
272 }
273 
274 static void pvscsi_process_request_ring(const struct pvscsi_adapter *adapter)
275 {
276 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
277 }
278 
279 static int scsi_is_rw(unsigned char op)
280 {
281 	return op == READ_6  || op == WRITE_6 ||
282 	       op == READ_10 || op == WRITE_10 ||
283 	       op == READ_12 || op == WRITE_12 ||
284 	       op == READ_16 || op == WRITE_16;
285 }
286 
287 static void pvscsi_kick_io(const struct pvscsi_adapter *adapter,
288 			   unsigned char op)
289 {
290 	if (scsi_is_rw(op)) {
291 		struct PVSCSIRingsState *s = adapter->rings_state;
292 
293 		if (!adapter->use_req_threshold ||
294 		    s->reqProdIdx - s->reqConsIdx >= s->reqCallThreshold)
295 			pvscsi_kick_rw_io(adapter);
296 	} else {
297 		pvscsi_process_request_ring(adapter);
298 	}
299 }
300 
301 static void ll_adapter_reset(const struct pvscsi_adapter *adapter)
302 {
303 	dev_dbg(pvscsi_dev(adapter), "Adapter Reset on %p\n", adapter);
304 
305 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
306 }
307 
308 static void ll_bus_reset(const struct pvscsi_adapter *adapter)
309 {
310 	dev_dbg(pvscsi_dev(adapter), "Resetting bus on %p\n", adapter);
311 
312 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_BUS, NULL, 0);
313 }
314 
315 static void ll_device_reset(const struct pvscsi_adapter *adapter, u32 target)
316 {
317 	struct PVSCSICmdDescResetDevice cmd = { 0 };
318 
319 	dev_dbg(pvscsi_dev(adapter), "Resetting device: target=%u\n", target);
320 
321 	cmd.target = target;
322 
323 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_DEVICE,
324 			      &cmd, sizeof(cmd));
325 }
326 
327 static void pvscsi_create_sg(struct pvscsi_ctx *ctx,
328 			     struct scatterlist *sg, unsigned count)
329 {
330 	unsigned i;
331 	struct PVSCSISGElement *sge;
332 
333 	BUG_ON(count > PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT);
334 
335 	sge = &ctx->sgl->sge[0];
336 	for (i = 0; i < count; i++, sg = sg_next(sg)) {
337 		sge[i].addr   = sg_dma_address(sg);
338 		sge[i].length = sg_dma_len(sg);
339 		sge[i].flags  = 0;
340 	}
341 }
342 
343 /*
344  * Map all data buffers for a command into PCI space and
345  * setup the scatter/gather list if needed.
346  */
347 static int pvscsi_map_buffers(struct pvscsi_adapter *adapter,
348 			      struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd,
349 			      struct PVSCSIRingReqDesc *e)
350 {
351 	unsigned count;
352 	unsigned bufflen = scsi_bufflen(cmd);
353 	struct scatterlist *sg;
354 
355 	e->dataLen = bufflen;
356 	e->dataAddr = 0;
357 	if (bufflen == 0)
358 		return 0;
359 
360 	sg = scsi_sglist(cmd);
361 	count = scsi_sg_count(cmd);
362 	if (count != 0) {
363 		int segs = scsi_dma_map(cmd);
364 
365 		if (segs == -ENOMEM) {
366 			scmd_printk(KERN_DEBUG, cmd,
367 				    "vmw_pvscsi: Failed to map cmd sglist for DMA.\n");
368 			return -ENOMEM;
369 		} else if (segs > 1) {
370 			pvscsi_create_sg(ctx, sg, segs);
371 
372 			e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
373 			ctx->sglPA = dma_map_single(&adapter->dev->dev,
374 					ctx->sgl, SGL_SIZE, DMA_TO_DEVICE);
375 			if (dma_mapping_error(&adapter->dev->dev, ctx->sglPA)) {
376 				scmd_printk(KERN_ERR, cmd,
377 					    "vmw_pvscsi: Failed to map ctx sglist for DMA.\n");
378 				scsi_dma_unmap(cmd);
379 				ctx->sglPA = 0;
380 				return -ENOMEM;
381 			}
382 			e->dataAddr = ctx->sglPA;
383 		} else
384 			e->dataAddr = sg_dma_address(sg);
385 	} else {
386 		/*
387 		 * In case there is no S/G list, scsi_sglist points
388 		 * directly to the buffer.
389 		 */
390 		ctx->dataPA = dma_map_single(&adapter->dev->dev, sg, bufflen,
391 					     cmd->sc_data_direction);
392 		if (dma_mapping_error(&adapter->dev->dev, ctx->dataPA)) {
393 			scmd_printk(KERN_DEBUG, cmd,
394 				    "vmw_pvscsi: Failed to map direct data buffer for DMA.\n");
395 			return -ENOMEM;
396 		}
397 		e->dataAddr = ctx->dataPA;
398 	}
399 
400 	return 0;
401 }
402 
403 /*
404  * The device incorrectly doesn't clear the first byte of the sense
405  * buffer in some cases. We have to do it ourselves.
406  * Otherwise we run into trouble when SWIOTLB is forced.
407  */
408 static void pvscsi_patch_sense(struct scsi_cmnd *cmd)
409 {
410 	if (cmd->sense_buffer)
411 		cmd->sense_buffer[0] = 0;
412 }
413 
414 static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter,
415 				 struct pvscsi_ctx *ctx)
416 {
417 	struct scsi_cmnd *cmd;
418 	unsigned bufflen;
419 
420 	cmd = ctx->cmd;
421 	bufflen = scsi_bufflen(cmd);
422 
423 	if (bufflen != 0) {
424 		unsigned count = scsi_sg_count(cmd);
425 
426 		if (count != 0) {
427 			scsi_dma_unmap(cmd);
428 			if (ctx->sglPA) {
429 				dma_unmap_single(&adapter->dev->dev, ctx->sglPA,
430 						 SGL_SIZE, DMA_TO_DEVICE);
431 				ctx->sglPA = 0;
432 			}
433 		} else
434 			dma_unmap_single(&adapter->dev->dev, ctx->dataPA,
435 					 bufflen, cmd->sc_data_direction);
436 	}
437 	if (cmd->sense_buffer)
438 		dma_unmap_single(&adapter->dev->dev, ctx->sensePA,
439 				 SCSI_SENSE_BUFFERSIZE, DMA_FROM_DEVICE);
440 }
441 
442 static int pvscsi_allocate_rings(struct pvscsi_adapter *adapter)
443 {
444 	adapter->rings_state = dma_alloc_coherent(&adapter->dev->dev, PAGE_SIZE,
445 			&adapter->ringStatePA, GFP_KERNEL);
446 	if (!adapter->rings_state)
447 		return -ENOMEM;
448 
449 	adapter->req_pages = min(PVSCSI_MAX_NUM_PAGES_REQ_RING,
450 				 pvscsi_ring_pages);
451 	adapter->req_depth = adapter->req_pages
452 					* PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
453 	adapter->req_ring = dma_alloc_coherent(&adapter->dev->dev,
454 			adapter->req_pages * PAGE_SIZE, &adapter->reqRingPA,
455 			GFP_KERNEL);
456 	if (!adapter->req_ring)
457 		return -ENOMEM;
458 
459 	adapter->cmp_pages = min(PVSCSI_MAX_NUM_PAGES_CMP_RING,
460 				 pvscsi_ring_pages);
461 	adapter->cmp_ring = dma_alloc_coherent(&adapter->dev->dev,
462 			adapter->cmp_pages * PAGE_SIZE, &adapter->cmpRingPA,
463 			GFP_KERNEL);
464 	if (!adapter->cmp_ring)
465 		return -ENOMEM;
466 
467 	BUG_ON(!IS_ALIGNED(adapter->ringStatePA, PAGE_SIZE));
468 	BUG_ON(!IS_ALIGNED(adapter->reqRingPA, PAGE_SIZE));
469 	BUG_ON(!IS_ALIGNED(adapter->cmpRingPA, PAGE_SIZE));
470 
471 	if (!adapter->use_msg)
472 		return 0;
473 
474 	adapter->msg_pages = min(PVSCSI_MAX_NUM_PAGES_MSG_RING,
475 				 pvscsi_msg_ring_pages);
476 	adapter->msg_ring = dma_alloc_coherent(&adapter->dev->dev,
477 			adapter->msg_pages * PAGE_SIZE, &adapter->msgRingPA,
478 			GFP_KERNEL);
479 	if (!adapter->msg_ring)
480 		return -ENOMEM;
481 	BUG_ON(!IS_ALIGNED(adapter->msgRingPA, PAGE_SIZE));
482 
483 	return 0;
484 }
485 
486 static void pvscsi_setup_all_rings(const struct pvscsi_adapter *adapter)
487 {
488 	struct PVSCSICmdDescSetupRings cmd = { 0 };
489 	dma_addr_t base;
490 	unsigned i;
491 
492 	cmd.ringsStatePPN   = adapter->ringStatePA >> PAGE_SHIFT;
493 	cmd.reqRingNumPages = adapter->req_pages;
494 	cmd.cmpRingNumPages = adapter->cmp_pages;
495 
496 	base = adapter->reqRingPA;
497 	for (i = 0; i < adapter->req_pages; i++) {
498 		cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
499 		base += PAGE_SIZE;
500 	}
501 
502 	base = adapter->cmpRingPA;
503 	for (i = 0; i < adapter->cmp_pages; i++) {
504 		cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
505 		base += PAGE_SIZE;
506 	}
507 
508 	memset(adapter->rings_state, 0, PAGE_SIZE);
509 	memset(adapter->req_ring, 0, adapter->req_pages * PAGE_SIZE);
510 	memset(adapter->cmp_ring, 0, adapter->cmp_pages * PAGE_SIZE);
511 
512 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_RINGS,
513 			      &cmd, sizeof(cmd));
514 
515 	if (adapter->use_msg) {
516 		struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
517 
518 		cmd_msg.numPages = adapter->msg_pages;
519 
520 		base = adapter->msgRingPA;
521 		for (i = 0; i < adapter->msg_pages; i++) {
522 			cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
523 			base += PAGE_SIZE;
524 		}
525 		memset(adapter->msg_ring, 0, adapter->msg_pages * PAGE_SIZE);
526 
527 		pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_MSG_RING,
528 				      &cmd_msg, sizeof(cmd_msg));
529 	}
530 }
531 
532 static int pvscsi_change_queue_depth(struct scsi_device *sdev, int qdepth)
533 {
534 	if (!sdev->tagged_supported)
535 		qdepth = 1;
536 	return scsi_change_queue_depth(sdev, qdepth);
537 }
538 
539 /*
540  * Pull a completion descriptor off and pass the completion back
541  * to the SCSI mid layer.
542  */
543 static void pvscsi_complete_request(struct pvscsi_adapter *adapter,
544 				    const struct PVSCSIRingCmpDesc *e)
545 {
546 	struct pvscsi_ctx *ctx;
547 	struct scsi_cmnd *cmd;
548 	struct completion *abort_cmp;
549 	u32 btstat = e->hostStatus;
550 	u32 sdstat = e->scsiStatus;
551 
552 	ctx = pvscsi_get_context(adapter, e->context);
553 	cmd = ctx->cmd;
554 	abort_cmp = ctx->abort_cmp;
555 	pvscsi_unmap_buffers(adapter, ctx);
556 	if (sdstat != SAM_STAT_CHECK_CONDITION)
557 		pvscsi_patch_sense(cmd);
558 	pvscsi_release_context(adapter, ctx);
559 	if (abort_cmp) {
560 		/*
561 		 * The command was requested to be aborted. Just signal that
562 		 * the request completed and swallow the actual cmd completion
563 		 * here. The abort handler will post a completion for this
564 		 * command indicating that it got successfully aborted.
565 		 */
566 		complete(abort_cmp);
567 		return;
568 	}
569 
570 	cmd->result = 0;
571 	if (sdstat != SAM_STAT_GOOD &&
572 	    (btstat == BTSTAT_SUCCESS ||
573 	     btstat == BTSTAT_LINKED_COMMAND_COMPLETED ||
574 	     btstat == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG)) {
575 		if (sdstat == SAM_STAT_COMMAND_TERMINATED) {
576 			cmd->result = (DID_RESET << 16);
577 		} else {
578 			cmd->result = (DID_OK << 16) | sdstat;
579 			if (sdstat == SAM_STAT_CHECK_CONDITION &&
580 			    cmd->sense_buffer)
581 				cmd->result |= (DRIVER_SENSE << 24);
582 		}
583 	} else
584 		switch (btstat) {
585 		case BTSTAT_SUCCESS:
586 		case BTSTAT_LINKED_COMMAND_COMPLETED:
587 		case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
588 			/*
589 			 * Commands like INQUIRY may transfer less data than
590 			 * requested by the initiator via bufflen. Set residual
591 			 * count to make upper layer aware of the actual amount
592 			 * of data returned.
593 			 */
594 			scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
595 			cmd->result = (DID_OK << 16);
596 			break;
597 
598 		case BTSTAT_DATARUN:
599 		case BTSTAT_DATA_UNDERRUN:
600 			/* Report residual data in underruns */
601 			scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen);
602 			cmd->result = (DID_ERROR << 16);
603 			break;
604 
605 		case BTSTAT_SELTIMEO:
606 			/* Our emulation returns this for non-connected devs */
607 			cmd->result = (DID_BAD_TARGET << 16);
608 			break;
609 
610 		case BTSTAT_LUNMISMATCH:
611 		case BTSTAT_TAGREJECT:
612 		case BTSTAT_BADMSG:
613 			cmd->result = (DRIVER_INVALID << 24);
614 			fallthrough;
615 
616 		case BTSTAT_HAHARDWARE:
617 		case BTSTAT_INVPHASE:
618 		case BTSTAT_HATIMEOUT:
619 		case BTSTAT_NORESPONSE:
620 		case BTSTAT_DISCONNECT:
621 		case BTSTAT_HASOFTWARE:
622 		case BTSTAT_BUSFREE:
623 		case BTSTAT_SENSFAILED:
624 			cmd->result |= (DID_ERROR << 16);
625 			break;
626 
627 		case BTSTAT_SENTRST:
628 		case BTSTAT_RECVRST:
629 		case BTSTAT_BUSRESET:
630 			cmd->result = (DID_RESET << 16);
631 			break;
632 
633 		case BTSTAT_ABORTQUEUE:
634 			cmd->result = (DID_BUS_BUSY << 16);
635 			break;
636 
637 		case BTSTAT_SCSIPARITY:
638 			cmd->result = (DID_PARITY << 16);
639 			break;
640 
641 		default:
642 			cmd->result = (DID_ERROR << 16);
643 			scmd_printk(KERN_DEBUG, cmd,
644 				    "Unknown completion status: 0x%x\n",
645 				    btstat);
646 	}
647 
648 	dev_dbg(&cmd->device->sdev_gendev,
649 		"cmd=%p %x ctx=%p result=0x%x status=0x%x,%x\n",
650 		cmd, cmd->cmnd[0], ctx, cmd->result, btstat, sdstat);
651 
652 	cmd->scsi_done(cmd);
653 }
654 
655 /*
656  * barrier usage : Since the PVSCSI device is emulated, there could be cases
657  * where we may want to serialize some accesses between the driver and the
658  * emulation layer. We use compiler barriers instead of the more expensive
659  * memory barriers because PVSCSI is only supported on X86 which has strong
660  * memory access ordering.
661  */
662 static void pvscsi_process_completion_ring(struct pvscsi_adapter *adapter)
663 {
664 	struct PVSCSIRingsState *s = adapter->rings_state;
665 	struct PVSCSIRingCmpDesc *ring = adapter->cmp_ring;
666 	u32 cmp_entries = s->cmpNumEntriesLog2;
667 
668 	while (s->cmpConsIdx != s->cmpProdIdx) {
669 		struct PVSCSIRingCmpDesc *e = ring + (s->cmpConsIdx &
670 						      MASK(cmp_entries));
671 		/*
672 		 * This barrier() ensures that *e is not dereferenced while
673 		 * the device emulation still writes data into the slot.
674 		 * Since the device emulation advances s->cmpProdIdx only after
675 		 * updating the slot we want to check it first.
676 		 */
677 		barrier();
678 		pvscsi_complete_request(adapter, e);
679 		/*
680 		 * This barrier() ensures that compiler doesn't reorder write
681 		 * to s->cmpConsIdx before the read of (*e) inside
682 		 * pvscsi_complete_request. Otherwise, device emulation may
683 		 * overwrite *e before we had a chance to read it.
684 		 */
685 		barrier();
686 		s->cmpConsIdx++;
687 	}
688 }
689 
690 /*
691  * Translate a Linux SCSI request into a request ring entry.
692  */
693 static int pvscsi_queue_ring(struct pvscsi_adapter *adapter,
694 			     struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd)
695 {
696 	struct PVSCSIRingsState *s;
697 	struct PVSCSIRingReqDesc *e;
698 	struct scsi_device *sdev;
699 	u32 req_entries;
700 
701 	s = adapter->rings_state;
702 	sdev = cmd->device;
703 	req_entries = s->reqNumEntriesLog2;
704 
705 	/*
706 	 * If this condition holds, we might have room on the request ring, but
707 	 * we might not have room on the completion ring for the response.
708 	 * However, we have already ruled out this possibility - we would not
709 	 * have successfully allocated a context if it were true, since we only
710 	 * have one context per request entry.  Check for it anyway, since it
711 	 * would be a serious bug.
712 	 */
713 	if (s->reqProdIdx - s->cmpConsIdx >= 1 << req_entries) {
714 		scmd_printk(KERN_ERR, cmd, "vmw_pvscsi: "
715 			    "ring full: reqProdIdx=%d cmpConsIdx=%d\n",
716 			    s->reqProdIdx, s->cmpConsIdx);
717 		return -1;
718 	}
719 
720 	e = adapter->req_ring + (s->reqProdIdx & MASK(req_entries));
721 
722 	e->bus    = sdev->channel;
723 	e->target = sdev->id;
724 	memset(e->lun, 0, sizeof(e->lun));
725 	e->lun[1] = sdev->lun;
726 
727 	if (cmd->sense_buffer) {
728 		ctx->sensePA = dma_map_single(&adapter->dev->dev,
729 				cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE,
730 				DMA_FROM_DEVICE);
731 		if (dma_mapping_error(&adapter->dev->dev, ctx->sensePA)) {
732 			scmd_printk(KERN_DEBUG, cmd,
733 				    "vmw_pvscsi: Failed to map sense buffer for DMA.\n");
734 			ctx->sensePA = 0;
735 			return -ENOMEM;
736 		}
737 		e->senseAddr = ctx->sensePA;
738 		e->senseLen = SCSI_SENSE_BUFFERSIZE;
739 	} else {
740 		e->senseLen  = 0;
741 		e->senseAddr = 0;
742 	}
743 	e->cdbLen   = cmd->cmd_len;
744 	e->vcpuHint = smp_processor_id();
745 	memcpy(e->cdb, cmd->cmnd, e->cdbLen);
746 
747 	e->tag = SIMPLE_QUEUE_TAG;
748 
749 	if (cmd->sc_data_direction == DMA_FROM_DEVICE)
750 		e->flags = PVSCSI_FLAG_CMD_DIR_TOHOST;
751 	else if (cmd->sc_data_direction == DMA_TO_DEVICE)
752 		e->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE;
753 	else if (cmd->sc_data_direction == DMA_NONE)
754 		e->flags = PVSCSI_FLAG_CMD_DIR_NONE;
755 	else
756 		e->flags = 0;
757 
758 	if (pvscsi_map_buffers(adapter, ctx, cmd, e) != 0) {
759 		if (cmd->sense_buffer) {
760 			dma_unmap_single(&adapter->dev->dev, ctx->sensePA,
761 					 SCSI_SENSE_BUFFERSIZE,
762 					 DMA_FROM_DEVICE);
763 			ctx->sensePA = 0;
764 		}
765 		return -ENOMEM;
766 	}
767 
768 	e->context = pvscsi_map_context(adapter, ctx);
769 
770 	barrier();
771 
772 	s->reqProdIdx++;
773 
774 	return 0;
775 }
776 
777 static int pvscsi_queue_lck(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *))
778 {
779 	struct Scsi_Host *host = cmd->device->host;
780 	struct pvscsi_adapter *adapter = shost_priv(host);
781 	struct pvscsi_ctx *ctx;
782 	unsigned long flags;
783 	unsigned char op;
784 
785 	spin_lock_irqsave(&adapter->hw_lock, flags);
786 
787 	ctx = pvscsi_acquire_context(adapter, cmd);
788 	if (!ctx || pvscsi_queue_ring(adapter, ctx, cmd) != 0) {
789 		if (ctx)
790 			pvscsi_release_context(adapter, ctx);
791 		spin_unlock_irqrestore(&adapter->hw_lock, flags);
792 		return SCSI_MLQUEUE_HOST_BUSY;
793 	}
794 
795 	cmd->scsi_done = done;
796 	op = cmd->cmnd[0];
797 
798 	dev_dbg(&cmd->device->sdev_gendev,
799 		"queued cmd %p, ctx %p, op=%x\n", cmd, ctx, op);
800 
801 	spin_unlock_irqrestore(&adapter->hw_lock, flags);
802 
803 	pvscsi_kick_io(adapter, op);
804 
805 	return 0;
806 }
807 
808 static DEF_SCSI_QCMD(pvscsi_queue)
809 
810 static int pvscsi_abort(struct scsi_cmnd *cmd)
811 {
812 	struct pvscsi_adapter *adapter = shost_priv(cmd->device->host);
813 	struct pvscsi_ctx *ctx;
814 	unsigned long flags;
815 	int result = SUCCESS;
816 	DECLARE_COMPLETION_ONSTACK(abort_cmp);
817 	int done;
818 
819 	scmd_printk(KERN_DEBUG, cmd, "task abort on host %u, %p\n",
820 		    adapter->host->host_no, cmd);
821 
822 	spin_lock_irqsave(&adapter->hw_lock, flags);
823 
824 	/*
825 	 * Poll the completion ring first - we might be trying to abort
826 	 * a command that is waiting to be dispatched in the completion ring.
827 	 */
828 	pvscsi_process_completion_ring(adapter);
829 
830 	/*
831 	 * If there is no context for the command, it either already succeeded
832 	 * or else was never properly issued.  Not our problem.
833 	 */
834 	ctx = pvscsi_find_context(adapter, cmd);
835 	if (!ctx) {
836 		scmd_printk(KERN_DEBUG, cmd, "Failed to abort cmd %p\n", cmd);
837 		goto out;
838 	}
839 
840 	/*
841 	 * Mark that the command has been requested to be aborted and issue
842 	 * the abort.
843 	 */
844 	ctx->abort_cmp = &abort_cmp;
845 
846 	pvscsi_abort_cmd(adapter, ctx);
847 	spin_unlock_irqrestore(&adapter->hw_lock, flags);
848 	/* Wait for 2 secs for the completion. */
849 	done = wait_for_completion_timeout(&abort_cmp, msecs_to_jiffies(2000));
850 	spin_lock_irqsave(&adapter->hw_lock, flags);
851 
852 	if (!done) {
853 		/*
854 		 * Failed to abort the command, unmark the fact that it
855 		 * was requested to be aborted.
856 		 */
857 		ctx->abort_cmp = NULL;
858 		result = FAILED;
859 		scmd_printk(KERN_DEBUG, cmd,
860 			    "Failed to get completion for aborted cmd %p\n",
861 			    cmd);
862 		goto out;
863 	}
864 
865 	/*
866 	 * Successfully aborted the command.
867 	 */
868 	cmd->result = (DID_ABORT << 16);
869 	cmd->scsi_done(cmd);
870 
871 out:
872 	spin_unlock_irqrestore(&adapter->hw_lock, flags);
873 	return result;
874 }
875 
876 /*
877  * Abort all outstanding requests.  This is only safe to use if the completion
878  * ring will never be walked again or the device has been reset, because it
879  * destroys the 1-1 mapping between context field passed to emulation and our
880  * request structure.
881  */
882 static void pvscsi_reset_all(struct pvscsi_adapter *adapter)
883 {
884 	unsigned i;
885 
886 	for (i = 0; i < adapter->req_depth; i++) {
887 		struct pvscsi_ctx *ctx = &adapter->cmd_map[i];
888 		struct scsi_cmnd *cmd = ctx->cmd;
889 		if (cmd) {
890 			scmd_printk(KERN_ERR, cmd,
891 				    "Forced reset on cmd %p\n", cmd);
892 			pvscsi_unmap_buffers(adapter, ctx);
893 			pvscsi_patch_sense(cmd);
894 			pvscsi_release_context(adapter, ctx);
895 			cmd->result = (DID_RESET << 16);
896 			cmd->scsi_done(cmd);
897 		}
898 	}
899 }
900 
901 static int pvscsi_host_reset(struct scsi_cmnd *cmd)
902 {
903 	struct Scsi_Host *host = cmd->device->host;
904 	struct pvscsi_adapter *adapter = shost_priv(host);
905 	unsigned long flags;
906 	bool use_msg;
907 
908 	scmd_printk(KERN_INFO, cmd, "SCSI Host reset\n");
909 
910 	spin_lock_irqsave(&adapter->hw_lock, flags);
911 
912 	use_msg = adapter->use_msg;
913 
914 	if (use_msg) {
915 		adapter->use_msg = false;
916 		spin_unlock_irqrestore(&adapter->hw_lock, flags);
917 
918 		/*
919 		 * Now that we know that the ISR won't add more work on the
920 		 * workqueue we can safely flush any outstanding work.
921 		 */
922 		flush_workqueue(adapter->workqueue);
923 		spin_lock_irqsave(&adapter->hw_lock, flags);
924 	}
925 
926 	/*
927 	 * We're going to tear down the entire ring structure and set it back
928 	 * up, so stalling new requests until all completions are flushed and
929 	 * the rings are back in place.
930 	 */
931 
932 	pvscsi_process_request_ring(adapter);
933 
934 	ll_adapter_reset(adapter);
935 
936 	/*
937 	 * Now process any completions.  Note we do this AFTER adapter reset,
938 	 * which is strange, but stops races where completions get posted
939 	 * between processing the ring and issuing the reset.  The backend will
940 	 * not touch the ring memory after reset, so the immediately pre-reset
941 	 * completion ring state is still valid.
942 	 */
943 	pvscsi_process_completion_ring(adapter);
944 
945 	pvscsi_reset_all(adapter);
946 	adapter->use_msg = use_msg;
947 	pvscsi_setup_all_rings(adapter);
948 	pvscsi_unmask_intr(adapter);
949 
950 	spin_unlock_irqrestore(&adapter->hw_lock, flags);
951 
952 	return SUCCESS;
953 }
954 
955 static int pvscsi_bus_reset(struct scsi_cmnd *cmd)
956 {
957 	struct Scsi_Host *host = cmd->device->host;
958 	struct pvscsi_adapter *adapter = shost_priv(host);
959 	unsigned long flags;
960 
961 	scmd_printk(KERN_INFO, cmd, "SCSI Bus reset\n");
962 
963 	/*
964 	 * We don't want to queue new requests for this bus after
965 	 * flushing all pending requests to emulation, since new
966 	 * requests could then sneak in during this bus reset phase,
967 	 * so take the lock now.
968 	 */
969 	spin_lock_irqsave(&adapter->hw_lock, flags);
970 
971 	pvscsi_process_request_ring(adapter);
972 	ll_bus_reset(adapter);
973 	pvscsi_process_completion_ring(adapter);
974 
975 	spin_unlock_irqrestore(&adapter->hw_lock, flags);
976 
977 	return SUCCESS;
978 }
979 
980 static int pvscsi_device_reset(struct scsi_cmnd *cmd)
981 {
982 	struct Scsi_Host *host = cmd->device->host;
983 	struct pvscsi_adapter *adapter = shost_priv(host);
984 	unsigned long flags;
985 
986 	scmd_printk(KERN_INFO, cmd, "SCSI device reset on scsi%u:%u\n",
987 		    host->host_no, cmd->device->id);
988 
989 	/*
990 	 * We don't want to queue new requests for this device after flushing
991 	 * all pending requests to emulation, since new requests could then
992 	 * sneak in during this device reset phase, so take the lock now.
993 	 */
994 	spin_lock_irqsave(&adapter->hw_lock, flags);
995 
996 	pvscsi_process_request_ring(adapter);
997 	ll_device_reset(adapter, cmd->device->id);
998 	pvscsi_process_completion_ring(adapter);
999 
1000 	spin_unlock_irqrestore(&adapter->hw_lock, flags);
1001 
1002 	return SUCCESS;
1003 }
1004 
1005 static struct scsi_host_template pvscsi_template;
1006 
1007 static const char *pvscsi_info(struct Scsi_Host *host)
1008 {
1009 	struct pvscsi_adapter *adapter = shost_priv(host);
1010 	static char buf[256];
1011 
1012 	sprintf(buf, "VMware PVSCSI storage adapter rev %d, req/cmp/msg rings: "
1013 		"%u/%u/%u pages, cmd_per_lun=%u", adapter->rev,
1014 		adapter->req_pages, adapter->cmp_pages, adapter->msg_pages,
1015 		pvscsi_template.cmd_per_lun);
1016 
1017 	return buf;
1018 }
1019 
1020 static struct scsi_host_template pvscsi_template = {
1021 	.module				= THIS_MODULE,
1022 	.name				= "VMware PVSCSI Host Adapter",
1023 	.proc_name			= "vmw_pvscsi",
1024 	.info				= pvscsi_info,
1025 	.queuecommand			= pvscsi_queue,
1026 	.this_id			= -1,
1027 	.sg_tablesize			= PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT,
1028 	.dma_boundary			= UINT_MAX,
1029 	.max_sectors			= 0xffff,
1030 	.change_queue_depth		= pvscsi_change_queue_depth,
1031 	.eh_abort_handler		= pvscsi_abort,
1032 	.eh_device_reset_handler	= pvscsi_device_reset,
1033 	.eh_bus_reset_handler		= pvscsi_bus_reset,
1034 	.eh_host_reset_handler		= pvscsi_host_reset,
1035 };
1036 
1037 static void pvscsi_process_msg(const struct pvscsi_adapter *adapter,
1038 			       const struct PVSCSIRingMsgDesc *e)
1039 {
1040 	struct PVSCSIRingsState *s = adapter->rings_state;
1041 	struct Scsi_Host *host = adapter->host;
1042 	struct scsi_device *sdev;
1043 
1044 	printk(KERN_INFO "vmw_pvscsi: msg type: 0x%x - MSG RING: %u/%u (%u) \n",
1045 	       e->type, s->msgProdIdx, s->msgConsIdx, s->msgNumEntriesLog2);
1046 
1047 	BUILD_BUG_ON(PVSCSI_MSG_LAST != 2);
1048 
1049 	if (e->type == PVSCSI_MSG_DEV_ADDED) {
1050 		struct PVSCSIMsgDescDevStatusChanged *desc;
1051 		desc = (struct PVSCSIMsgDescDevStatusChanged *)e;
1052 
1053 		printk(KERN_INFO
1054 		       "vmw_pvscsi: msg: device added at scsi%u:%u:%u\n",
1055 		       desc->bus, desc->target, desc->lun[1]);
1056 
1057 		if (!scsi_host_get(host))
1058 			return;
1059 
1060 		sdev = scsi_device_lookup(host, desc->bus, desc->target,
1061 					  desc->lun[1]);
1062 		if (sdev) {
1063 			printk(KERN_INFO "vmw_pvscsi: device already exists\n");
1064 			scsi_device_put(sdev);
1065 		} else
1066 			scsi_add_device(adapter->host, desc->bus,
1067 					desc->target, desc->lun[1]);
1068 
1069 		scsi_host_put(host);
1070 	} else if (e->type == PVSCSI_MSG_DEV_REMOVED) {
1071 		struct PVSCSIMsgDescDevStatusChanged *desc;
1072 		desc = (struct PVSCSIMsgDescDevStatusChanged *)e;
1073 
1074 		printk(KERN_INFO
1075 		       "vmw_pvscsi: msg: device removed at scsi%u:%u:%u\n",
1076 		       desc->bus, desc->target, desc->lun[1]);
1077 
1078 		if (!scsi_host_get(host))
1079 			return;
1080 
1081 		sdev = scsi_device_lookup(host, desc->bus, desc->target,
1082 					  desc->lun[1]);
1083 		if (sdev) {
1084 			scsi_remove_device(sdev);
1085 			scsi_device_put(sdev);
1086 		} else
1087 			printk(KERN_INFO
1088 			       "vmw_pvscsi: failed to lookup scsi%u:%u:%u\n",
1089 			       desc->bus, desc->target, desc->lun[1]);
1090 
1091 		scsi_host_put(host);
1092 	}
1093 }
1094 
1095 static int pvscsi_msg_pending(const struct pvscsi_adapter *adapter)
1096 {
1097 	struct PVSCSIRingsState *s = adapter->rings_state;
1098 
1099 	return s->msgProdIdx != s->msgConsIdx;
1100 }
1101 
1102 static void pvscsi_process_msg_ring(const struct pvscsi_adapter *adapter)
1103 {
1104 	struct PVSCSIRingsState *s = adapter->rings_state;
1105 	struct PVSCSIRingMsgDesc *ring = adapter->msg_ring;
1106 	u32 msg_entries = s->msgNumEntriesLog2;
1107 
1108 	while (pvscsi_msg_pending(adapter)) {
1109 		struct PVSCSIRingMsgDesc *e = ring + (s->msgConsIdx &
1110 						      MASK(msg_entries));
1111 
1112 		barrier();
1113 		pvscsi_process_msg(adapter, e);
1114 		barrier();
1115 		s->msgConsIdx++;
1116 	}
1117 }
1118 
1119 static void pvscsi_msg_workqueue_handler(struct work_struct *data)
1120 {
1121 	struct pvscsi_adapter *adapter;
1122 
1123 	adapter = container_of(data, struct pvscsi_adapter, work);
1124 
1125 	pvscsi_process_msg_ring(adapter);
1126 }
1127 
1128 static int pvscsi_setup_msg_workqueue(struct pvscsi_adapter *adapter)
1129 {
1130 	char name[32];
1131 
1132 	if (!pvscsi_use_msg)
1133 		return 0;
1134 
1135 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,
1136 			 PVSCSI_CMD_SETUP_MSG_RING);
1137 
1138 	if (pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS) == -1)
1139 		return 0;
1140 
1141 	snprintf(name, sizeof(name),
1142 		 "vmw_pvscsi_wq_%u", adapter->host->host_no);
1143 
1144 	adapter->workqueue = create_singlethread_workqueue(name);
1145 	if (!adapter->workqueue) {
1146 		printk(KERN_ERR "vmw_pvscsi: failed to create work queue\n");
1147 		return 0;
1148 	}
1149 	INIT_WORK(&adapter->work, pvscsi_msg_workqueue_handler);
1150 
1151 	return 1;
1152 }
1153 
1154 static bool pvscsi_setup_req_threshold(struct pvscsi_adapter *adapter,
1155 				      bool enable)
1156 {
1157 	u32 val;
1158 
1159 	if (!pvscsi_use_req_threshold)
1160 		return false;
1161 
1162 	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND,
1163 			 PVSCSI_CMD_SETUP_REQCALLTHRESHOLD);
1164 	val = pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS);
1165 	if (val == -1) {
1166 		printk(KERN_INFO "vmw_pvscsi: device does not support req_threshold\n");
1167 		return false;
1168 	} else {
1169 		struct PVSCSICmdDescSetupReqCall cmd_msg = { 0 };
1170 		cmd_msg.enable = enable;
1171 		printk(KERN_INFO
1172 		       "vmw_pvscsi: %sabling reqCallThreshold\n",
1173 			enable ? "en" : "dis");
1174 		pvscsi_write_cmd_desc(adapter,
1175 				      PVSCSI_CMD_SETUP_REQCALLTHRESHOLD,
1176 				      &cmd_msg, sizeof(cmd_msg));
1177 		return pvscsi_reg_read(adapter,
1178 				       PVSCSI_REG_OFFSET_COMMAND_STATUS) != 0;
1179 	}
1180 }
1181 
1182 static irqreturn_t pvscsi_isr(int irq, void *devp)
1183 {
1184 	struct pvscsi_adapter *adapter = devp;
1185 	unsigned long flags;
1186 
1187 	spin_lock_irqsave(&adapter->hw_lock, flags);
1188 	pvscsi_process_completion_ring(adapter);
1189 	if (adapter->use_msg && pvscsi_msg_pending(adapter))
1190 		queue_work(adapter->workqueue, &adapter->work);
1191 	spin_unlock_irqrestore(&adapter->hw_lock, flags);
1192 
1193 	return IRQ_HANDLED;
1194 }
1195 
1196 static irqreturn_t pvscsi_shared_isr(int irq, void *devp)
1197 {
1198 	struct pvscsi_adapter *adapter = devp;
1199 	u32 val = pvscsi_read_intr_status(adapter);
1200 
1201 	if (!(val & PVSCSI_INTR_ALL_SUPPORTED))
1202 		return IRQ_NONE;
1203 	pvscsi_write_intr_status(devp, val);
1204 	return pvscsi_isr(irq, devp);
1205 }
1206 
1207 static void pvscsi_free_sgls(const struct pvscsi_adapter *adapter)
1208 {
1209 	struct pvscsi_ctx *ctx = adapter->cmd_map;
1210 	unsigned i;
1211 
1212 	for (i = 0; i < adapter->req_depth; ++i, ++ctx)
1213 		free_pages((unsigned long)ctx->sgl, get_order(SGL_SIZE));
1214 }
1215 
1216 static void pvscsi_shutdown_intr(struct pvscsi_adapter *adapter)
1217 {
1218 	free_irq(pci_irq_vector(adapter->dev, 0), adapter);
1219 	pci_free_irq_vectors(adapter->dev);
1220 }
1221 
1222 static void pvscsi_release_resources(struct pvscsi_adapter *adapter)
1223 {
1224 	if (adapter->workqueue)
1225 		destroy_workqueue(adapter->workqueue);
1226 
1227 	if (adapter->mmioBase)
1228 		pci_iounmap(adapter->dev, adapter->mmioBase);
1229 
1230 	pci_release_regions(adapter->dev);
1231 
1232 	if (adapter->cmd_map) {
1233 		pvscsi_free_sgls(adapter);
1234 		kfree(adapter->cmd_map);
1235 	}
1236 
1237 	if (adapter->rings_state)
1238 		dma_free_coherent(&adapter->dev->dev, PAGE_SIZE,
1239 				    adapter->rings_state, adapter->ringStatePA);
1240 
1241 	if (adapter->req_ring)
1242 		dma_free_coherent(&adapter->dev->dev,
1243 				    adapter->req_pages * PAGE_SIZE,
1244 				    adapter->req_ring, adapter->reqRingPA);
1245 
1246 	if (adapter->cmp_ring)
1247 		dma_free_coherent(&adapter->dev->dev,
1248 				    adapter->cmp_pages * PAGE_SIZE,
1249 				    adapter->cmp_ring, adapter->cmpRingPA);
1250 
1251 	if (adapter->msg_ring)
1252 		dma_free_coherent(&adapter->dev->dev,
1253 				    adapter->msg_pages * PAGE_SIZE,
1254 				    adapter->msg_ring, adapter->msgRingPA);
1255 }
1256 
1257 /*
1258  * Allocate scatter gather lists.
1259  *
1260  * These are statically allocated.  Trying to be clever was not worth it.
1261  *
1262  * Dynamic allocation can fail, and we can't go deep into the memory
1263  * allocator, since we're a SCSI driver, and trying too hard to allocate
1264  * memory might generate disk I/O.  We also don't want to fail disk I/O
1265  * in that case because we can't get an allocation - the I/O could be
1266  * trying to swap out data to free memory.  Since that is pathological,
1267  * just use a statically allocated scatter list.
1268  *
1269  */
1270 static int pvscsi_allocate_sg(struct pvscsi_adapter *adapter)
1271 {
1272 	struct pvscsi_ctx *ctx;
1273 	int i;
1274 
1275 	ctx = adapter->cmd_map;
1276 	BUILD_BUG_ON(sizeof(struct pvscsi_sg_list) > SGL_SIZE);
1277 
1278 	for (i = 0; i < adapter->req_depth; ++i, ++ctx) {
1279 		ctx->sgl = (void *)__get_free_pages(GFP_KERNEL,
1280 						    get_order(SGL_SIZE));
1281 		ctx->sglPA = 0;
1282 		BUG_ON(!IS_ALIGNED(((unsigned long)ctx->sgl), PAGE_SIZE));
1283 		if (!ctx->sgl) {
1284 			for (; i >= 0; --i, --ctx) {
1285 				free_pages((unsigned long)ctx->sgl,
1286 					   get_order(SGL_SIZE));
1287 				ctx->sgl = NULL;
1288 			}
1289 			return -ENOMEM;
1290 		}
1291 	}
1292 
1293 	return 0;
1294 }
1295 
1296 /*
1297  * Query the device, fetch the config info and return the
1298  * maximum number of targets on the adapter. In case of
1299  * failure due to any reason return default i.e. 16.
1300  */
1301 static u32 pvscsi_get_max_targets(struct pvscsi_adapter *adapter)
1302 {
1303 	struct PVSCSICmdDescConfigCmd cmd;
1304 	struct PVSCSIConfigPageHeader *header;
1305 	struct device *dev;
1306 	dma_addr_t configPagePA;
1307 	void *config_page;
1308 	u32 numPhys = 16;
1309 
1310 	dev = pvscsi_dev(adapter);
1311 	config_page = dma_alloc_coherent(&adapter->dev->dev, PAGE_SIZE,
1312 			&configPagePA, GFP_KERNEL);
1313 	if (!config_page) {
1314 		dev_warn(dev, "vmw_pvscsi: failed to allocate memory for config page\n");
1315 		goto exit;
1316 	}
1317 	BUG_ON(configPagePA & ~PAGE_MASK);
1318 
1319 	/* Fetch config info from the device. */
1320 	cmd.configPageAddress = ((u64)PVSCSI_CONFIG_CONTROLLER_ADDRESS) << 32;
1321 	cmd.configPageNum = PVSCSI_CONFIG_PAGE_CONTROLLER;
1322 	cmd.cmpAddr = configPagePA;
1323 	cmd._pad = 0;
1324 
1325 	/*
1326 	 * Mark the completion page header with error values. If the device
1327 	 * completes the command successfully, it sets the status values to
1328 	 * indicate success.
1329 	 */
1330 	header = config_page;
1331 	memset(header, 0, sizeof *header);
1332 	header->hostStatus = BTSTAT_INVPARAM;
1333 	header->scsiStatus = SDSTAT_CHECK;
1334 
1335 	pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_CONFIG, &cmd, sizeof cmd);
1336 
1337 	if (header->hostStatus == BTSTAT_SUCCESS &&
1338 	    header->scsiStatus == SDSTAT_GOOD) {
1339 		struct PVSCSIConfigPageController *config;
1340 
1341 		config = config_page;
1342 		numPhys = config->numPhys;
1343 	} else
1344 		dev_warn(dev, "vmw_pvscsi: PVSCSI_CMD_CONFIG failed. hostStatus = 0x%x, scsiStatus = 0x%x\n",
1345 			 header->hostStatus, header->scsiStatus);
1346 	dma_free_coherent(&adapter->dev->dev, PAGE_SIZE, config_page,
1347 			  configPagePA);
1348 exit:
1349 	return numPhys;
1350 }
1351 
1352 static int pvscsi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1353 {
1354 	unsigned int irq_flag = PCI_IRQ_MSIX | PCI_IRQ_MSI | PCI_IRQ_LEGACY;
1355 	struct pvscsi_adapter *adapter;
1356 	struct pvscsi_adapter adapter_temp;
1357 	struct Scsi_Host *host = NULL;
1358 	unsigned int i;
1359 	int error;
1360 	u32 max_id;
1361 
1362 	error = -ENODEV;
1363 
1364 	if (pci_enable_device(pdev))
1365 		return error;
1366 
1367 	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
1368 		printk(KERN_INFO "vmw_pvscsi: using 64bit dma\n");
1369 	} else if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) {
1370 		printk(KERN_INFO "vmw_pvscsi: using 32bit dma\n");
1371 	} else {
1372 		printk(KERN_ERR "vmw_pvscsi: failed to set DMA mask\n");
1373 		goto out_disable_device;
1374 	}
1375 
1376 	/*
1377 	 * Let's use a temp pvscsi_adapter struct until we find the number of
1378 	 * targets on the adapter, after that we will switch to the real
1379 	 * allocated struct.
1380 	 */
1381 	adapter = &adapter_temp;
1382 	memset(adapter, 0, sizeof(*adapter));
1383 	adapter->dev  = pdev;
1384 	adapter->rev = pdev->revision;
1385 
1386 	if (pci_request_regions(pdev, "vmw_pvscsi")) {
1387 		printk(KERN_ERR "vmw_pvscsi: pci memory selection failed\n");
1388 		goto out_disable_device;
1389 	}
1390 
1391 	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1392 		if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE_IO))
1393 			continue;
1394 
1395 		if (pci_resource_len(pdev, i) < PVSCSI_MEM_SPACE_SIZE)
1396 			continue;
1397 
1398 		break;
1399 	}
1400 
1401 	if (i == DEVICE_COUNT_RESOURCE) {
1402 		printk(KERN_ERR
1403 		       "vmw_pvscsi: adapter has no suitable MMIO region\n");
1404 		goto out_release_resources_and_disable;
1405 	}
1406 
1407 	adapter->mmioBase = pci_iomap(pdev, i, PVSCSI_MEM_SPACE_SIZE);
1408 
1409 	if (!adapter->mmioBase) {
1410 		printk(KERN_ERR
1411 		       "vmw_pvscsi: can't iomap for BAR %d memsize %lu\n",
1412 		       i, PVSCSI_MEM_SPACE_SIZE);
1413 		goto out_release_resources_and_disable;
1414 	}
1415 
1416 	pci_set_master(pdev);
1417 
1418 	/*
1419 	 * Ask the device for max number of targets before deciding the
1420 	 * default pvscsi_ring_pages value.
1421 	 */
1422 	max_id = pvscsi_get_max_targets(adapter);
1423 	printk(KERN_INFO "vmw_pvscsi: max_id: %u\n", max_id);
1424 
1425 	if (pvscsi_ring_pages == 0)
1426 		/*
1427 		 * Set the right default value. Up to 16 it is 8, above it is
1428 		 * max.
1429 		 */
1430 		pvscsi_ring_pages = (max_id > 16) ?
1431 			PVSCSI_SETUP_RINGS_MAX_NUM_PAGES :
1432 			PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
1433 	printk(KERN_INFO
1434 	       "vmw_pvscsi: setting ring_pages to %d\n",
1435 	       pvscsi_ring_pages);
1436 
1437 	pvscsi_template.can_queue =
1438 		min(PVSCSI_MAX_NUM_PAGES_REQ_RING, pvscsi_ring_pages) *
1439 		PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
1440 	pvscsi_template.cmd_per_lun =
1441 		min(pvscsi_template.can_queue, pvscsi_cmd_per_lun);
1442 	host = scsi_host_alloc(&pvscsi_template, sizeof(struct pvscsi_adapter));
1443 	if (!host) {
1444 		printk(KERN_ERR "vmw_pvscsi: failed to allocate host\n");
1445 		goto out_release_resources_and_disable;
1446 	}
1447 
1448 	/*
1449 	 * Let's use the real pvscsi_adapter struct here onwards.
1450 	 */
1451 	adapter = shost_priv(host);
1452 	memset(adapter, 0, sizeof(*adapter));
1453 	adapter->dev  = pdev;
1454 	adapter->host = host;
1455 	/*
1456 	 * Copy back what we already have to the allocated adapter struct.
1457 	 */
1458 	adapter->rev = adapter_temp.rev;
1459 	adapter->mmioBase = adapter_temp.mmioBase;
1460 
1461 	spin_lock_init(&adapter->hw_lock);
1462 	host->max_channel = 0;
1463 	host->max_lun     = 1;
1464 	host->max_cmd_len = 16;
1465 	host->max_id      = max_id;
1466 
1467 	pci_set_drvdata(pdev, host);
1468 
1469 	ll_adapter_reset(adapter);
1470 
1471 	adapter->use_msg = pvscsi_setup_msg_workqueue(adapter);
1472 
1473 	error = pvscsi_allocate_rings(adapter);
1474 	if (error) {
1475 		printk(KERN_ERR "vmw_pvscsi: unable to allocate ring memory\n");
1476 		goto out_release_resources;
1477 	}
1478 
1479 	/*
1480 	 * From this point on we should reset the adapter if anything goes
1481 	 * wrong.
1482 	 */
1483 	pvscsi_setup_all_rings(adapter);
1484 
1485 	adapter->cmd_map = kcalloc(adapter->req_depth,
1486 				   sizeof(struct pvscsi_ctx), GFP_KERNEL);
1487 	if (!adapter->cmd_map) {
1488 		printk(KERN_ERR "vmw_pvscsi: failed to allocate memory.\n");
1489 		error = -ENOMEM;
1490 		goto out_reset_adapter;
1491 	}
1492 
1493 	INIT_LIST_HEAD(&adapter->cmd_pool);
1494 	for (i = 0; i < adapter->req_depth; i++) {
1495 		struct pvscsi_ctx *ctx = adapter->cmd_map + i;
1496 		list_add(&ctx->list, &adapter->cmd_pool);
1497 	}
1498 
1499 	error = pvscsi_allocate_sg(adapter);
1500 	if (error) {
1501 		printk(KERN_ERR "vmw_pvscsi: unable to allocate s/g table\n");
1502 		goto out_reset_adapter;
1503 	}
1504 
1505 	if (pvscsi_disable_msix)
1506 		irq_flag &= ~PCI_IRQ_MSIX;
1507 	if (pvscsi_disable_msi)
1508 		irq_flag &= ~PCI_IRQ_MSI;
1509 
1510 	error = pci_alloc_irq_vectors(adapter->dev, 1, 1, irq_flag);
1511 	if (error < 0)
1512 		goto out_reset_adapter;
1513 
1514 	adapter->use_req_threshold = pvscsi_setup_req_threshold(adapter, true);
1515 	printk(KERN_DEBUG "vmw_pvscsi: driver-based request coalescing %sabled\n",
1516 	       adapter->use_req_threshold ? "en" : "dis");
1517 
1518 	if (adapter->dev->msix_enabled || adapter->dev->msi_enabled) {
1519 		printk(KERN_INFO "vmw_pvscsi: using MSI%s\n",
1520 			adapter->dev->msix_enabled ? "-X" : "");
1521 		error = request_irq(pci_irq_vector(pdev, 0), pvscsi_isr,
1522 				0, "vmw_pvscsi", adapter);
1523 	} else {
1524 		printk(KERN_INFO "vmw_pvscsi: using INTx\n");
1525 		error = request_irq(pci_irq_vector(pdev, 0), pvscsi_shared_isr,
1526 				IRQF_SHARED, "vmw_pvscsi", adapter);
1527 	}
1528 
1529 	if (error) {
1530 		printk(KERN_ERR
1531 		       "vmw_pvscsi: unable to request IRQ: %d\n", error);
1532 		goto out_reset_adapter;
1533 	}
1534 
1535 	error = scsi_add_host(host, &pdev->dev);
1536 	if (error) {
1537 		printk(KERN_ERR
1538 		       "vmw_pvscsi: scsi_add_host failed: %d\n", error);
1539 		goto out_reset_adapter;
1540 	}
1541 
1542 	dev_info(&pdev->dev, "VMware PVSCSI rev %d host #%u\n",
1543 		 adapter->rev, host->host_no);
1544 
1545 	pvscsi_unmask_intr(adapter);
1546 
1547 	scsi_scan_host(host);
1548 
1549 	return 0;
1550 
1551 out_reset_adapter:
1552 	ll_adapter_reset(adapter);
1553 out_release_resources:
1554 	pvscsi_shutdown_intr(adapter);
1555 	pvscsi_release_resources(adapter);
1556 	scsi_host_put(host);
1557 out_disable_device:
1558 	pci_disable_device(pdev);
1559 
1560 	return error;
1561 
1562 out_release_resources_and_disable:
1563 	pvscsi_shutdown_intr(adapter);
1564 	pvscsi_release_resources(adapter);
1565 	goto out_disable_device;
1566 }
1567 
1568 static void __pvscsi_shutdown(struct pvscsi_adapter *adapter)
1569 {
1570 	pvscsi_mask_intr(adapter);
1571 
1572 	if (adapter->workqueue)
1573 		flush_workqueue(adapter->workqueue);
1574 
1575 	pvscsi_shutdown_intr(adapter);
1576 
1577 	pvscsi_process_request_ring(adapter);
1578 	pvscsi_process_completion_ring(adapter);
1579 	ll_adapter_reset(adapter);
1580 }
1581 
1582 static void pvscsi_shutdown(struct pci_dev *dev)
1583 {
1584 	struct Scsi_Host *host = pci_get_drvdata(dev);
1585 	struct pvscsi_adapter *adapter = shost_priv(host);
1586 
1587 	__pvscsi_shutdown(adapter);
1588 }
1589 
1590 static void pvscsi_remove(struct pci_dev *pdev)
1591 {
1592 	struct Scsi_Host *host = pci_get_drvdata(pdev);
1593 	struct pvscsi_adapter *adapter = shost_priv(host);
1594 
1595 	scsi_remove_host(host);
1596 
1597 	__pvscsi_shutdown(adapter);
1598 	pvscsi_release_resources(adapter);
1599 
1600 	scsi_host_put(host);
1601 
1602 	pci_disable_device(pdev);
1603 }
1604 
1605 static struct pci_driver pvscsi_pci_driver = {
1606 	.name		= "vmw_pvscsi",
1607 	.id_table	= pvscsi_pci_tbl,
1608 	.probe		= pvscsi_probe,
1609 	.remove		= pvscsi_remove,
1610 	.shutdown       = pvscsi_shutdown,
1611 };
1612 
1613 static int __init pvscsi_init(void)
1614 {
1615 	pr_info("%s - version %s\n",
1616 		PVSCSI_LINUX_DRIVER_DESC, PVSCSI_DRIVER_VERSION_STRING);
1617 	return pci_register_driver(&pvscsi_pci_driver);
1618 }
1619 
1620 static void __exit pvscsi_exit(void)
1621 {
1622 	pci_unregister_driver(&pvscsi_pci_driver);
1623 }
1624 
1625 module_init(pvscsi_init);
1626 module_exit(pvscsi_exit);
1627