1 /* 2 * Linux driver for VMware's para-virtualized SCSI HBA. 3 * 4 * Copyright (C) 2008-2014, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 */ 21 22 #include <linux/kernel.h> 23 #include <linux/module.h> 24 #include <linux/interrupt.h> 25 #include <linux/slab.h> 26 #include <linux/workqueue.h> 27 #include <linux/pci.h> 28 29 #include <scsi/scsi.h> 30 #include <scsi/scsi_host.h> 31 #include <scsi/scsi_cmnd.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_tcq.h> 34 35 #include "vmw_pvscsi.h" 36 37 #define PVSCSI_LINUX_DRIVER_DESC "VMware PVSCSI driver" 38 39 MODULE_DESCRIPTION(PVSCSI_LINUX_DRIVER_DESC); 40 MODULE_AUTHOR("VMware, Inc."); 41 MODULE_LICENSE("GPL"); 42 MODULE_VERSION(PVSCSI_DRIVER_VERSION_STRING); 43 44 #define PVSCSI_DEFAULT_NUM_PAGES_PER_RING 8 45 #define PVSCSI_DEFAULT_NUM_PAGES_MSG_RING 1 46 #define PVSCSI_DEFAULT_QUEUE_DEPTH 254 47 #define SGL_SIZE PAGE_SIZE 48 49 struct pvscsi_sg_list { 50 struct PVSCSISGElement sge[PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT]; 51 }; 52 53 struct pvscsi_ctx { 54 /* 55 * The index of the context in cmd_map serves as the context ID for a 56 * 1-to-1 mapping completions back to requests. 57 */ 58 struct scsi_cmnd *cmd; 59 struct pvscsi_sg_list *sgl; 60 struct list_head list; 61 dma_addr_t dataPA; 62 dma_addr_t sensePA; 63 dma_addr_t sglPA; 64 struct completion *abort_cmp; 65 }; 66 67 struct pvscsi_adapter { 68 char *mmioBase; 69 u8 rev; 70 bool use_msg; 71 bool use_req_threshold; 72 73 spinlock_t hw_lock; 74 75 struct workqueue_struct *workqueue; 76 struct work_struct work; 77 78 struct PVSCSIRingReqDesc *req_ring; 79 unsigned req_pages; 80 unsigned req_depth; 81 dma_addr_t reqRingPA; 82 83 struct PVSCSIRingCmpDesc *cmp_ring; 84 unsigned cmp_pages; 85 dma_addr_t cmpRingPA; 86 87 struct PVSCSIRingMsgDesc *msg_ring; 88 unsigned msg_pages; 89 dma_addr_t msgRingPA; 90 91 struct PVSCSIRingsState *rings_state; 92 dma_addr_t ringStatePA; 93 94 struct pci_dev *dev; 95 struct Scsi_Host *host; 96 97 struct list_head cmd_pool; 98 struct pvscsi_ctx *cmd_map; 99 }; 100 101 102 /* Command line parameters */ 103 static int pvscsi_ring_pages; 104 static int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING; 105 static int pvscsi_cmd_per_lun = PVSCSI_DEFAULT_QUEUE_DEPTH; 106 static bool pvscsi_disable_msi; 107 static bool pvscsi_disable_msix; 108 static bool pvscsi_use_msg = true; 109 static bool pvscsi_use_req_threshold = true; 110 111 #define PVSCSI_RW (S_IRUSR | S_IWUSR) 112 113 module_param_named(ring_pages, pvscsi_ring_pages, int, PVSCSI_RW); 114 MODULE_PARM_DESC(ring_pages, "Number of pages per req/cmp ring - (default=" 115 __stringify(PVSCSI_DEFAULT_NUM_PAGES_PER_RING) 116 "[up to 16 targets]," 117 __stringify(PVSCSI_SETUP_RINGS_MAX_NUM_PAGES) 118 "[for 16+ targets])"); 119 120 module_param_named(msg_ring_pages, pvscsi_msg_ring_pages, int, PVSCSI_RW); 121 MODULE_PARM_DESC(msg_ring_pages, "Number of pages for the msg ring - (default=" 122 __stringify(PVSCSI_DEFAULT_NUM_PAGES_MSG_RING) ")"); 123 124 module_param_named(cmd_per_lun, pvscsi_cmd_per_lun, int, PVSCSI_RW); 125 MODULE_PARM_DESC(cmd_per_lun, "Maximum commands per lun - (default=" 126 __stringify(PVSCSI_DEFAULT_QUEUE_DEPTH) ")"); 127 128 module_param_named(disable_msi, pvscsi_disable_msi, bool, PVSCSI_RW); 129 MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)"); 130 131 module_param_named(disable_msix, pvscsi_disable_msix, bool, PVSCSI_RW); 132 MODULE_PARM_DESC(disable_msix, "Disable MSI-X use in driver - (default=0)"); 133 134 module_param_named(use_msg, pvscsi_use_msg, bool, PVSCSI_RW); 135 MODULE_PARM_DESC(use_msg, "Use msg ring when available - (default=1)"); 136 137 module_param_named(use_req_threshold, pvscsi_use_req_threshold, 138 bool, PVSCSI_RW); 139 MODULE_PARM_DESC(use_req_threshold, "Use driver-based request coalescing if configured - (default=1)"); 140 141 static const struct pci_device_id pvscsi_pci_tbl[] = { 142 { PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_PVSCSI) }, 143 { 0 } 144 }; 145 146 MODULE_DEVICE_TABLE(pci, pvscsi_pci_tbl); 147 148 static struct device * 149 pvscsi_dev(const struct pvscsi_adapter *adapter) 150 { 151 return &(adapter->dev->dev); 152 } 153 154 static struct pvscsi_ctx * 155 pvscsi_find_context(const struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd) 156 { 157 struct pvscsi_ctx *ctx, *end; 158 159 end = &adapter->cmd_map[adapter->req_depth]; 160 for (ctx = adapter->cmd_map; ctx < end; ctx++) 161 if (ctx->cmd == cmd) 162 return ctx; 163 164 return NULL; 165 } 166 167 static struct pvscsi_ctx * 168 pvscsi_acquire_context(struct pvscsi_adapter *adapter, struct scsi_cmnd *cmd) 169 { 170 struct pvscsi_ctx *ctx; 171 172 if (list_empty(&adapter->cmd_pool)) 173 return NULL; 174 175 ctx = list_first_entry(&adapter->cmd_pool, struct pvscsi_ctx, list); 176 ctx->cmd = cmd; 177 list_del(&ctx->list); 178 179 return ctx; 180 } 181 182 static void pvscsi_release_context(struct pvscsi_adapter *adapter, 183 struct pvscsi_ctx *ctx) 184 { 185 ctx->cmd = NULL; 186 ctx->abort_cmp = NULL; 187 list_add(&ctx->list, &adapter->cmd_pool); 188 } 189 190 /* 191 * Map a pvscsi_ctx struct to a context ID field value; we map to a simple 192 * non-zero integer. ctx always points to an entry in cmd_map array, hence 193 * the return value is always >=1. 194 */ 195 static u64 pvscsi_map_context(const struct pvscsi_adapter *adapter, 196 const struct pvscsi_ctx *ctx) 197 { 198 return ctx - adapter->cmd_map + 1; 199 } 200 201 static struct pvscsi_ctx * 202 pvscsi_get_context(const struct pvscsi_adapter *adapter, u64 context) 203 { 204 return &adapter->cmd_map[context - 1]; 205 } 206 207 static void pvscsi_reg_write(const struct pvscsi_adapter *adapter, 208 u32 offset, u32 val) 209 { 210 writel(val, adapter->mmioBase + offset); 211 } 212 213 static u32 pvscsi_reg_read(const struct pvscsi_adapter *adapter, u32 offset) 214 { 215 return readl(adapter->mmioBase + offset); 216 } 217 218 static u32 pvscsi_read_intr_status(const struct pvscsi_adapter *adapter) 219 { 220 return pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_INTR_STATUS); 221 } 222 223 static void pvscsi_write_intr_status(const struct pvscsi_adapter *adapter, 224 u32 val) 225 { 226 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_STATUS, val); 227 } 228 229 static void pvscsi_unmask_intr(const struct pvscsi_adapter *adapter) 230 { 231 u32 intr_bits; 232 233 intr_bits = PVSCSI_INTR_CMPL_MASK; 234 if (adapter->use_msg) 235 intr_bits |= PVSCSI_INTR_MSG_MASK; 236 237 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, intr_bits); 238 } 239 240 static void pvscsi_mask_intr(const struct pvscsi_adapter *adapter) 241 { 242 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_INTR_MASK, 0); 243 } 244 245 static void pvscsi_write_cmd_desc(const struct pvscsi_adapter *adapter, 246 u32 cmd, const void *desc, size_t len) 247 { 248 const u32 *ptr = desc; 249 size_t i; 250 251 len /= sizeof(*ptr); 252 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, cmd); 253 for (i = 0; i < len; i++) 254 pvscsi_reg_write(adapter, 255 PVSCSI_REG_OFFSET_COMMAND_DATA, ptr[i]); 256 } 257 258 static void pvscsi_abort_cmd(const struct pvscsi_adapter *adapter, 259 const struct pvscsi_ctx *ctx) 260 { 261 struct PVSCSICmdDescAbortCmd cmd = { 0 }; 262 263 cmd.target = ctx->cmd->device->id; 264 cmd.context = pvscsi_map_context(adapter, ctx); 265 266 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ABORT_CMD, &cmd, sizeof(cmd)); 267 } 268 269 static void pvscsi_kick_rw_io(const struct pvscsi_adapter *adapter) 270 { 271 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_RW_IO, 0); 272 } 273 274 static void pvscsi_process_request_ring(const struct pvscsi_adapter *adapter) 275 { 276 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0); 277 } 278 279 static int scsi_is_rw(unsigned char op) 280 { 281 return op == READ_6 || op == WRITE_6 || 282 op == READ_10 || op == WRITE_10 || 283 op == READ_12 || op == WRITE_12 || 284 op == READ_16 || op == WRITE_16; 285 } 286 287 static void pvscsi_kick_io(const struct pvscsi_adapter *adapter, 288 unsigned char op) 289 { 290 if (scsi_is_rw(op)) { 291 struct PVSCSIRingsState *s = adapter->rings_state; 292 293 if (!adapter->use_req_threshold || 294 s->reqProdIdx - s->reqConsIdx >= s->reqCallThreshold) 295 pvscsi_kick_rw_io(adapter); 296 } else { 297 pvscsi_process_request_ring(adapter); 298 } 299 } 300 301 static void ll_adapter_reset(const struct pvscsi_adapter *adapter) 302 { 303 dev_dbg(pvscsi_dev(adapter), "Adapter Reset on %p\n", adapter); 304 305 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_ADAPTER_RESET, NULL, 0); 306 } 307 308 static void ll_bus_reset(const struct pvscsi_adapter *adapter) 309 { 310 dev_dbg(pvscsi_dev(adapter), "Resetting bus on %p\n", adapter); 311 312 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_BUS, NULL, 0); 313 } 314 315 static void ll_device_reset(const struct pvscsi_adapter *adapter, u32 target) 316 { 317 struct PVSCSICmdDescResetDevice cmd = { 0 }; 318 319 dev_dbg(pvscsi_dev(adapter), "Resetting device: target=%u\n", target); 320 321 cmd.target = target; 322 323 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_RESET_DEVICE, 324 &cmd, sizeof(cmd)); 325 } 326 327 static void pvscsi_create_sg(struct pvscsi_ctx *ctx, 328 struct scatterlist *sg, unsigned count) 329 { 330 unsigned i; 331 struct PVSCSISGElement *sge; 332 333 BUG_ON(count > PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT); 334 335 sge = &ctx->sgl->sge[0]; 336 for (i = 0; i < count; i++, sg = sg_next(sg)) { 337 sge[i].addr = sg_dma_address(sg); 338 sge[i].length = sg_dma_len(sg); 339 sge[i].flags = 0; 340 } 341 } 342 343 /* 344 * Map all data buffers for a command into PCI space and 345 * setup the scatter/gather list if needed. 346 */ 347 static int pvscsi_map_buffers(struct pvscsi_adapter *adapter, 348 struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd, 349 struct PVSCSIRingReqDesc *e) 350 { 351 unsigned count; 352 unsigned bufflen = scsi_bufflen(cmd); 353 struct scatterlist *sg; 354 355 e->dataLen = bufflen; 356 e->dataAddr = 0; 357 if (bufflen == 0) 358 return 0; 359 360 sg = scsi_sglist(cmd); 361 count = scsi_sg_count(cmd); 362 if (count != 0) { 363 int segs = scsi_dma_map(cmd); 364 365 if (segs == -ENOMEM) { 366 scmd_printk(KERN_DEBUG, cmd, 367 "vmw_pvscsi: Failed to map cmd sglist for DMA.\n"); 368 return -ENOMEM; 369 } else if (segs > 1) { 370 pvscsi_create_sg(ctx, sg, segs); 371 372 e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST; 373 ctx->sglPA = dma_map_single(&adapter->dev->dev, 374 ctx->sgl, SGL_SIZE, DMA_TO_DEVICE); 375 if (dma_mapping_error(&adapter->dev->dev, ctx->sglPA)) { 376 scmd_printk(KERN_ERR, cmd, 377 "vmw_pvscsi: Failed to map ctx sglist for DMA.\n"); 378 scsi_dma_unmap(cmd); 379 ctx->sglPA = 0; 380 return -ENOMEM; 381 } 382 e->dataAddr = ctx->sglPA; 383 } else 384 e->dataAddr = sg_dma_address(sg); 385 } else { 386 /* 387 * In case there is no S/G list, scsi_sglist points 388 * directly to the buffer. 389 */ 390 ctx->dataPA = dma_map_single(&adapter->dev->dev, sg, bufflen, 391 cmd->sc_data_direction); 392 if (dma_mapping_error(&adapter->dev->dev, ctx->dataPA)) { 393 scmd_printk(KERN_DEBUG, cmd, 394 "vmw_pvscsi: Failed to map direct data buffer for DMA.\n"); 395 return -ENOMEM; 396 } 397 e->dataAddr = ctx->dataPA; 398 } 399 400 return 0; 401 } 402 403 /* 404 * The device incorrectly doesn't clear the first byte of the sense 405 * buffer in some cases. We have to do it ourselves. 406 * Otherwise we run into trouble when SWIOTLB is forced. 407 */ 408 static void pvscsi_patch_sense(struct scsi_cmnd *cmd) 409 { 410 if (cmd->sense_buffer) 411 cmd->sense_buffer[0] = 0; 412 } 413 414 static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter, 415 struct pvscsi_ctx *ctx) 416 { 417 struct scsi_cmnd *cmd; 418 unsigned bufflen; 419 420 cmd = ctx->cmd; 421 bufflen = scsi_bufflen(cmd); 422 423 if (bufflen != 0) { 424 unsigned count = scsi_sg_count(cmd); 425 426 if (count != 0) { 427 scsi_dma_unmap(cmd); 428 if (ctx->sglPA) { 429 dma_unmap_single(&adapter->dev->dev, ctx->sglPA, 430 SGL_SIZE, DMA_TO_DEVICE); 431 ctx->sglPA = 0; 432 } 433 } else 434 dma_unmap_single(&adapter->dev->dev, ctx->dataPA, 435 bufflen, cmd->sc_data_direction); 436 } 437 if (cmd->sense_buffer) 438 dma_unmap_single(&adapter->dev->dev, ctx->sensePA, 439 SCSI_SENSE_BUFFERSIZE, DMA_FROM_DEVICE); 440 } 441 442 static int pvscsi_allocate_rings(struct pvscsi_adapter *adapter) 443 { 444 adapter->rings_state = dma_alloc_coherent(&adapter->dev->dev, PAGE_SIZE, 445 &adapter->ringStatePA, GFP_KERNEL); 446 if (!adapter->rings_state) 447 return -ENOMEM; 448 449 adapter->req_pages = min(PVSCSI_MAX_NUM_PAGES_REQ_RING, 450 pvscsi_ring_pages); 451 adapter->req_depth = adapter->req_pages 452 * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE; 453 adapter->req_ring = dma_alloc_coherent(&adapter->dev->dev, 454 adapter->req_pages * PAGE_SIZE, &adapter->reqRingPA, 455 GFP_KERNEL); 456 if (!adapter->req_ring) 457 return -ENOMEM; 458 459 adapter->cmp_pages = min(PVSCSI_MAX_NUM_PAGES_CMP_RING, 460 pvscsi_ring_pages); 461 adapter->cmp_ring = dma_alloc_coherent(&adapter->dev->dev, 462 adapter->cmp_pages * PAGE_SIZE, &adapter->cmpRingPA, 463 GFP_KERNEL); 464 if (!adapter->cmp_ring) 465 return -ENOMEM; 466 467 BUG_ON(!IS_ALIGNED(adapter->ringStatePA, PAGE_SIZE)); 468 BUG_ON(!IS_ALIGNED(adapter->reqRingPA, PAGE_SIZE)); 469 BUG_ON(!IS_ALIGNED(adapter->cmpRingPA, PAGE_SIZE)); 470 471 if (!adapter->use_msg) 472 return 0; 473 474 adapter->msg_pages = min(PVSCSI_MAX_NUM_PAGES_MSG_RING, 475 pvscsi_msg_ring_pages); 476 adapter->msg_ring = dma_alloc_coherent(&adapter->dev->dev, 477 adapter->msg_pages * PAGE_SIZE, &adapter->msgRingPA, 478 GFP_KERNEL); 479 if (!adapter->msg_ring) 480 return -ENOMEM; 481 BUG_ON(!IS_ALIGNED(adapter->msgRingPA, PAGE_SIZE)); 482 483 return 0; 484 } 485 486 static void pvscsi_setup_all_rings(const struct pvscsi_adapter *adapter) 487 { 488 struct PVSCSICmdDescSetupRings cmd = { 0 }; 489 dma_addr_t base; 490 unsigned i; 491 492 cmd.ringsStatePPN = adapter->ringStatePA >> PAGE_SHIFT; 493 cmd.reqRingNumPages = adapter->req_pages; 494 cmd.cmpRingNumPages = adapter->cmp_pages; 495 496 base = adapter->reqRingPA; 497 for (i = 0; i < adapter->req_pages; i++) { 498 cmd.reqRingPPNs[i] = base >> PAGE_SHIFT; 499 base += PAGE_SIZE; 500 } 501 502 base = adapter->cmpRingPA; 503 for (i = 0; i < adapter->cmp_pages; i++) { 504 cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT; 505 base += PAGE_SIZE; 506 } 507 508 memset(adapter->rings_state, 0, PAGE_SIZE); 509 memset(adapter->req_ring, 0, adapter->req_pages * PAGE_SIZE); 510 memset(adapter->cmp_ring, 0, adapter->cmp_pages * PAGE_SIZE); 511 512 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_RINGS, 513 &cmd, sizeof(cmd)); 514 515 if (adapter->use_msg) { 516 struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 }; 517 518 cmd_msg.numPages = adapter->msg_pages; 519 520 base = adapter->msgRingPA; 521 for (i = 0; i < adapter->msg_pages; i++) { 522 cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT; 523 base += PAGE_SIZE; 524 } 525 memset(adapter->msg_ring, 0, adapter->msg_pages * PAGE_SIZE); 526 527 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_SETUP_MSG_RING, 528 &cmd_msg, sizeof(cmd_msg)); 529 } 530 } 531 532 static int pvscsi_change_queue_depth(struct scsi_device *sdev, int qdepth) 533 { 534 if (!sdev->tagged_supported) 535 qdepth = 1; 536 return scsi_change_queue_depth(sdev, qdepth); 537 } 538 539 /* 540 * Pull a completion descriptor off and pass the completion back 541 * to the SCSI mid layer. 542 */ 543 static void pvscsi_complete_request(struct pvscsi_adapter *adapter, 544 const struct PVSCSIRingCmpDesc *e) 545 { 546 struct pvscsi_ctx *ctx; 547 struct scsi_cmnd *cmd; 548 struct completion *abort_cmp; 549 u32 btstat = e->hostStatus; 550 u32 sdstat = e->scsiStatus; 551 552 ctx = pvscsi_get_context(adapter, e->context); 553 cmd = ctx->cmd; 554 abort_cmp = ctx->abort_cmp; 555 pvscsi_unmap_buffers(adapter, ctx); 556 if (sdstat != SAM_STAT_CHECK_CONDITION) 557 pvscsi_patch_sense(cmd); 558 pvscsi_release_context(adapter, ctx); 559 if (abort_cmp) { 560 /* 561 * The command was requested to be aborted. Just signal that 562 * the request completed and swallow the actual cmd completion 563 * here. The abort handler will post a completion for this 564 * command indicating that it got successfully aborted. 565 */ 566 complete(abort_cmp); 567 return; 568 } 569 570 cmd->result = 0; 571 if (sdstat != SAM_STAT_GOOD && 572 (btstat == BTSTAT_SUCCESS || 573 btstat == BTSTAT_LINKED_COMMAND_COMPLETED || 574 btstat == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG)) { 575 if (sdstat == SAM_STAT_COMMAND_TERMINATED) { 576 cmd->result = (DID_RESET << 16); 577 } else { 578 cmd->result = (DID_OK << 16) | sdstat; 579 if (sdstat == SAM_STAT_CHECK_CONDITION && 580 cmd->sense_buffer) 581 cmd->result |= (DRIVER_SENSE << 24); 582 } 583 } else 584 switch (btstat) { 585 case BTSTAT_SUCCESS: 586 case BTSTAT_LINKED_COMMAND_COMPLETED: 587 case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG: 588 /* 589 * Commands like INQUIRY may transfer less data than 590 * requested by the initiator via bufflen. Set residual 591 * count to make upper layer aware of the actual amount 592 * of data returned. 593 */ 594 scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen); 595 cmd->result = (DID_OK << 16); 596 break; 597 598 case BTSTAT_DATARUN: 599 case BTSTAT_DATA_UNDERRUN: 600 /* Report residual data in underruns */ 601 scsi_set_resid(cmd, scsi_bufflen(cmd) - e->dataLen); 602 cmd->result = (DID_ERROR << 16); 603 break; 604 605 case BTSTAT_SELTIMEO: 606 /* Our emulation returns this for non-connected devs */ 607 cmd->result = (DID_BAD_TARGET << 16); 608 break; 609 610 case BTSTAT_LUNMISMATCH: 611 case BTSTAT_TAGREJECT: 612 case BTSTAT_BADMSG: 613 cmd->result = (DRIVER_INVALID << 24); 614 fallthrough; 615 616 case BTSTAT_HAHARDWARE: 617 case BTSTAT_INVPHASE: 618 case BTSTAT_HATIMEOUT: 619 case BTSTAT_NORESPONSE: 620 case BTSTAT_DISCONNECT: 621 case BTSTAT_HASOFTWARE: 622 case BTSTAT_BUSFREE: 623 case BTSTAT_SENSFAILED: 624 cmd->result |= (DID_ERROR << 16); 625 break; 626 627 case BTSTAT_SENTRST: 628 case BTSTAT_RECVRST: 629 case BTSTAT_BUSRESET: 630 cmd->result = (DID_RESET << 16); 631 break; 632 633 case BTSTAT_ABORTQUEUE: 634 cmd->result = (DID_BUS_BUSY << 16); 635 break; 636 637 case BTSTAT_SCSIPARITY: 638 cmd->result = (DID_PARITY << 16); 639 break; 640 641 default: 642 cmd->result = (DID_ERROR << 16); 643 scmd_printk(KERN_DEBUG, cmd, 644 "Unknown completion status: 0x%x\n", 645 btstat); 646 } 647 648 dev_dbg(&cmd->device->sdev_gendev, 649 "cmd=%p %x ctx=%p result=0x%x status=0x%x,%x\n", 650 cmd, cmd->cmnd[0], ctx, cmd->result, btstat, sdstat); 651 652 cmd->scsi_done(cmd); 653 } 654 655 /* 656 * barrier usage : Since the PVSCSI device is emulated, there could be cases 657 * where we may want to serialize some accesses between the driver and the 658 * emulation layer. We use compiler barriers instead of the more expensive 659 * memory barriers because PVSCSI is only supported on X86 which has strong 660 * memory access ordering. 661 */ 662 static void pvscsi_process_completion_ring(struct pvscsi_adapter *adapter) 663 { 664 struct PVSCSIRingsState *s = adapter->rings_state; 665 struct PVSCSIRingCmpDesc *ring = adapter->cmp_ring; 666 u32 cmp_entries = s->cmpNumEntriesLog2; 667 668 while (s->cmpConsIdx != s->cmpProdIdx) { 669 struct PVSCSIRingCmpDesc *e = ring + (s->cmpConsIdx & 670 MASK(cmp_entries)); 671 /* 672 * This barrier() ensures that *e is not dereferenced while 673 * the device emulation still writes data into the slot. 674 * Since the device emulation advances s->cmpProdIdx only after 675 * updating the slot we want to check it first. 676 */ 677 barrier(); 678 pvscsi_complete_request(adapter, e); 679 /* 680 * This barrier() ensures that compiler doesn't reorder write 681 * to s->cmpConsIdx before the read of (*e) inside 682 * pvscsi_complete_request. Otherwise, device emulation may 683 * overwrite *e before we had a chance to read it. 684 */ 685 barrier(); 686 s->cmpConsIdx++; 687 } 688 } 689 690 /* 691 * Translate a Linux SCSI request into a request ring entry. 692 */ 693 static int pvscsi_queue_ring(struct pvscsi_adapter *adapter, 694 struct pvscsi_ctx *ctx, struct scsi_cmnd *cmd) 695 { 696 struct PVSCSIRingsState *s; 697 struct PVSCSIRingReqDesc *e; 698 struct scsi_device *sdev; 699 u32 req_entries; 700 701 s = adapter->rings_state; 702 sdev = cmd->device; 703 req_entries = s->reqNumEntriesLog2; 704 705 /* 706 * If this condition holds, we might have room on the request ring, but 707 * we might not have room on the completion ring for the response. 708 * However, we have already ruled out this possibility - we would not 709 * have successfully allocated a context if it were true, since we only 710 * have one context per request entry. Check for it anyway, since it 711 * would be a serious bug. 712 */ 713 if (s->reqProdIdx - s->cmpConsIdx >= 1 << req_entries) { 714 scmd_printk(KERN_ERR, cmd, "vmw_pvscsi: " 715 "ring full: reqProdIdx=%d cmpConsIdx=%d\n", 716 s->reqProdIdx, s->cmpConsIdx); 717 return -1; 718 } 719 720 e = adapter->req_ring + (s->reqProdIdx & MASK(req_entries)); 721 722 e->bus = sdev->channel; 723 e->target = sdev->id; 724 memset(e->lun, 0, sizeof(e->lun)); 725 e->lun[1] = sdev->lun; 726 727 if (cmd->sense_buffer) { 728 ctx->sensePA = dma_map_single(&adapter->dev->dev, 729 cmd->sense_buffer, SCSI_SENSE_BUFFERSIZE, 730 DMA_FROM_DEVICE); 731 if (dma_mapping_error(&adapter->dev->dev, ctx->sensePA)) { 732 scmd_printk(KERN_DEBUG, cmd, 733 "vmw_pvscsi: Failed to map sense buffer for DMA.\n"); 734 ctx->sensePA = 0; 735 return -ENOMEM; 736 } 737 e->senseAddr = ctx->sensePA; 738 e->senseLen = SCSI_SENSE_BUFFERSIZE; 739 } else { 740 e->senseLen = 0; 741 e->senseAddr = 0; 742 } 743 e->cdbLen = cmd->cmd_len; 744 e->vcpuHint = smp_processor_id(); 745 memcpy(e->cdb, cmd->cmnd, e->cdbLen); 746 747 e->tag = SIMPLE_QUEUE_TAG; 748 749 if (cmd->sc_data_direction == DMA_FROM_DEVICE) 750 e->flags = PVSCSI_FLAG_CMD_DIR_TOHOST; 751 else if (cmd->sc_data_direction == DMA_TO_DEVICE) 752 e->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE; 753 else if (cmd->sc_data_direction == DMA_NONE) 754 e->flags = PVSCSI_FLAG_CMD_DIR_NONE; 755 else 756 e->flags = 0; 757 758 if (pvscsi_map_buffers(adapter, ctx, cmd, e) != 0) { 759 if (cmd->sense_buffer) { 760 dma_unmap_single(&adapter->dev->dev, ctx->sensePA, 761 SCSI_SENSE_BUFFERSIZE, 762 DMA_FROM_DEVICE); 763 ctx->sensePA = 0; 764 } 765 return -ENOMEM; 766 } 767 768 e->context = pvscsi_map_context(adapter, ctx); 769 770 barrier(); 771 772 s->reqProdIdx++; 773 774 return 0; 775 } 776 777 static int pvscsi_queue_lck(struct scsi_cmnd *cmd, void (*done)(struct scsi_cmnd *)) 778 { 779 struct Scsi_Host *host = cmd->device->host; 780 struct pvscsi_adapter *adapter = shost_priv(host); 781 struct pvscsi_ctx *ctx; 782 unsigned long flags; 783 unsigned char op; 784 785 spin_lock_irqsave(&adapter->hw_lock, flags); 786 787 ctx = pvscsi_acquire_context(adapter, cmd); 788 if (!ctx || pvscsi_queue_ring(adapter, ctx, cmd) != 0) { 789 if (ctx) 790 pvscsi_release_context(adapter, ctx); 791 spin_unlock_irqrestore(&adapter->hw_lock, flags); 792 return SCSI_MLQUEUE_HOST_BUSY; 793 } 794 795 cmd->scsi_done = done; 796 op = cmd->cmnd[0]; 797 798 dev_dbg(&cmd->device->sdev_gendev, 799 "queued cmd %p, ctx %p, op=%x\n", cmd, ctx, op); 800 801 spin_unlock_irqrestore(&adapter->hw_lock, flags); 802 803 pvscsi_kick_io(adapter, op); 804 805 return 0; 806 } 807 808 static DEF_SCSI_QCMD(pvscsi_queue) 809 810 static int pvscsi_abort(struct scsi_cmnd *cmd) 811 { 812 struct pvscsi_adapter *adapter = shost_priv(cmd->device->host); 813 struct pvscsi_ctx *ctx; 814 unsigned long flags; 815 int result = SUCCESS; 816 DECLARE_COMPLETION_ONSTACK(abort_cmp); 817 int done; 818 819 scmd_printk(KERN_DEBUG, cmd, "task abort on host %u, %p\n", 820 adapter->host->host_no, cmd); 821 822 spin_lock_irqsave(&adapter->hw_lock, flags); 823 824 /* 825 * Poll the completion ring first - we might be trying to abort 826 * a command that is waiting to be dispatched in the completion ring. 827 */ 828 pvscsi_process_completion_ring(adapter); 829 830 /* 831 * If there is no context for the command, it either already succeeded 832 * or else was never properly issued. Not our problem. 833 */ 834 ctx = pvscsi_find_context(adapter, cmd); 835 if (!ctx) { 836 scmd_printk(KERN_DEBUG, cmd, "Failed to abort cmd %p\n", cmd); 837 goto out; 838 } 839 840 /* 841 * Mark that the command has been requested to be aborted and issue 842 * the abort. 843 */ 844 ctx->abort_cmp = &abort_cmp; 845 846 pvscsi_abort_cmd(adapter, ctx); 847 spin_unlock_irqrestore(&adapter->hw_lock, flags); 848 /* Wait for 2 secs for the completion. */ 849 done = wait_for_completion_timeout(&abort_cmp, msecs_to_jiffies(2000)); 850 spin_lock_irqsave(&adapter->hw_lock, flags); 851 852 if (!done) { 853 /* 854 * Failed to abort the command, unmark the fact that it 855 * was requested to be aborted. 856 */ 857 ctx->abort_cmp = NULL; 858 result = FAILED; 859 scmd_printk(KERN_DEBUG, cmd, 860 "Failed to get completion for aborted cmd %p\n", 861 cmd); 862 goto out; 863 } 864 865 /* 866 * Successfully aborted the command. 867 */ 868 cmd->result = (DID_ABORT << 16); 869 cmd->scsi_done(cmd); 870 871 out: 872 spin_unlock_irqrestore(&adapter->hw_lock, flags); 873 return result; 874 } 875 876 /* 877 * Abort all outstanding requests. This is only safe to use if the completion 878 * ring will never be walked again or the device has been reset, because it 879 * destroys the 1-1 mapping between context field passed to emulation and our 880 * request structure. 881 */ 882 static void pvscsi_reset_all(struct pvscsi_adapter *adapter) 883 { 884 unsigned i; 885 886 for (i = 0; i < adapter->req_depth; i++) { 887 struct pvscsi_ctx *ctx = &adapter->cmd_map[i]; 888 struct scsi_cmnd *cmd = ctx->cmd; 889 if (cmd) { 890 scmd_printk(KERN_ERR, cmd, 891 "Forced reset on cmd %p\n", cmd); 892 pvscsi_unmap_buffers(adapter, ctx); 893 pvscsi_patch_sense(cmd); 894 pvscsi_release_context(adapter, ctx); 895 cmd->result = (DID_RESET << 16); 896 cmd->scsi_done(cmd); 897 } 898 } 899 } 900 901 static int pvscsi_host_reset(struct scsi_cmnd *cmd) 902 { 903 struct Scsi_Host *host = cmd->device->host; 904 struct pvscsi_adapter *adapter = shost_priv(host); 905 unsigned long flags; 906 bool use_msg; 907 908 scmd_printk(KERN_INFO, cmd, "SCSI Host reset\n"); 909 910 spin_lock_irqsave(&adapter->hw_lock, flags); 911 912 use_msg = adapter->use_msg; 913 914 if (use_msg) { 915 adapter->use_msg = false; 916 spin_unlock_irqrestore(&adapter->hw_lock, flags); 917 918 /* 919 * Now that we know that the ISR won't add more work on the 920 * workqueue we can safely flush any outstanding work. 921 */ 922 flush_workqueue(adapter->workqueue); 923 spin_lock_irqsave(&adapter->hw_lock, flags); 924 } 925 926 /* 927 * We're going to tear down the entire ring structure and set it back 928 * up, so stalling new requests until all completions are flushed and 929 * the rings are back in place. 930 */ 931 932 pvscsi_process_request_ring(adapter); 933 934 ll_adapter_reset(adapter); 935 936 /* 937 * Now process any completions. Note we do this AFTER adapter reset, 938 * which is strange, but stops races where completions get posted 939 * between processing the ring and issuing the reset. The backend will 940 * not touch the ring memory after reset, so the immediately pre-reset 941 * completion ring state is still valid. 942 */ 943 pvscsi_process_completion_ring(adapter); 944 945 pvscsi_reset_all(adapter); 946 adapter->use_msg = use_msg; 947 pvscsi_setup_all_rings(adapter); 948 pvscsi_unmask_intr(adapter); 949 950 spin_unlock_irqrestore(&adapter->hw_lock, flags); 951 952 return SUCCESS; 953 } 954 955 static int pvscsi_bus_reset(struct scsi_cmnd *cmd) 956 { 957 struct Scsi_Host *host = cmd->device->host; 958 struct pvscsi_adapter *adapter = shost_priv(host); 959 unsigned long flags; 960 961 scmd_printk(KERN_INFO, cmd, "SCSI Bus reset\n"); 962 963 /* 964 * We don't want to queue new requests for this bus after 965 * flushing all pending requests to emulation, since new 966 * requests could then sneak in during this bus reset phase, 967 * so take the lock now. 968 */ 969 spin_lock_irqsave(&adapter->hw_lock, flags); 970 971 pvscsi_process_request_ring(adapter); 972 ll_bus_reset(adapter); 973 pvscsi_process_completion_ring(adapter); 974 975 spin_unlock_irqrestore(&adapter->hw_lock, flags); 976 977 return SUCCESS; 978 } 979 980 static int pvscsi_device_reset(struct scsi_cmnd *cmd) 981 { 982 struct Scsi_Host *host = cmd->device->host; 983 struct pvscsi_adapter *adapter = shost_priv(host); 984 unsigned long flags; 985 986 scmd_printk(KERN_INFO, cmd, "SCSI device reset on scsi%u:%u\n", 987 host->host_no, cmd->device->id); 988 989 /* 990 * We don't want to queue new requests for this device after flushing 991 * all pending requests to emulation, since new requests could then 992 * sneak in during this device reset phase, so take the lock now. 993 */ 994 spin_lock_irqsave(&adapter->hw_lock, flags); 995 996 pvscsi_process_request_ring(adapter); 997 ll_device_reset(adapter, cmd->device->id); 998 pvscsi_process_completion_ring(adapter); 999 1000 spin_unlock_irqrestore(&adapter->hw_lock, flags); 1001 1002 return SUCCESS; 1003 } 1004 1005 static struct scsi_host_template pvscsi_template; 1006 1007 static const char *pvscsi_info(struct Scsi_Host *host) 1008 { 1009 struct pvscsi_adapter *adapter = shost_priv(host); 1010 static char buf[256]; 1011 1012 sprintf(buf, "VMware PVSCSI storage adapter rev %d, req/cmp/msg rings: " 1013 "%u/%u/%u pages, cmd_per_lun=%u", adapter->rev, 1014 adapter->req_pages, adapter->cmp_pages, adapter->msg_pages, 1015 pvscsi_template.cmd_per_lun); 1016 1017 return buf; 1018 } 1019 1020 static struct scsi_host_template pvscsi_template = { 1021 .module = THIS_MODULE, 1022 .name = "VMware PVSCSI Host Adapter", 1023 .proc_name = "vmw_pvscsi", 1024 .info = pvscsi_info, 1025 .queuecommand = pvscsi_queue, 1026 .this_id = -1, 1027 .sg_tablesize = PVSCSI_MAX_NUM_SG_ENTRIES_PER_SEGMENT, 1028 .dma_boundary = UINT_MAX, 1029 .max_sectors = 0xffff, 1030 .change_queue_depth = pvscsi_change_queue_depth, 1031 .eh_abort_handler = pvscsi_abort, 1032 .eh_device_reset_handler = pvscsi_device_reset, 1033 .eh_bus_reset_handler = pvscsi_bus_reset, 1034 .eh_host_reset_handler = pvscsi_host_reset, 1035 }; 1036 1037 static void pvscsi_process_msg(const struct pvscsi_adapter *adapter, 1038 const struct PVSCSIRingMsgDesc *e) 1039 { 1040 struct PVSCSIRingsState *s = adapter->rings_state; 1041 struct Scsi_Host *host = adapter->host; 1042 struct scsi_device *sdev; 1043 1044 printk(KERN_INFO "vmw_pvscsi: msg type: 0x%x - MSG RING: %u/%u (%u) \n", 1045 e->type, s->msgProdIdx, s->msgConsIdx, s->msgNumEntriesLog2); 1046 1047 BUILD_BUG_ON(PVSCSI_MSG_LAST != 2); 1048 1049 if (e->type == PVSCSI_MSG_DEV_ADDED) { 1050 struct PVSCSIMsgDescDevStatusChanged *desc; 1051 desc = (struct PVSCSIMsgDescDevStatusChanged *)e; 1052 1053 printk(KERN_INFO 1054 "vmw_pvscsi: msg: device added at scsi%u:%u:%u\n", 1055 desc->bus, desc->target, desc->lun[1]); 1056 1057 if (!scsi_host_get(host)) 1058 return; 1059 1060 sdev = scsi_device_lookup(host, desc->bus, desc->target, 1061 desc->lun[1]); 1062 if (sdev) { 1063 printk(KERN_INFO "vmw_pvscsi: device already exists\n"); 1064 scsi_device_put(sdev); 1065 } else 1066 scsi_add_device(adapter->host, desc->bus, 1067 desc->target, desc->lun[1]); 1068 1069 scsi_host_put(host); 1070 } else if (e->type == PVSCSI_MSG_DEV_REMOVED) { 1071 struct PVSCSIMsgDescDevStatusChanged *desc; 1072 desc = (struct PVSCSIMsgDescDevStatusChanged *)e; 1073 1074 printk(KERN_INFO 1075 "vmw_pvscsi: msg: device removed at scsi%u:%u:%u\n", 1076 desc->bus, desc->target, desc->lun[1]); 1077 1078 if (!scsi_host_get(host)) 1079 return; 1080 1081 sdev = scsi_device_lookup(host, desc->bus, desc->target, 1082 desc->lun[1]); 1083 if (sdev) { 1084 scsi_remove_device(sdev); 1085 scsi_device_put(sdev); 1086 } else 1087 printk(KERN_INFO 1088 "vmw_pvscsi: failed to lookup scsi%u:%u:%u\n", 1089 desc->bus, desc->target, desc->lun[1]); 1090 1091 scsi_host_put(host); 1092 } 1093 } 1094 1095 static int pvscsi_msg_pending(const struct pvscsi_adapter *adapter) 1096 { 1097 struct PVSCSIRingsState *s = adapter->rings_state; 1098 1099 return s->msgProdIdx != s->msgConsIdx; 1100 } 1101 1102 static void pvscsi_process_msg_ring(const struct pvscsi_adapter *adapter) 1103 { 1104 struct PVSCSIRingsState *s = adapter->rings_state; 1105 struct PVSCSIRingMsgDesc *ring = adapter->msg_ring; 1106 u32 msg_entries = s->msgNumEntriesLog2; 1107 1108 while (pvscsi_msg_pending(adapter)) { 1109 struct PVSCSIRingMsgDesc *e = ring + (s->msgConsIdx & 1110 MASK(msg_entries)); 1111 1112 barrier(); 1113 pvscsi_process_msg(adapter, e); 1114 barrier(); 1115 s->msgConsIdx++; 1116 } 1117 } 1118 1119 static void pvscsi_msg_workqueue_handler(struct work_struct *data) 1120 { 1121 struct pvscsi_adapter *adapter; 1122 1123 adapter = container_of(data, struct pvscsi_adapter, work); 1124 1125 pvscsi_process_msg_ring(adapter); 1126 } 1127 1128 static int pvscsi_setup_msg_workqueue(struct pvscsi_adapter *adapter) 1129 { 1130 char name[32]; 1131 1132 if (!pvscsi_use_msg) 1133 return 0; 1134 1135 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, 1136 PVSCSI_CMD_SETUP_MSG_RING); 1137 1138 if (pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS) == -1) 1139 return 0; 1140 1141 snprintf(name, sizeof(name), 1142 "vmw_pvscsi_wq_%u", adapter->host->host_no); 1143 1144 adapter->workqueue = create_singlethread_workqueue(name); 1145 if (!adapter->workqueue) { 1146 printk(KERN_ERR "vmw_pvscsi: failed to create work queue\n"); 1147 return 0; 1148 } 1149 INIT_WORK(&adapter->work, pvscsi_msg_workqueue_handler); 1150 1151 return 1; 1152 } 1153 1154 static bool pvscsi_setup_req_threshold(struct pvscsi_adapter *adapter, 1155 bool enable) 1156 { 1157 u32 val; 1158 1159 if (!pvscsi_use_req_threshold) 1160 return false; 1161 1162 pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, 1163 PVSCSI_CMD_SETUP_REQCALLTHRESHOLD); 1164 val = pvscsi_reg_read(adapter, PVSCSI_REG_OFFSET_COMMAND_STATUS); 1165 if (val == -1) { 1166 printk(KERN_INFO "vmw_pvscsi: device does not support req_threshold\n"); 1167 return false; 1168 } else { 1169 struct PVSCSICmdDescSetupReqCall cmd_msg = { 0 }; 1170 cmd_msg.enable = enable; 1171 printk(KERN_INFO 1172 "vmw_pvscsi: %sabling reqCallThreshold\n", 1173 enable ? "en" : "dis"); 1174 pvscsi_write_cmd_desc(adapter, 1175 PVSCSI_CMD_SETUP_REQCALLTHRESHOLD, 1176 &cmd_msg, sizeof(cmd_msg)); 1177 return pvscsi_reg_read(adapter, 1178 PVSCSI_REG_OFFSET_COMMAND_STATUS) != 0; 1179 } 1180 } 1181 1182 static irqreturn_t pvscsi_isr(int irq, void *devp) 1183 { 1184 struct pvscsi_adapter *adapter = devp; 1185 unsigned long flags; 1186 1187 spin_lock_irqsave(&adapter->hw_lock, flags); 1188 pvscsi_process_completion_ring(adapter); 1189 if (adapter->use_msg && pvscsi_msg_pending(adapter)) 1190 queue_work(adapter->workqueue, &adapter->work); 1191 spin_unlock_irqrestore(&adapter->hw_lock, flags); 1192 1193 return IRQ_HANDLED; 1194 } 1195 1196 static irqreturn_t pvscsi_shared_isr(int irq, void *devp) 1197 { 1198 struct pvscsi_adapter *adapter = devp; 1199 u32 val = pvscsi_read_intr_status(adapter); 1200 1201 if (!(val & PVSCSI_INTR_ALL_SUPPORTED)) 1202 return IRQ_NONE; 1203 pvscsi_write_intr_status(devp, val); 1204 return pvscsi_isr(irq, devp); 1205 } 1206 1207 static void pvscsi_free_sgls(const struct pvscsi_adapter *adapter) 1208 { 1209 struct pvscsi_ctx *ctx = adapter->cmd_map; 1210 unsigned i; 1211 1212 for (i = 0; i < adapter->req_depth; ++i, ++ctx) 1213 free_pages((unsigned long)ctx->sgl, get_order(SGL_SIZE)); 1214 } 1215 1216 static void pvscsi_shutdown_intr(struct pvscsi_adapter *adapter) 1217 { 1218 free_irq(pci_irq_vector(adapter->dev, 0), adapter); 1219 pci_free_irq_vectors(adapter->dev); 1220 } 1221 1222 static void pvscsi_release_resources(struct pvscsi_adapter *adapter) 1223 { 1224 if (adapter->workqueue) 1225 destroy_workqueue(adapter->workqueue); 1226 1227 if (adapter->mmioBase) 1228 pci_iounmap(adapter->dev, adapter->mmioBase); 1229 1230 pci_release_regions(adapter->dev); 1231 1232 if (adapter->cmd_map) { 1233 pvscsi_free_sgls(adapter); 1234 kfree(adapter->cmd_map); 1235 } 1236 1237 if (adapter->rings_state) 1238 dma_free_coherent(&adapter->dev->dev, PAGE_SIZE, 1239 adapter->rings_state, adapter->ringStatePA); 1240 1241 if (adapter->req_ring) 1242 dma_free_coherent(&adapter->dev->dev, 1243 adapter->req_pages * PAGE_SIZE, 1244 adapter->req_ring, adapter->reqRingPA); 1245 1246 if (adapter->cmp_ring) 1247 dma_free_coherent(&adapter->dev->dev, 1248 adapter->cmp_pages * PAGE_SIZE, 1249 adapter->cmp_ring, adapter->cmpRingPA); 1250 1251 if (adapter->msg_ring) 1252 dma_free_coherent(&adapter->dev->dev, 1253 adapter->msg_pages * PAGE_SIZE, 1254 adapter->msg_ring, adapter->msgRingPA); 1255 } 1256 1257 /* 1258 * Allocate scatter gather lists. 1259 * 1260 * These are statically allocated. Trying to be clever was not worth it. 1261 * 1262 * Dynamic allocation can fail, and we can't go deep into the memory 1263 * allocator, since we're a SCSI driver, and trying too hard to allocate 1264 * memory might generate disk I/O. We also don't want to fail disk I/O 1265 * in that case because we can't get an allocation - the I/O could be 1266 * trying to swap out data to free memory. Since that is pathological, 1267 * just use a statically allocated scatter list. 1268 * 1269 */ 1270 static int pvscsi_allocate_sg(struct pvscsi_adapter *adapter) 1271 { 1272 struct pvscsi_ctx *ctx; 1273 int i; 1274 1275 ctx = adapter->cmd_map; 1276 BUILD_BUG_ON(sizeof(struct pvscsi_sg_list) > SGL_SIZE); 1277 1278 for (i = 0; i < adapter->req_depth; ++i, ++ctx) { 1279 ctx->sgl = (void *)__get_free_pages(GFP_KERNEL, 1280 get_order(SGL_SIZE)); 1281 ctx->sglPA = 0; 1282 BUG_ON(!IS_ALIGNED(((unsigned long)ctx->sgl), PAGE_SIZE)); 1283 if (!ctx->sgl) { 1284 for (; i >= 0; --i, --ctx) { 1285 free_pages((unsigned long)ctx->sgl, 1286 get_order(SGL_SIZE)); 1287 ctx->sgl = NULL; 1288 } 1289 return -ENOMEM; 1290 } 1291 } 1292 1293 return 0; 1294 } 1295 1296 /* 1297 * Query the device, fetch the config info and return the 1298 * maximum number of targets on the adapter. In case of 1299 * failure due to any reason return default i.e. 16. 1300 */ 1301 static u32 pvscsi_get_max_targets(struct pvscsi_adapter *adapter) 1302 { 1303 struct PVSCSICmdDescConfigCmd cmd; 1304 struct PVSCSIConfigPageHeader *header; 1305 struct device *dev; 1306 dma_addr_t configPagePA; 1307 void *config_page; 1308 u32 numPhys = 16; 1309 1310 dev = pvscsi_dev(adapter); 1311 config_page = dma_alloc_coherent(&adapter->dev->dev, PAGE_SIZE, 1312 &configPagePA, GFP_KERNEL); 1313 if (!config_page) { 1314 dev_warn(dev, "vmw_pvscsi: failed to allocate memory for config page\n"); 1315 goto exit; 1316 } 1317 BUG_ON(configPagePA & ~PAGE_MASK); 1318 1319 /* Fetch config info from the device. */ 1320 cmd.configPageAddress = ((u64)PVSCSI_CONFIG_CONTROLLER_ADDRESS) << 32; 1321 cmd.configPageNum = PVSCSI_CONFIG_PAGE_CONTROLLER; 1322 cmd.cmpAddr = configPagePA; 1323 cmd._pad = 0; 1324 1325 /* 1326 * Mark the completion page header with error values. If the device 1327 * completes the command successfully, it sets the status values to 1328 * indicate success. 1329 */ 1330 header = config_page; 1331 memset(header, 0, sizeof *header); 1332 header->hostStatus = BTSTAT_INVPARAM; 1333 header->scsiStatus = SDSTAT_CHECK; 1334 1335 pvscsi_write_cmd_desc(adapter, PVSCSI_CMD_CONFIG, &cmd, sizeof cmd); 1336 1337 if (header->hostStatus == BTSTAT_SUCCESS && 1338 header->scsiStatus == SDSTAT_GOOD) { 1339 struct PVSCSIConfigPageController *config; 1340 1341 config = config_page; 1342 numPhys = config->numPhys; 1343 } else 1344 dev_warn(dev, "vmw_pvscsi: PVSCSI_CMD_CONFIG failed. hostStatus = 0x%x, scsiStatus = 0x%x\n", 1345 header->hostStatus, header->scsiStatus); 1346 dma_free_coherent(&adapter->dev->dev, PAGE_SIZE, config_page, 1347 configPagePA); 1348 exit: 1349 return numPhys; 1350 } 1351 1352 static int pvscsi_probe(struct pci_dev *pdev, const struct pci_device_id *id) 1353 { 1354 unsigned int irq_flag = PCI_IRQ_MSIX | PCI_IRQ_MSI | PCI_IRQ_LEGACY; 1355 struct pvscsi_adapter *adapter; 1356 struct pvscsi_adapter adapter_temp; 1357 struct Scsi_Host *host = NULL; 1358 unsigned int i; 1359 int error; 1360 u32 max_id; 1361 1362 error = -ENODEV; 1363 1364 if (pci_enable_device(pdev)) 1365 return error; 1366 1367 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 1368 printk(KERN_INFO "vmw_pvscsi: using 64bit dma\n"); 1369 } else if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32))) { 1370 printk(KERN_INFO "vmw_pvscsi: using 32bit dma\n"); 1371 } else { 1372 printk(KERN_ERR "vmw_pvscsi: failed to set DMA mask\n"); 1373 goto out_disable_device; 1374 } 1375 1376 /* 1377 * Let's use a temp pvscsi_adapter struct until we find the number of 1378 * targets on the adapter, after that we will switch to the real 1379 * allocated struct. 1380 */ 1381 adapter = &adapter_temp; 1382 memset(adapter, 0, sizeof(*adapter)); 1383 adapter->dev = pdev; 1384 adapter->rev = pdev->revision; 1385 1386 if (pci_request_regions(pdev, "vmw_pvscsi")) { 1387 printk(KERN_ERR "vmw_pvscsi: pci memory selection failed\n"); 1388 goto out_disable_device; 1389 } 1390 1391 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { 1392 if ((pci_resource_flags(pdev, i) & PCI_BASE_ADDRESS_SPACE_IO)) 1393 continue; 1394 1395 if (pci_resource_len(pdev, i) < PVSCSI_MEM_SPACE_SIZE) 1396 continue; 1397 1398 break; 1399 } 1400 1401 if (i == DEVICE_COUNT_RESOURCE) { 1402 printk(KERN_ERR 1403 "vmw_pvscsi: adapter has no suitable MMIO region\n"); 1404 goto out_release_resources_and_disable; 1405 } 1406 1407 adapter->mmioBase = pci_iomap(pdev, i, PVSCSI_MEM_SPACE_SIZE); 1408 1409 if (!adapter->mmioBase) { 1410 printk(KERN_ERR 1411 "vmw_pvscsi: can't iomap for BAR %d memsize %lu\n", 1412 i, PVSCSI_MEM_SPACE_SIZE); 1413 goto out_release_resources_and_disable; 1414 } 1415 1416 pci_set_master(pdev); 1417 1418 /* 1419 * Ask the device for max number of targets before deciding the 1420 * default pvscsi_ring_pages value. 1421 */ 1422 max_id = pvscsi_get_max_targets(adapter); 1423 printk(KERN_INFO "vmw_pvscsi: max_id: %u\n", max_id); 1424 1425 if (pvscsi_ring_pages == 0) 1426 /* 1427 * Set the right default value. Up to 16 it is 8, above it is 1428 * max. 1429 */ 1430 pvscsi_ring_pages = (max_id > 16) ? 1431 PVSCSI_SETUP_RINGS_MAX_NUM_PAGES : 1432 PVSCSI_DEFAULT_NUM_PAGES_PER_RING; 1433 printk(KERN_INFO 1434 "vmw_pvscsi: setting ring_pages to %d\n", 1435 pvscsi_ring_pages); 1436 1437 pvscsi_template.can_queue = 1438 min(PVSCSI_MAX_NUM_PAGES_REQ_RING, pvscsi_ring_pages) * 1439 PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE; 1440 pvscsi_template.cmd_per_lun = 1441 min(pvscsi_template.can_queue, pvscsi_cmd_per_lun); 1442 host = scsi_host_alloc(&pvscsi_template, sizeof(struct pvscsi_adapter)); 1443 if (!host) { 1444 printk(KERN_ERR "vmw_pvscsi: failed to allocate host\n"); 1445 goto out_release_resources_and_disable; 1446 } 1447 1448 /* 1449 * Let's use the real pvscsi_adapter struct here onwards. 1450 */ 1451 adapter = shost_priv(host); 1452 memset(adapter, 0, sizeof(*adapter)); 1453 adapter->dev = pdev; 1454 adapter->host = host; 1455 /* 1456 * Copy back what we already have to the allocated adapter struct. 1457 */ 1458 adapter->rev = adapter_temp.rev; 1459 adapter->mmioBase = adapter_temp.mmioBase; 1460 1461 spin_lock_init(&adapter->hw_lock); 1462 host->max_channel = 0; 1463 host->max_lun = 1; 1464 host->max_cmd_len = 16; 1465 host->max_id = max_id; 1466 1467 pci_set_drvdata(pdev, host); 1468 1469 ll_adapter_reset(adapter); 1470 1471 adapter->use_msg = pvscsi_setup_msg_workqueue(adapter); 1472 1473 error = pvscsi_allocate_rings(adapter); 1474 if (error) { 1475 printk(KERN_ERR "vmw_pvscsi: unable to allocate ring memory\n"); 1476 goto out_release_resources; 1477 } 1478 1479 /* 1480 * From this point on we should reset the adapter if anything goes 1481 * wrong. 1482 */ 1483 pvscsi_setup_all_rings(adapter); 1484 1485 adapter->cmd_map = kcalloc(adapter->req_depth, 1486 sizeof(struct pvscsi_ctx), GFP_KERNEL); 1487 if (!adapter->cmd_map) { 1488 printk(KERN_ERR "vmw_pvscsi: failed to allocate memory.\n"); 1489 error = -ENOMEM; 1490 goto out_reset_adapter; 1491 } 1492 1493 INIT_LIST_HEAD(&adapter->cmd_pool); 1494 for (i = 0; i < adapter->req_depth; i++) { 1495 struct pvscsi_ctx *ctx = adapter->cmd_map + i; 1496 list_add(&ctx->list, &adapter->cmd_pool); 1497 } 1498 1499 error = pvscsi_allocate_sg(adapter); 1500 if (error) { 1501 printk(KERN_ERR "vmw_pvscsi: unable to allocate s/g table\n"); 1502 goto out_reset_adapter; 1503 } 1504 1505 if (pvscsi_disable_msix) 1506 irq_flag &= ~PCI_IRQ_MSIX; 1507 if (pvscsi_disable_msi) 1508 irq_flag &= ~PCI_IRQ_MSI; 1509 1510 error = pci_alloc_irq_vectors(adapter->dev, 1, 1, irq_flag); 1511 if (error < 0) 1512 goto out_reset_adapter; 1513 1514 adapter->use_req_threshold = pvscsi_setup_req_threshold(adapter, true); 1515 printk(KERN_DEBUG "vmw_pvscsi: driver-based request coalescing %sabled\n", 1516 adapter->use_req_threshold ? "en" : "dis"); 1517 1518 if (adapter->dev->msix_enabled || adapter->dev->msi_enabled) { 1519 printk(KERN_INFO "vmw_pvscsi: using MSI%s\n", 1520 adapter->dev->msix_enabled ? "-X" : ""); 1521 error = request_irq(pci_irq_vector(pdev, 0), pvscsi_isr, 1522 0, "vmw_pvscsi", adapter); 1523 } else { 1524 printk(KERN_INFO "vmw_pvscsi: using INTx\n"); 1525 error = request_irq(pci_irq_vector(pdev, 0), pvscsi_shared_isr, 1526 IRQF_SHARED, "vmw_pvscsi", adapter); 1527 } 1528 1529 if (error) { 1530 printk(KERN_ERR 1531 "vmw_pvscsi: unable to request IRQ: %d\n", error); 1532 goto out_reset_adapter; 1533 } 1534 1535 error = scsi_add_host(host, &pdev->dev); 1536 if (error) { 1537 printk(KERN_ERR 1538 "vmw_pvscsi: scsi_add_host failed: %d\n", error); 1539 goto out_reset_adapter; 1540 } 1541 1542 dev_info(&pdev->dev, "VMware PVSCSI rev %d host #%u\n", 1543 adapter->rev, host->host_no); 1544 1545 pvscsi_unmask_intr(adapter); 1546 1547 scsi_scan_host(host); 1548 1549 return 0; 1550 1551 out_reset_adapter: 1552 ll_adapter_reset(adapter); 1553 out_release_resources: 1554 pvscsi_shutdown_intr(adapter); 1555 pvscsi_release_resources(adapter); 1556 scsi_host_put(host); 1557 out_disable_device: 1558 pci_disable_device(pdev); 1559 1560 return error; 1561 1562 out_release_resources_and_disable: 1563 pvscsi_shutdown_intr(adapter); 1564 pvscsi_release_resources(adapter); 1565 goto out_disable_device; 1566 } 1567 1568 static void __pvscsi_shutdown(struct pvscsi_adapter *adapter) 1569 { 1570 pvscsi_mask_intr(adapter); 1571 1572 if (adapter->workqueue) 1573 flush_workqueue(adapter->workqueue); 1574 1575 pvscsi_shutdown_intr(adapter); 1576 1577 pvscsi_process_request_ring(adapter); 1578 pvscsi_process_completion_ring(adapter); 1579 ll_adapter_reset(adapter); 1580 } 1581 1582 static void pvscsi_shutdown(struct pci_dev *dev) 1583 { 1584 struct Scsi_Host *host = pci_get_drvdata(dev); 1585 struct pvscsi_adapter *adapter = shost_priv(host); 1586 1587 __pvscsi_shutdown(adapter); 1588 } 1589 1590 static void pvscsi_remove(struct pci_dev *pdev) 1591 { 1592 struct Scsi_Host *host = pci_get_drvdata(pdev); 1593 struct pvscsi_adapter *adapter = shost_priv(host); 1594 1595 scsi_remove_host(host); 1596 1597 __pvscsi_shutdown(adapter); 1598 pvscsi_release_resources(adapter); 1599 1600 scsi_host_put(host); 1601 1602 pci_disable_device(pdev); 1603 } 1604 1605 static struct pci_driver pvscsi_pci_driver = { 1606 .name = "vmw_pvscsi", 1607 .id_table = pvscsi_pci_tbl, 1608 .probe = pvscsi_probe, 1609 .remove = pvscsi_remove, 1610 .shutdown = pvscsi_shutdown, 1611 }; 1612 1613 static int __init pvscsi_init(void) 1614 { 1615 pr_info("%s - version %s\n", 1616 PVSCSI_LINUX_DRIVER_DESC, PVSCSI_DRIVER_VERSION_STRING); 1617 return pci_register_driver(&pvscsi_pci_driver); 1618 } 1619 1620 static void __exit pvscsi_exit(void) 1621 { 1622 pci_unregister_driver(&pvscsi_pci_driver); 1623 } 1624 1625 module_init(pvscsi_init); 1626 module_exit(pvscsi_exit); 1627