1 /******************************************************************************* 2 * Vhost kernel TCM fabric driver for virtio SCSI initiators 3 * 4 * (C) Copyright 2010-2013 Datera, Inc. 5 * (C) Copyright 2010-2012 IBM Corp. 6 * 7 * Licensed to the Linux Foundation under the General Public License (GPL) version 2. 8 * 9 * Authors: Nicholas A. Bellinger <nab@daterainc.com> 10 * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com> 11 * 12 * This program is free software; you can redistribute it and/or modify 13 * it under the terms of the GNU General Public License as published by 14 * the Free Software Foundation; either version 2 of the License, or 15 * (at your option) any later version. 16 * 17 * This program is distributed in the hope that it will be useful, 18 * but WITHOUT ANY WARRANTY; without even the implied warranty of 19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 * GNU General Public License for more details. 21 * 22 ****************************************************************************/ 23 24 #include <linux/module.h> 25 #include <linux/moduleparam.h> 26 #include <generated/utsrelease.h> 27 #include <linux/utsname.h> 28 #include <linux/init.h> 29 #include <linux/slab.h> 30 #include <linux/kthread.h> 31 #include <linux/types.h> 32 #include <linux/string.h> 33 #include <linux/configfs.h> 34 #include <linux/ctype.h> 35 #include <linux/compat.h> 36 #include <linux/eventfd.h> 37 #include <linux/fs.h> 38 #include <linux/vmalloc.h> 39 #include <linux/miscdevice.h> 40 #include <asm/unaligned.h> 41 #include <scsi/scsi_common.h> 42 #include <scsi/scsi_proto.h> 43 #include <target/target_core_base.h> 44 #include <target/target_core_fabric.h> 45 #include <linux/vhost.h> 46 #include <linux/virtio_scsi.h> 47 #include <linux/llist.h> 48 #include <linux/bitmap.h> 49 50 #include "vhost.h" 51 52 #define VHOST_SCSI_VERSION "v0.1" 53 #define VHOST_SCSI_NAMELEN 256 54 #define VHOST_SCSI_MAX_CDB_SIZE 32 55 #define VHOST_SCSI_DEFAULT_TAGS 256 56 #define VHOST_SCSI_PREALLOC_SGLS 2048 57 #define VHOST_SCSI_PREALLOC_UPAGES 2048 58 #define VHOST_SCSI_PREALLOC_PROT_SGLS 2048 59 60 struct vhost_scsi_inflight { 61 /* Wait for the flush operation to finish */ 62 struct completion comp; 63 /* Refcount for the inflight reqs */ 64 struct kref kref; 65 }; 66 67 struct vhost_scsi_cmd { 68 /* Descriptor from vhost_get_vq_desc() for virt_queue segment */ 69 int tvc_vq_desc; 70 /* virtio-scsi initiator task attribute */ 71 int tvc_task_attr; 72 /* virtio-scsi response incoming iovecs */ 73 int tvc_in_iovs; 74 /* virtio-scsi initiator data direction */ 75 enum dma_data_direction tvc_data_direction; 76 /* Expected data transfer length from virtio-scsi header */ 77 u32 tvc_exp_data_len; 78 /* The Tag from include/linux/virtio_scsi.h:struct virtio_scsi_cmd_req */ 79 u64 tvc_tag; 80 /* The number of scatterlists associated with this cmd */ 81 u32 tvc_sgl_count; 82 u32 tvc_prot_sgl_count; 83 /* Saved unpacked SCSI LUN for vhost_scsi_submission_work() */ 84 u32 tvc_lun; 85 /* Pointer to the SGL formatted memory from virtio-scsi */ 86 struct scatterlist *tvc_sgl; 87 struct scatterlist *tvc_prot_sgl; 88 struct page **tvc_upages; 89 /* Pointer to response header iovec */ 90 struct iovec tvc_resp_iov; 91 /* Pointer to vhost_scsi for our device */ 92 struct vhost_scsi *tvc_vhost; 93 /* Pointer to vhost_virtqueue for the cmd */ 94 struct vhost_virtqueue *tvc_vq; 95 /* Pointer to vhost nexus memory */ 96 struct vhost_scsi_nexus *tvc_nexus; 97 /* The TCM I/O descriptor that is accessed via container_of() */ 98 struct se_cmd tvc_se_cmd; 99 /* work item used for cmwq dispatch to vhost_scsi_submission_work() */ 100 struct work_struct work; 101 /* Copy of the incoming SCSI command descriptor block (CDB) */ 102 unsigned char tvc_cdb[VHOST_SCSI_MAX_CDB_SIZE]; 103 /* Sense buffer that will be mapped into outgoing status */ 104 unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER]; 105 /* Completed commands list, serviced from vhost worker thread */ 106 struct llist_node tvc_completion_list; 107 /* Used to track inflight cmd */ 108 struct vhost_scsi_inflight *inflight; 109 }; 110 111 struct vhost_scsi_nexus { 112 /* Pointer to TCM session for I_T Nexus */ 113 struct se_session *tvn_se_sess; 114 }; 115 116 struct vhost_scsi_tpg { 117 /* Vhost port target portal group tag for TCM */ 118 u16 tport_tpgt; 119 /* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */ 120 int tv_tpg_port_count; 121 /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */ 122 int tv_tpg_vhost_count; 123 /* Used for enabling T10-PI with legacy devices */ 124 int tv_fabric_prot_type; 125 /* list for vhost_scsi_list */ 126 struct list_head tv_tpg_list; 127 /* Used to protect access for tpg_nexus */ 128 struct mutex tv_tpg_mutex; 129 /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */ 130 struct vhost_scsi_nexus *tpg_nexus; 131 /* Pointer back to vhost_scsi_tport */ 132 struct vhost_scsi_tport *tport; 133 /* Returned by vhost_scsi_make_tpg() */ 134 struct se_portal_group se_tpg; 135 /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */ 136 struct vhost_scsi *vhost_scsi; 137 }; 138 139 struct vhost_scsi_tport { 140 /* SCSI protocol the tport is providing */ 141 u8 tport_proto_id; 142 /* Binary World Wide unique Port Name for Vhost Target port */ 143 u64 tport_wwpn; 144 /* ASCII formatted WWPN for Vhost Target port */ 145 char tport_name[VHOST_SCSI_NAMELEN]; 146 /* Returned by vhost_scsi_make_tport() */ 147 struct se_wwn tport_wwn; 148 }; 149 150 struct vhost_scsi_evt { 151 /* event to be sent to guest */ 152 struct virtio_scsi_event event; 153 /* event list, serviced from vhost worker thread */ 154 struct llist_node list; 155 }; 156 157 enum { 158 VHOST_SCSI_VQ_CTL = 0, 159 VHOST_SCSI_VQ_EVT = 1, 160 VHOST_SCSI_VQ_IO = 2, 161 }; 162 163 /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */ 164 enum { 165 VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) | 166 (1ULL << VIRTIO_SCSI_F_T10_PI) 167 }; 168 169 #define VHOST_SCSI_MAX_TARGET 256 170 #define VHOST_SCSI_MAX_VQ 128 171 #define VHOST_SCSI_MAX_EVENT 128 172 173 struct vhost_scsi_virtqueue { 174 struct vhost_virtqueue vq; 175 /* 176 * Reference counting for inflight reqs, used for flush operation. At 177 * each time, one reference tracks new commands submitted, while we 178 * wait for another one to reach 0. 179 */ 180 struct vhost_scsi_inflight inflights[2]; 181 /* 182 * Indicate current inflight in use, protected by vq->mutex. 183 * Writers must also take dev mutex and flush under it. 184 */ 185 int inflight_idx; 186 }; 187 188 struct vhost_scsi { 189 /* Protected by vhost_scsi->dev.mutex */ 190 struct vhost_scsi_tpg **vs_tpg; 191 char vs_vhost_wwpn[TRANSPORT_IQN_LEN]; 192 193 struct vhost_dev dev; 194 struct vhost_scsi_virtqueue vqs[VHOST_SCSI_MAX_VQ]; 195 196 struct vhost_work vs_completion_work; /* cmd completion work item */ 197 struct llist_head vs_completion_list; /* cmd completion queue */ 198 199 struct vhost_work vs_event_work; /* evt injection work item */ 200 struct llist_head vs_event_list; /* evt injection queue */ 201 202 bool vs_events_missed; /* any missed events, protected by vq->mutex */ 203 int vs_events_nr; /* num of pending events, protected by vq->mutex */ 204 }; 205 206 /* 207 * Context for processing request and control queue operations. 208 */ 209 struct vhost_scsi_ctx { 210 int head; 211 unsigned int out, in; 212 size_t req_size, rsp_size; 213 size_t out_size, in_size; 214 u8 *target, *lunp; 215 void *req; 216 struct iov_iter out_iter; 217 }; 218 219 static struct workqueue_struct *vhost_scsi_workqueue; 220 221 /* Global spinlock to protect vhost_scsi TPG list for vhost IOCTL access */ 222 static DEFINE_MUTEX(vhost_scsi_mutex); 223 static LIST_HEAD(vhost_scsi_list); 224 225 static void vhost_scsi_done_inflight(struct kref *kref) 226 { 227 struct vhost_scsi_inflight *inflight; 228 229 inflight = container_of(kref, struct vhost_scsi_inflight, kref); 230 complete(&inflight->comp); 231 } 232 233 static void vhost_scsi_init_inflight(struct vhost_scsi *vs, 234 struct vhost_scsi_inflight *old_inflight[]) 235 { 236 struct vhost_scsi_inflight *new_inflight; 237 struct vhost_virtqueue *vq; 238 int idx, i; 239 240 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { 241 vq = &vs->vqs[i].vq; 242 243 mutex_lock(&vq->mutex); 244 245 /* store old infight */ 246 idx = vs->vqs[i].inflight_idx; 247 if (old_inflight) 248 old_inflight[i] = &vs->vqs[i].inflights[idx]; 249 250 /* setup new infight */ 251 vs->vqs[i].inflight_idx = idx ^ 1; 252 new_inflight = &vs->vqs[i].inflights[idx ^ 1]; 253 kref_init(&new_inflight->kref); 254 init_completion(&new_inflight->comp); 255 256 mutex_unlock(&vq->mutex); 257 } 258 } 259 260 static struct vhost_scsi_inflight * 261 vhost_scsi_get_inflight(struct vhost_virtqueue *vq) 262 { 263 struct vhost_scsi_inflight *inflight; 264 struct vhost_scsi_virtqueue *svq; 265 266 svq = container_of(vq, struct vhost_scsi_virtqueue, vq); 267 inflight = &svq->inflights[svq->inflight_idx]; 268 kref_get(&inflight->kref); 269 270 return inflight; 271 } 272 273 static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight) 274 { 275 kref_put(&inflight->kref, vhost_scsi_done_inflight); 276 } 277 278 static int vhost_scsi_check_true(struct se_portal_group *se_tpg) 279 { 280 return 1; 281 } 282 283 static int vhost_scsi_check_false(struct se_portal_group *se_tpg) 284 { 285 return 0; 286 } 287 288 static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg) 289 { 290 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 291 struct vhost_scsi_tpg, se_tpg); 292 struct vhost_scsi_tport *tport = tpg->tport; 293 294 return &tport->tport_name[0]; 295 } 296 297 static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg) 298 { 299 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 300 struct vhost_scsi_tpg, se_tpg); 301 return tpg->tport_tpgt; 302 } 303 304 static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg) 305 { 306 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 307 struct vhost_scsi_tpg, se_tpg); 308 309 return tpg->tv_fabric_prot_type; 310 } 311 312 static u32 vhost_scsi_tpg_get_inst_index(struct se_portal_group *se_tpg) 313 { 314 return 1; 315 } 316 317 static void vhost_scsi_release_cmd(struct se_cmd *se_cmd) 318 { 319 struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd, 320 struct vhost_scsi_cmd, tvc_se_cmd); 321 struct se_session *se_sess = tv_cmd->tvc_nexus->tvn_se_sess; 322 int i; 323 324 if (tv_cmd->tvc_sgl_count) { 325 for (i = 0; i < tv_cmd->tvc_sgl_count; i++) 326 put_page(sg_page(&tv_cmd->tvc_sgl[i])); 327 } 328 if (tv_cmd->tvc_prot_sgl_count) { 329 for (i = 0; i < tv_cmd->tvc_prot_sgl_count; i++) 330 put_page(sg_page(&tv_cmd->tvc_prot_sgl[i])); 331 } 332 333 vhost_scsi_put_inflight(tv_cmd->inflight); 334 target_free_tag(se_sess, se_cmd); 335 } 336 337 static u32 vhost_scsi_sess_get_index(struct se_session *se_sess) 338 { 339 return 0; 340 } 341 342 static int vhost_scsi_write_pending(struct se_cmd *se_cmd) 343 { 344 /* Go ahead and process the write immediately */ 345 target_execute_cmd(se_cmd); 346 return 0; 347 } 348 349 static int vhost_scsi_write_pending_status(struct se_cmd *se_cmd) 350 { 351 return 0; 352 } 353 354 static void vhost_scsi_set_default_node_attrs(struct se_node_acl *nacl) 355 { 356 return; 357 } 358 359 static int vhost_scsi_get_cmd_state(struct se_cmd *se_cmd) 360 { 361 return 0; 362 } 363 364 static void vhost_scsi_complete_cmd(struct vhost_scsi_cmd *cmd) 365 { 366 struct vhost_scsi *vs = cmd->tvc_vhost; 367 368 llist_add(&cmd->tvc_completion_list, &vs->vs_completion_list); 369 370 vhost_work_queue(&vs->dev, &vs->vs_completion_work); 371 } 372 373 static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd) 374 { 375 struct vhost_scsi_cmd *cmd = container_of(se_cmd, 376 struct vhost_scsi_cmd, tvc_se_cmd); 377 vhost_scsi_complete_cmd(cmd); 378 return 0; 379 } 380 381 static int vhost_scsi_queue_status(struct se_cmd *se_cmd) 382 { 383 struct vhost_scsi_cmd *cmd = container_of(se_cmd, 384 struct vhost_scsi_cmd, tvc_se_cmd); 385 vhost_scsi_complete_cmd(cmd); 386 return 0; 387 } 388 389 static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd) 390 { 391 return; 392 } 393 394 static void vhost_scsi_aborted_task(struct se_cmd *se_cmd) 395 { 396 return; 397 } 398 399 static void vhost_scsi_free_evt(struct vhost_scsi *vs, struct vhost_scsi_evt *evt) 400 { 401 vs->vs_events_nr--; 402 kfree(evt); 403 } 404 405 static struct vhost_scsi_evt * 406 vhost_scsi_allocate_evt(struct vhost_scsi *vs, 407 u32 event, u32 reason) 408 { 409 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; 410 struct vhost_scsi_evt *evt; 411 412 if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) { 413 vs->vs_events_missed = true; 414 return NULL; 415 } 416 417 evt = kzalloc(sizeof(*evt), GFP_KERNEL); 418 if (!evt) { 419 vq_err(vq, "Failed to allocate vhost_scsi_evt\n"); 420 vs->vs_events_missed = true; 421 return NULL; 422 } 423 424 evt->event.event = cpu_to_vhost32(vq, event); 425 evt->event.reason = cpu_to_vhost32(vq, reason); 426 vs->vs_events_nr++; 427 428 return evt; 429 } 430 431 static void vhost_scsi_free_cmd(struct vhost_scsi_cmd *cmd) 432 { 433 struct se_cmd *se_cmd = &cmd->tvc_se_cmd; 434 435 /* TODO locking against target/backend threads? */ 436 transport_generic_free_cmd(se_cmd, 0); 437 438 } 439 440 static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd) 441 { 442 return target_put_sess_cmd(se_cmd); 443 } 444 445 static void 446 vhost_scsi_do_evt_work(struct vhost_scsi *vs, struct vhost_scsi_evt *evt) 447 { 448 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; 449 struct virtio_scsi_event *event = &evt->event; 450 struct virtio_scsi_event __user *eventp; 451 unsigned out, in; 452 int head, ret; 453 454 if (!vq->private_data) { 455 vs->vs_events_missed = true; 456 return; 457 } 458 459 again: 460 vhost_disable_notify(&vs->dev, vq); 461 head = vhost_get_vq_desc(vq, vq->iov, 462 ARRAY_SIZE(vq->iov), &out, &in, 463 NULL, NULL); 464 if (head < 0) { 465 vs->vs_events_missed = true; 466 return; 467 } 468 if (head == vq->num) { 469 if (vhost_enable_notify(&vs->dev, vq)) 470 goto again; 471 vs->vs_events_missed = true; 472 return; 473 } 474 475 if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) { 476 vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n", 477 vq->iov[out].iov_len); 478 vs->vs_events_missed = true; 479 return; 480 } 481 482 if (vs->vs_events_missed) { 483 event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED); 484 vs->vs_events_missed = false; 485 } 486 487 eventp = vq->iov[out].iov_base; 488 ret = __copy_to_user(eventp, event, sizeof(*event)); 489 if (!ret) 490 vhost_add_used_and_signal(&vs->dev, vq, head, 0); 491 else 492 vq_err(vq, "Faulted on vhost_scsi_send_event\n"); 493 } 494 495 static void vhost_scsi_evt_work(struct vhost_work *work) 496 { 497 struct vhost_scsi *vs = container_of(work, struct vhost_scsi, 498 vs_event_work); 499 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; 500 struct vhost_scsi_evt *evt, *t; 501 struct llist_node *llnode; 502 503 mutex_lock(&vq->mutex); 504 llnode = llist_del_all(&vs->vs_event_list); 505 llist_for_each_entry_safe(evt, t, llnode, list) { 506 vhost_scsi_do_evt_work(vs, evt); 507 vhost_scsi_free_evt(vs, evt); 508 } 509 mutex_unlock(&vq->mutex); 510 } 511 512 /* Fill in status and signal that we are done processing this command 513 * 514 * This is scheduled in the vhost work queue so we are called with the owner 515 * process mm and can access the vring. 516 */ 517 static void vhost_scsi_complete_cmd_work(struct vhost_work *work) 518 { 519 struct vhost_scsi *vs = container_of(work, struct vhost_scsi, 520 vs_completion_work); 521 DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ); 522 struct virtio_scsi_cmd_resp v_rsp; 523 struct vhost_scsi_cmd *cmd, *t; 524 struct llist_node *llnode; 525 struct se_cmd *se_cmd; 526 struct iov_iter iov_iter; 527 int ret, vq; 528 529 bitmap_zero(signal, VHOST_SCSI_MAX_VQ); 530 llnode = llist_del_all(&vs->vs_completion_list); 531 llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) { 532 se_cmd = &cmd->tvc_se_cmd; 533 534 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__, 535 cmd, se_cmd->residual_count, se_cmd->scsi_status); 536 537 memset(&v_rsp, 0, sizeof(v_rsp)); 538 v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq, se_cmd->residual_count); 539 /* TODO is status_qualifier field needed? */ 540 v_rsp.status = se_cmd->scsi_status; 541 v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq, 542 se_cmd->scsi_sense_length); 543 memcpy(v_rsp.sense, cmd->tvc_sense_buf, 544 se_cmd->scsi_sense_length); 545 546 iov_iter_init(&iov_iter, READ, &cmd->tvc_resp_iov, 547 cmd->tvc_in_iovs, sizeof(v_rsp)); 548 ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter); 549 if (likely(ret == sizeof(v_rsp))) { 550 struct vhost_scsi_virtqueue *q; 551 vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0); 552 q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); 553 vq = q - vs->vqs; 554 __set_bit(vq, signal); 555 } else 556 pr_err("Faulted on virtio_scsi_cmd_resp\n"); 557 558 vhost_scsi_free_cmd(cmd); 559 } 560 561 vq = -1; 562 while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1)) 563 < VHOST_SCSI_MAX_VQ) 564 vhost_signal(&vs->dev, &vs->vqs[vq].vq); 565 } 566 567 static struct vhost_scsi_cmd * 568 vhost_scsi_get_tag(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg, 569 unsigned char *cdb, u64 scsi_tag, u16 lun, u8 task_attr, 570 u32 exp_data_len, int data_direction) 571 { 572 struct vhost_scsi_cmd *cmd; 573 struct vhost_scsi_nexus *tv_nexus; 574 struct se_session *se_sess; 575 struct scatterlist *sg, *prot_sg; 576 struct page **pages; 577 int tag, cpu; 578 579 tv_nexus = tpg->tpg_nexus; 580 if (!tv_nexus) { 581 pr_err("Unable to locate active struct vhost_scsi_nexus\n"); 582 return ERR_PTR(-EIO); 583 } 584 se_sess = tv_nexus->tvn_se_sess; 585 586 tag = sbitmap_queue_get(&se_sess->sess_tag_pool, &cpu); 587 if (tag < 0) { 588 pr_err("Unable to obtain tag for vhost_scsi_cmd\n"); 589 return ERR_PTR(-ENOMEM); 590 } 591 592 cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[tag]; 593 sg = cmd->tvc_sgl; 594 prot_sg = cmd->tvc_prot_sgl; 595 pages = cmd->tvc_upages; 596 memset(cmd, 0, sizeof(*cmd)); 597 cmd->tvc_sgl = sg; 598 cmd->tvc_prot_sgl = prot_sg; 599 cmd->tvc_upages = pages; 600 cmd->tvc_se_cmd.map_tag = tag; 601 cmd->tvc_se_cmd.map_cpu = cpu; 602 cmd->tvc_tag = scsi_tag; 603 cmd->tvc_lun = lun; 604 cmd->tvc_task_attr = task_attr; 605 cmd->tvc_exp_data_len = exp_data_len; 606 cmd->tvc_data_direction = data_direction; 607 cmd->tvc_nexus = tv_nexus; 608 cmd->inflight = vhost_scsi_get_inflight(vq); 609 610 memcpy(cmd->tvc_cdb, cdb, VHOST_SCSI_MAX_CDB_SIZE); 611 612 return cmd; 613 } 614 615 /* 616 * Map a user memory range into a scatterlist 617 * 618 * Returns the number of scatterlist entries used or -errno on error. 619 */ 620 static int 621 vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd, 622 struct iov_iter *iter, 623 struct scatterlist *sgl, 624 bool write) 625 { 626 struct page **pages = cmd->tvc_upages; 627 struct scatterlist *sg = sgl; 628 ssize_t bytes; 629 size_t offset; 630 unsigned int npages = 0; 631 632 bytes = iov_iter_get_pages(iter, pages, LONG_MAX, 633 VHOST_SCSI_PREALLOC_UPAGES, &offset); 634 /* No pages were pinned */ 635 if (bytes <= 0) 636 return bytes < 0 ? bytes : -EFAULT; 637 638 iov_iter_advance(iter, bytes); 639 640 while (bytes) { 641 unsigned n = min_t(unsigned, PAGE_SIZE - offset, bytes); 642 sg_set_page(sg++, pages[npages++], n, offset); 643 bytes -= n; 644 offset = 0; 645 } 646 return npages; 647 } 648 649 static int 650 vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls) 651 { 652 int sgl_count = 0; 653 654 if (!iter || !iter->iov) { 655 pr_err("%s: iter->iov is NULL, but expected bytes: %zu" 656 " present\n", __func__, bytes); 657 return -EINVAL; 658 } 659 660 sgl_count = iov_iter_npages(iter, 0xffff); 661 if (sgl_count > max_sgls) { 662 pr_err("%s: requested sgl_count: %d exceeds pre-allocated" 663 " max_sgls: %d\n", __func__, sgl_count, max_sgls); 664 return -EINVAL; 665 } 666 return sgl_count; 667 } 668 669 static int 670 vhost_scsi_iov_to_sgl(struct vhost_scsi_cmd *cmd, bool write, 671 struct iov_iter *iter, 672 struct scatterlist *sg, int sg_count) 673 { 674 struct scatterlist *p = sg; 675 int ret; 676 677 while (iov_iter_count(iter)) { 678 ret = vhost_scsi_map_to_sgl(cmd, iter, sg, write); 679 if (ret < 0) { 680 while (p < sg) { 681 struct page *page = sg_page(p++); 682 if (page) 683 put_page(page); 684 } 685 return ret; 686 } 687 sg += ret; 688 } 689 return 0; 690 } 691 692 static int 693 vhost_scsi_mapal(struct vhost_scsi_cmd *cmd, 694 size_t prot_bytes, struct iov_iter *prot_iter, 695 size_t data_bytes, struct iov_iter *data_iter) 696 { 697 int sgl_count, ret; 698 bool write = (cmd->tvc_data_direction == DMA_FROM_DEVICE); 699 700 if (prot_bytes) { 701 sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes, 702 VHOST_SCSI_PREALLOC_PROT_SGLS); 703 if (sgl_count < 0) 704 return sgl_count; 705 706 sg_init_table(cmd->tvc_prot_sgl, sgl_count); 707 cmd->tvc_prot_sgl_count = sgl_count; 708 pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__, 709 cmd->tvc_prot_sgl, cmd->tvc_prot_sgl_count); 710 711 ret = vhost_scsi_iov_to_sgl(cmd, write, prot_iter, 712 cmd->tvc_prot_sgl, 713 cmd->tvc_prot_sgl_count); 714 if (ret < 0) { 715 cmd->tvc_prot_sgl_count = 0; 716 return ret; 717 } 718 } 719 sgl_count = vhost_scsi_calc_sgls(data_iter, data_bytes, 720 VHOST_SCSI_PREALLOC_SGLS); 721 if (sgl_count < 0) 722 return sgl_count; 723 724 sg_init_table(cmd->tvc_sgl, sgl_count); 725 cmd->tvc_sgl_count = sgl_count; 726 pr_debug("%s data_sg %p data_sgl_count %u\n", __func__, 727 cmd->tvc_sgl, cmd->tvc_sgl_count); 728 729 ret = vhost_scsi_iov_to_sgl(cmd, write, data_iter, 730 cmd->tvc_sgl, cmd->tvc_sgl_count); 731 if (ret < 0) { 732 cmd->tvc_sgl_count = 0; 733 return ret; 734 } 735 return 0; 736 } 737 738 static int vhost_scsi_to_tcm_attr(int attr) 739 { 740 switch (attr) { 741 case VIRTIO_SCSI_S_SIMPLE: 742 return TCM_SIMPLE_TAG; 743 case VIRTIO_SCSI_S_ORDERED: 744 return TCM_ORDERED_TAG; 745 case VIRTIO_SCSI_S_HEAD: 746 return TCM_HEAD_TAG; 747 case VIRTIO_SCSI_S_ACA: 748 return TCM_ACA_TAG; 749 default: 750 break; 751 } 752 return TCM_SIMPLE_TAG; 753 } 754 755 static void vhost_scsi_submission_work(struct work_struct *work) 756 { 757 struct vhost_scsi_cmd *cmd = 758 container_of(work, struct vhost_scsi_cmd, work); 759 struct vhost_scsi_nexus *tv_nexus; 760 struct se_cmd *se_cmd = &cmd->tvc_se_cmd; 761 struct scatterlist *sg_ptr, *sg_prot_ptr = NULL; 762 int rc; 763 764 /* FIXME: BIDI operation */ 765 if (cmd->tvc_sgl_count) { 766 sg_ptr = cmd->tvc_sgl; 767 768 if (cmd->tvc_prot_sgl_count) 769 sg_prot_ptr = cmd->tvc_prot_sgl; 770 else 771 se_cmd->prot_pto = true; 772 } else { 773 sg_ptr = NULL; 774 } 775 tv_nexus = cmd->tvc_nexus; 776 777 se_cmd->tag = 0; 778 rc = target_submit_cmd_map_sgls(se_cmd, tv_nexus->tvn_se_sess, 779 cmd->tvc_cdb, &cmd->tvc_sense_buf[0], 780 cmd->tvc_lun, cmd->tvc_exp_data_len, 781 vhost_scsi_to_tcm_attr(cmd->tvc_task_attr), 782 cmd->tvc_data_direction, TARGET_SCF_ACK_KREF, 783 sg_ptr, cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr, 784 cmd->tvc_prot_sgl_count); 785 if (rc < 0) { 786 transport_send_check_condition_and_sense(se_cmd, 787 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE, 0); 788 transport_generic_free_cmd(se_cmd, 0); 789 } 790 } 791 792 static void 793 vhost_scsi_send_bad_target(struct vhost_scsi *vs, 794 struct vhost_virtqueue *vq, 795 int head, unsigned out) 796 { 797 struct virtio_scsi_cmd_resp __user *resp; 798 struct virtio_scsi_cmd_resp rsp; 799 int ret; 800 801 memset(&rsp, 0, sizeof(rsp)); 802 rsp.response = VIRTIO_SCSI_S_BAD_TARGET; 803 resp = vq->iov[out].iov_base; 804 ret = __copy_to_user(resp, &rsp, sizeof(rsp)); 805 if (!ret) 806 vhost_add_used_and_signal(&vs->dev, vq, head, 0); 807 else 808 pr_err("Faulted on virtio_scsi_cmd_resp\n"); 809 } 810 811 static int 812 vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq, 813 struct vhost_scsi_ctx *vc) 814 { 815 int ret = -ENXIO; 816 817 vc->head = vhost_get_vq_desc(vq, vq->iov, 818 ARRAY_SIZE(vq->iov), &vc->out, &vc->in, 819 NULL, NULL); 820 821 pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n", 822 vc->head, vc->out, vc->in); 823 824 /* On error, stop handling until the next kick. */ 825 if (unlikely(vc->head < 0)) 826 goto done; 827 828 /* Nothing new? Wait for eventfd to tell us they refilled. */ 829 if (vc->head == vq->num) { 830 if (unlikely(vhost_enable_notify(&vs->dev, vq))) { 831 vhost_disable_notify(&vs->dev, vq); 832 ret = -EAGAIN; 833 } 834 goto done; 835 } 836 837 /* 838 * Get the size of request and response buffers. 839 * FIXME: Not correct for BIDI operation 840 */ 841 vc->out_size = iov_length(vq->iov, vc->out); 842 vc->in_size = iov_length(&vq->iov[vc->out], vc->in); 843 844 /* 845 * Copy over the virtio-scsi request header, which for a 846 * ANY_LAYOUT enabled guest may span multiple iovecs, or a 847 * single iovec may contain both the header + outgoing 848 * WRITE payloads. 849 * 850 * copy_from_iter() will advance out_iter, so that it will 851 * point at the start of the outgoing WRITE payload, if 852 * DMA_TO_DEVICE is set. 853 */ 854 iov_iter_init(&vc->out_iter, WRITE, vq->iov, vc->out, vc->out_size); 855 ret = 0; 856 857 done: 858 return ret; 859 } 860 861 static int 862 vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc) 863 { 864 if (unlikely(vc->in_size < vc->rsp_size)) { 865 vq_err(vq, 866 "Response buf too small, need min %zu bytes got %zu", 867 vc->rsp_size, vc->in_size); 868 return -EINVAL; 869 } else if (unlikely(vc->out_size < vc->req_size)) { 870 vq_err(vq, 871 "Request buf too small, need min %zu bytes got %zu", 872 vc->req_size, vc->out_size); 873 return -EIO; 874 } 875 876 return 0; 877 } 878 879 static int 880 vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc, 881 struct vhost_scsi_tpg **tpgp) 882 { 883 int ret = -EIO; 884 885 if (unlikely(!copy_from_iter_full(vc->req, vc->req_size, 886 &vc->out_iter))) { 887 vq_err(vq, "Faulted on copy_from_iter_full\n"); 888 } else if (unlikely(*vc->lunp != 1)) { 889 /* virtio-scsi spec requires byte 0 of the lun to be 1 */ 890 vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp); 891 } else { 892 struct vhost_scsi_tpg **vs_tpg, *tpg; 893 894 vs_tpg = vq->private_data; /* validated at handler entry */ 895 896 tpg = READ_ONCE(vs_tpg[*vc->target]); 897 if (unlikely(!tpg)) { 898 vq_err(vq, "Target 0x%x does not exist\n", *vc->target); 899 } else { 900 if (tpgp) 901 *tpgp = tpg; 902 ret = 0; 903 } 904 } 905 906 return ret; 907 } 908 909 static void 910 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) 911 { 912 struct vhost_scsi_tpg **vs_tpg, *tpg; 913 struct virtio_scsi_cmd_req v_req; 914 struct virtio_scsi_cmd_req_pi v_req_pi; 915 struct vhost_scsi_ctx vc; 916 struct vhost_scsi_cmd *cmd; 917 struct iov_iter in_iter, prot_iter, data_iter; 918 u64 tag; 919 u32 exp_data_len, data_direction; 920 int ret, prot_bytes; 921 u16 lun; 922 u8 task_attr; 923 bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI); 924 void *cdb; 925 926 mutex_lock(&vq->mutex); 927 /* 928 * We can handle the vq only after the endpoint is setup by calling the 929 * VHOST_SCSI_SET_ENDPOINT ioctl. 930 */ 931 vs_tpg = vq->private_data; 932 if (!vs_tpg) 933 goto out; 934 935 memset(&vc, 0, sizeof(vc)); 936 vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp); 937 938 vhost_disable_notify(&vs->dev, vq); 939 940 for (;;) { 941 ret = vhost_scsi_get_desc(vs, vq, &vc); 942 if (ret) 943 goto err; 944 945 /* 946 * Setup pointers and values based upon different virtio-scsi 947 * request header if T10_PI is enabled in KVM guest. 948 */ 949 if (t10_pi) { 950 vc.req = &v_req_pi; 951 vc.req_size = sizeof(v_req_pi); 952 vc.lunp = &v_req_pi.lun[0]; 953 vc.target = &v_req_pi.lun[1]; 954 } else { 955 vc.req = &v_req; 956 vc.req_size = sizeof(v_req); 957 vc.lunp = &v_req.lun[0]; 958 vc.target = &v_req.lun[1]; 959 } 960 961 /* 962 * Validate the size of request and response buffers. 963 * Check for a sane response buffer so we can report 964 * early errors back to the guest. 965 */ 966 ret = vhost_scsi_chk_size(vq, &vc); 967 if (ret) 968 goto err; 969 970 ret = vhost_scsi_get_req(vq, &vc, &tpg); 971 if (ret) 972 goto err; 973 974 ret = -EIO; /* bad target on any error from here on */ 975 976 /* 977 * Determine data_direction by calculating the total outgoing 978 * iovec sizes + incoming iovec sizes vs. virtio-scsi request + 979 * response headers respectively. 980 * 981 * For DMA_TO_DEVICE this is out_iter, which is already pointing 982 * to the right place. 983 * 984 * For DMA_FROM_DEVICE, the iovec will be just past the end 985 * of the virtio-scsi response header in either the same 986 * or immediately following iovec. 987 * 988 * Any associated T10_PI bytes for the outgoing / incoming 989 * payloads are included in calculation of exp_data_len here. 990 */ 991 prot_bytes = 0; 992 993 if (vc.out_size > vc.req_size) { 994 data_direction = DMA_TO_DEVICE; 995 exp_data_len = vc.out_size - vc.req_size; 996 data_iter = vc.out_iter; 997 } else if (vc.in_size > vc.rsp_size) { 998 data_direction = DMA_FROM_DEVICE; 999 exp_data_len = vc.in_size - vc.rsp_size; 1000 1001 iov_iter_init(&in_iter, READ, &vq->iov[vc.out], vc.in, 1002 vc.rsp_size + exp_data_len); 1003 iov_iter_advance(&in_iter, vc.rsp_size); 1004 data_iter = in_iter; 1005 } else { 1006 data_direction = DMA_NONE; 1007 exp_data_len = 0; 1008 } 1009 /* 1010 * If T10_PI header + payload is present, setup prot_iter values 1011 * and recalculate data_iter for vhost_scsi_mapal() mapping to 1012 * host scatterlists via get_user_pages_fast(). 1013 */ 1014 if (t10_pi) { 1015 if (v_req_pi.pi_bytesout) { 1016 if (data_direction != DMA_TO_DEVICE) { 1017 vq_err(vq, "Received non zero pi_bytesout," 1018 " but wrong data_direction\n"); 1019 goto err; 1020 } 1021 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout); 1022 } else if (v_req_pi.pi_bytesin) { 1023 if (data_direction != DMA_FROM_DEVICE) { 1024 vq_err(vq, "Received non zero pi_bytesin," 1025 " but wrong data_direction\n"); 1026 goto err; 1027 } 1028 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin); 1029 } 1030 /* 1031 * Set prot_iter to data_iter and truncate it to 1032 * prot_bytes, and advance data_iter past any 1033 * preceeding prot_bytes that may be present. 1034 * 1035 * Also fix up the exp_data_len to reflect only the 1036 * actual data payload length. 1037 */ 1038 if (prot_bytes) { 1039 exp_data_len -= prot_bytes; 1040 prot_iter = data_iter; 1041 iov_iter_truncate(&prot_iter, prot_bytes); 1042 iov_iter_advance(&data_iter, prot_bytes); 1043 } 1044 tag = vhost64_to_cpu(vq, v_req_pi.tag); 1045 task_attr = v_req_pi.task_attr; 1046 cdb = &v_req_pi.cdb[0]; 1047 lun = ((v_req_pi.lun[2] << 8) | v_req_pi.lun[3]) & 0x3FFF; 1048 } else { 1049 tag = vhost64_to_cpu(vq, v_req.tag); 1050 task_attr = v_req.task_attr; 1051 cdb = &v_req.cdb[0]; 1052 lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF; 1053 } 1054 /* 1055 * Check that the received CDB size does not exceeded our 1056 * hardcoded max for vhost-scsi, then get a pre-allocated 1057 * cmd descriptor for the new virtio-scsi tag. 1058 * 1059 * TODO what if cdb was too small for varlen cdb header? 1060 */ 1061 if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) { 1062 vq_err(vq, "Received SCSI CDB with command_size: %d that" 1063 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", 1064 scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE); 1065 goto err; 1066 } 1067 cmd = vhost_scsi_get_tag(vq, tpg, cdb, tag, lun, task_attr, 1068 exp_data_len + prot_bytes, 1069 data_direction); 1070 if (IS_ERR(cmd)) { 1071 vq_err(vq, "vhost_scsi_get_tag failed %ld\n", 1072 PTR_ERR(cmd)); 1073 goto err; 1074 } 1075 cmd->tvc_vhost = vs; 1076 cmd->tvc_vq = vq; 1077 cmd->tvc_resp_iov = vq->iov[vc.out]; 1078 cmd->tvc_in_iovs = vc.in; 1079 1080 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n", 1081 cmd->tvc_cdb[0], cmd->tvc_lun); 1082 pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:" 1083 " %d\n", cmd, exp_data_len, prot_bytes, data_direction); 1084 1085 if (data_direction != DMA_NONE) { 1086 if (unlikely(vhost_scsi_mapal(cmd, prot_bytes, 1087 &prot_iter, exp_data_len, 1088 &data_iter))) { 1089 vq_err(vq, "Failed to map iov to sgl\n"); 1090 vhost_scsi_release_cmd(&cmd->tvc_se_cmd); 1091 goto err; 1092 } 1093 } 1094 /* 1095 * Save the descriptor from vhost_get_vq_desc() to be used to 1096 * complete the virtio-scsi request in TCM callback context via 1097 * vhost_scsi_queue_data_in() and vhost_scsi_queue_status() 1098 */ 1099 cmd->tvc_vq_desc = vc.head; 1100 /* 1101 * Dispatch cmd descriptor for cmwq execution in process 1102 * context provided by vhost_scsi_workqueue. This also ensures 1103 * cmd is executed on the same kworker CPU as this vhost 1104 * thread to gain positive L2 cache locality effects. 1105 */ 1106 INIT_WORK(&cmd->work, vhost_scsi_submission_work); 1107 queue_work(vhost_scsi_workqueue, &cmd->work); 1108 ret = 0; 1109 err: 1110 /* 1111 * ENXIO: No more requests, or read error, wait for next kick 1112 * EINVAL: Invalid response buffer, drop the request 1113 * EIO: Respond with bad target 1114 * EAGAIN: Pending request 1115 */ 1116 if (ret == -ENXIO) 1117 break; 1118 else if (ret == -EIO) 1119 vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out); 1120 } 1121 out: 1122 mutex_unlock(&vq->mutex); 1123 } 1124 1125 static void 1126 vhost_scsi_send_tmf_reject(struct vhost_scsi *vs, 1127 struct vhost_virtqueue *vq, 1128 struct vhost_scsi_ctx *vc) 1129 { 1130 struct virtio_scsi_ctrl_tmf_resp __user *resp; 1131 struct virtio_scsi_ctrl_tmf_resp rsp; 1132 int ret; 1133 1134 pr_debug("%s\n", __func__); 1135 memset(&rsp, 0, sizeof(rsp)); 1136 rsp.response = VIRTIO_SCSI_S_FUNCTION_REJECTED; 1137 resp = vq->iov[vc->out].iov_base; 1138 ret = __copy_to_user(resp, &rsp, sizeof(rsp)); 1139 if (!ret) 1140 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0); 1141 else 1142 pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n"); 1143 } 1144 1145 static void 1146 vhost_scsi_send_an_resp(struct vhost_scsi *vs, 1147 struct vhost_virtqueue *vq, 1148 struct vhost_scsi_ctx *vc) 1149 { 1150 struct virtio_scsi_ctrl_an_resp __user *resp; 1151 struct virtio_scsi_ctrl_an_resp rsp; 1152 int ret; 1153 1154 pr_debug("%s\n", __func__); 1155 memset(&rsp, 0, sizeof(rsp)); /* event_actual = 0 */ 1156 rsp.response = VIRTIO_SCSI_S_OK; 1157 resp = vq->iov[vc->out].iov_base; 1158 ret = __copy_to_user(resp, &rsp, sizeof(rsp)); 1159 if (!ret) 1160 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0); 1161 else 1162 pr_err("Faulted on virtio_scsi_ctrl_an_resp\n"); 1163 } 1164 1165 static void 1166 vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) 1167 { 1168 union { 1169 __virtio32 type; 1170 struct virtio_scsi_ctrl_an_req an; 1171 struct virtio_scsi_ctrl_tmf_req tmf; 1172 } v_req; 1173 struct vhost_scsi_ctx vc; 1174 size_t typ_size; 1175 int ret; 1176 1177 mutex_lock(&vq->mutex); 1178 /* 1179 * We can handle the vq only after the endpoint is setup by calling the 1180 * VHOST_SCSI_SET_ENDPOINT ioctl. 1181 */ 1182 if (!vq->private_data) 1183 goto out; 1184 1185 memset(&vc, 0, sizeof(vc)); 1186 1187 vhost_disable_notify(&vs->dev, vq); 1188 1189 for (;;) { 1190 ret = vhost_scsi_get_desc(vs, vq, &vc); 1191 if (ret) 1192 goto err; 1193 1194 /* 1195 * Get the request type first in order to setup 1196 * other parameters dependent on the type. 1197 */ 1198 vc.req = &v_req.type; 1199 typ_size = sizeof(v_req.type); 1200 1201 if (unlikely(!copy_from_iter_full(vc.req, typ_size, 1202 &vc.out_iter))) { 1203 vq_err(vq, "Faulted on copy_from_iter tmf type\n"); 1204 /* 1205 * The size of the response buffer depends on the 1206 * request type and must be validated against it. 1207 * Since the request type is not known, don't send 1208 * a response. 1209 */ 1210 continue; 1211 } 1212 1213 switch (v_req.type) { 1214 case VIRTIO_SCSI_T_TMF: 1215 vc.req = &v_req.tmf; 1216 vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req); 1217 vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp); 1218 vc.lunp = &v_req.tmf.lun[0]; 1219 vc.target = &v_req.tmf.lun[1]; 1220 break; 1221 case VIRTIO_SCSI_T_AN_QUERY: 1222 case VIRTIO_SCSI_T_AN_SUBSCRIBE: 1223 vc.req = &v_req.an; 1224 vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req); 1225 vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp); 1226 vc.lunp = &v_req.an.lun[0]; 1227 vc.target = NULL; 1228 break; 1229 default: 1230 vq_err(vq, "Unknown control request %d", v_req.type); 1231 continue; 1232 } 1233 1234 /* 1235 * Validate the size of request and response buffers. 1236 * Check for a sane response buffer so we can report 1237 * early errors back to the guest. 1238 */ 1239 ret = vhost_scsi_chk_size(vq, &vc); 1240 if (ret) 1241 goto err; 1242 1243 /* 1244 * Get the rest of the request now that its size is known. 1245 */ 1246 vc.req += typ_size; 1247 vc.req_size -= typ_size; 1248 1249 ret = vhost_scsi_get_req(vq, &vc, NULL); 1250 if (ret) 1251 goto err; 1252 1253 if (v_req.type == VIRTIO_SCSI_T_TMF) 1254 vhost_scsi_send_tmf_reject(vs, vq, &vc); 1255 else 1256 vhost_scsi_send_an_resp(vs, vq, &vc); 1257 err: 1258 /* 1259 * ENXIO: No more requests, or read error, wait for next kick 1260 * EINVAL: Invalid response buffer, drop the request 1261 * EIO: Respond with bad target 1262 * EAGAIN: Pending request 1263 */ 1264 if (ret == -ENXIO) 1265 break; 1266 else if (ret == -EIO) 1267 vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out); 1268 } 1269 out: 1270 mutex_unlock(&vq->mutex); 1271 } 1272 1273 static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) 1274 { 1275 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 1276 poll.work); 1277 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); 1278 1279 pr_debug("%s: The handling func for control queue.\n", __func__); 1280 vhost_scsi_ctl_handle_vq(vs, vq); 1281 } 1282 1283 static void 1284 vhost_scsi_send_evt(struct vhost_scsi *vs, 1285 struct vhost_scsi_tpg *tpg, 1286 struct se_lun *lun, 1287 u32 event, 1288 u32 reason) 1289 { 1290 struct vhost_scsi_evt *evt; 1291 1292 evt = vhost_scsi_allocate_evt(vs, event, reason); 1293 if (!evt) 1294 return; 1295 1296 if (tpg && lun) { 1297 /* TODO: share lun setup code with virtio-scsi.ko */ 1298 /* 1299 * Note: evt->event is zeroed when we allocate it and 1300 * lun[4-7] need to be zero according to virtio-scsi spec. 1301 */ 1302 evt->event.lun[0] = 0x01; 1303 evt->event.lun[1] = tpg->tport_tpgt; 1304 if (lun->unpacked_lun >= 256) 1305 evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ; 1306 evt->event.lun[3] = lun->unpacked_lun & 0xFF; 1307 } 1308 1309 llist_add(&evt->list, &vs->vs_event_list); 1310 vhost_work_queue(&vs->dev, &vs->vs_event_work); 1311 } 1312 1313 static void vhost_scsi_evt_handle_kick(struct vhost_work *work) 1314 { 1315 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 1316 poll.work); 1317 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); 1318 1319 mutex_lock(&vq->mutex); 1320 if (!vq->private_data) 1321 goto out; 1322 1323 if (vs->vs_events_missed) 1324 vhost_scsi_send_evt(vs, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT, 0); 1325 out: 1326 mutex_unlock(&vq->mutex); 1327 } 1328 1329 static void vhost_scsi_handle_kick(struct vhost_work *work) 1330 { 1331 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 1332 poll.work); 1333 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); 1334 1335 vhost_scsi_handle_vq(vs, vq); 1336 } 1337 1338 static void vhost_scsi_flush_vq(struct vhost_scsi *vs, int index) 1339 { 1340 vhost_poll_flush(&vs->vqs[index].vq.poll); 1341 } 1342 1343 /* Callers must hold dev mutex */ 1344 static void vhost_scsi_flush(struct vhost_scsi *vs) 1345 { 1346 struct vhost_scsi_inflight *old_inflight[VHOST_SCSI_MAX_VQ]; 1347 int i; 1348 1349 /* Init new inflight and remember the old inflight */ 1350 vhost_scsi_init_inflight(vs, old_inflight); 1351 1352 /* 1353 * The inflight->kref was initialized to 1. We decrement it here to 1354 * indicate the start of the flush operation so that it will reach 0 1355 * when all the reqs are finished. 1356 */ 1357 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) 1358 kref_put(&old_inflight[i]->kref, vhost_scsi_done_inflight); 1359 1360 /* Flush both the vhost poll and vhost work */ 1361 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) 1362 vhost_scsi_flush_vq(vs, i); 1363 vhost_work_flush(&vs->dev, &vs->vs_completion_work); 1364 vhost_work_flush(&vs->dev, &vs->vs_event_work); 1365 1366 /* Wait for all reqs issued before the flush to be finished */ 1367 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) 1368 wait_for_completion(&old_inflight[i]->comp); 1369 } 1370 1371 /* 1372 * Called from vhost_scsi_ioctl() context to walk the list of available 1373 * vhost_scsi_tpg with an active struct vhost_scsi_nexus 1374 * 1375 * The lock nesting rule is: 1376 * vhost_scsi_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex 1377 */ 1378 static int 1379 vhost_scsi_set_endpoint(struct vhost_scsi *vs, 1380 struct vhost_scsi_target *t) 1381 { 1382 struct se_portal_group *se_tpg; 1383 struct vhost_scsi_tport *tv_tport; 1384 struct vhost_scsi_tpg *tpg; 1385 struct vhost_scsi_tpg **vs_tpg; 1386 struct vhost_virtqueue *vq; 1387 int index, ret, i, len; 1388 bool match = false; 1389 1390 mutex_lock(&vhost_scsi_mutex); 1391 mutex_lock(&vs->dev.mutex); 1392 1393 /* Verify that ring has been setup correctly. */ 1394 for (index = 0; index < vs->dev.nvqs; ++index) { 1395 /* Verify that ring has been setup correctly. */ 1396 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) { 1397 ret = -EFAULT; 1398 goto out; 1399 } 1400 } 1401 1402 len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET; 1403 vs_tpg = kzalloc(len, GFP_KERNEL); 1404 if (!vs_tpg) { 1405 ret = -ENOMEM; 1406 goto out; 1407 } 1408 if (vs->vs_tpg) 1409 memcpy(vs_tpg, vs->vs_tpg, len); 1410 1411 list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) { 1412 mutex_lock(&tpg->tv_tpg_mutex); 1413 if (!tpg->tpg_nexus) { 1414 mutex_unlock(&tpg->tv_tpg_mutex); 1415 continue; 1416 } 1417 if (tpg->tv_tpg_vhost_count != 0) { 1418 mutex_unlock(&tpg->tv_tpg_mutex); 1419 continue; 1420 } 1421 tv_tport = tpg->tport; 1422 1423 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) { 1424 if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) { 1425 kfree(vs_tpg); 1426 mutex_unlock(&tpg->tv_tpg_mutex); 1427 ret = -EEXIST; 1428 goto out; 1429 } 1430 /* 1431 * In order to ensure individual vhost-scsi configfs 1432 * groups cannot be removed while in use by vhost ioctl, 1433 * go ahead and take an explicit se_tpg->tpg_group.cg_item 1434 * dependency now. 1435 */ 1436 se_tpg = &tpg->se_tpg; 1437 ret = target_depend_item(&se_tpg->tpg_group.cg_item); 1438 if (ret) { 1439 pr_warn("target_depend_item() failed: %d\n", ret); 1440 kfree(vs_tpg); 1441 mutex_unlock(&tpg->tv_tpg_mutex); 1442 goto out; 1443 } 1444 tpg->tv_tpg_vhost_count++; 1445 tpg->vhost_scsi = vs; 1446 vs_tpg[tpg->tport_tpgt] = tpg; 1447 smp_mb__after_atomic(); 1448 match = true; 1449 } 1450 mutex_unlock(&tpg->tv_tpg_mutex); 1451 } 1452 1453 if (match) { 1454 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn, 1455 sizeof(vs->vs_vhost_wwpn)); 1456 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { 1457 vq = &vs->vqs[i].vq; 1458 mutex_lock(&vq->mutex); 1459 vq->private_data = vs_tpg; 1460 vhost_vq_init_access(vq); 1461 mutex_unlock(&vq->mutex); 1462 } 1463 ret = 0; 1464 } else { 1465 ret = -EEXIST; 1466 } 1467 1468 /* 1469 * Act as synchronize_rcu to make sure access to 1470 * old vs->vs_tpg is finished. 1471 */ 1472 vhost_scsi_flush(vs); 1473 kfree(vs->vs_tpg); 1474 vs->vs_tpg = vs_tpg; 1475 1476 out: 1477 mutex_unlock(&vs->dev.mutex); 1478 mutex_unlock(&vhost_scsi_mutex); 1479 return ret; 1480 } 1481 1482 static int 1483 vhost_scsi_clear_endpoint(struct vhost_scsi *vs, 1484 struct vhost_scsi_target *t) 1485 { 1486 struct se_portal_group *se_tpg; 1487 struct vhost_scsi_tport *tv_tport; 1488 struct vhost_scsi_tpg *tpg; 1489 struct vhost_virtqueue *vq; 1490 bool match = false; 1491 int index, ret, i; 1492 u8 target; 1493 1494 mutex_lock(&vhost_scsi_mutex); 1495 mutex_lock(&vs->dev.mutex); 1496 /* Verify that ring has been setup correctly. */ 1497 for (index = 0; index < vs->dev.nvqs; ++index) { 1498 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) { 1499 ret = -EFAULT; 1500 goto err_dev; 1501 } 1502 } 1503 1504 if (!vs->vs_tpg) { 1505 ret = 0; 1506 goto err_dev; 1507 } 1508 1509 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) { 1510 target = i; 1511 tpg = vs->vs_tpg[target]; 1512 if (!tpg) 1513 continue; 1514 1515 mutex_lock(&tpg->tv_tpg_mutex); 1516 tv_tport = tpg->tport; 1517 if (!tv_tport) { 1518 ret = -ENODEV; 1519 goto err_tpg; 1520 } 1521 1522 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) { 1523 pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu" 1524 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n", 1525 tv_tport->tport_name, tpg->tport_tpgt, 1526 t->vhost_wwpn, t->vhost_tpgt); 1527 ret = -EINVAL; 1528 goto err_tpg; 1529 } 1530 tpg->tv_tpg_vhost_count--; 1531 tpg->vhost_scsi = NULL; 1532 vs->vs_tpg[target] = NULL; 1533 match = true; 1534 mutex_unlock(&tpg->tv_tpg_mutex); 1535 /* 1536 * Release se_tpg->tpg_group.cg_item configfs dependency now 1537 * to allow vhost-scsi WWPN se_tpg->tpg_group shutdown to occur. 1538 */ 1539 se_tpg = &tpg->se_tpg; 1540 target_undepend_item(&se_tpg->tpg_group.cg_item); 1541 } 1542 if (match) { 1543 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { 1544 vq = &vs->vqs[i].vq; 1545 mutex_lock(&vq->mutex); 1546 vq->private_data = NULL; 1547 mutex_unlock(&vq->mutex); 1548 } 1549 } 1550 /* 1551 * Act as synchronize_rcu to make sure access to 1552 * old vs->vs_tpg is finished. 1553 */ 1554 vhost_scsi_flush(vs); 1555 kfree(vs->vs_tpg); 1556 vs->vs_tpg = NULL; 1557 WARN_ON(vs->vs_events_nr); 1558 mutex_unlock(&vs->dev.mutex); 1559 mutex_unlock(&vhost_scsi_mutex); 1560 return 0; 1561 1562 err_tpg: 1563 mutex_unlock(&tpg->tv_tpg_mutex); 1564 err_dev: 1565 mutex_unlock(&vs->dev.mutex); 1566 mutex_unlock(&vhost_scsi_mutex); 1567 return ret; 1568 } 1569 1570 static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features) 1571 { 1572 struct vhost_virtqueue *vq; 1573 int i; 1574 1575 if (features & ~VHOST_SCSI_FEATURES) 1576 return -EOPNOTSUPP; 1577 1578 mutex_lock(&vs->dev.mutex); 1579 if ((features & (1 << VHOST_F_LOG_ALL)) && 1580 !vhost_log_access_ok(&vs->dev)) { 1581 mutex_unlock(&vs->dev.mutex); 1582 return -EFAULT; 1583 } 1584 1585 for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { 1586 vq = &vs->vqs[i].vq; 1587 mutex_lock(&vq->mutex); 1588 vq->acked_features = features; 1589 mutex_unlock(&vq->mutex); 1590 } 1591 mutex_unlock(&vs->dev.mutex); 1592 return 0; 1593 } 1594 1595 static int vhost_scsi_open(struct inode *inode, struct file *f) 1596 { 1597 struct vhost_scsi *vs; 1598 struct vhost_virtqueue **vqs; 1599 int r = -ENOMEM, i; 1600 1601 vs = kzalloc(sizeof(*vs), GFP_KERNEL | __GFP_NOWARN | __GFP_RETRY_MAYFAIL); 1602 if (!vs) { 1603 vs = vzalloc(sizeof(*vs)); 1604 if (!vs) 1605 goto err_vs; 1606 } 1607 1608 vqs = kmalloc_array(VHOST_SCSI_MAX_VQ, sizeof(*vqs), GFP_KERNEL); 1609 if (!vqs) 1610 goto err_vqs; 1611 1612 vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work); 1613 vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work); 1614 1615 vs->vs_events_nr = 0; 1616 vs->vs_events_missed = false; 1617 1618 vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq; 1619 vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; 1620 vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick; 1621 vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick; 1622 for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) { 1623 vqs[i] = &vs->vqs[i].vq; 1624 vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; 1625 } 1626 vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ); 1627 1628 vhost_scsi_init_inflight(vs, NULL); 1629 1630 f->private_data = vs; 1631 return 0; 1632 1633 err_vqs: 1634 kvfree(vs); 1635 err_vs: 1636 return r; 1637 } 1638 1639 static int vhost_scsi_release(struct inode *inode, struct file *f) 1640 { 1641 struct vhost_scsi *vs = f->private_data; 1642 struct vhost_scsi_target t; 1643 1644 mutex_lock(&vs->dev.mutex); 1645 memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn)); 1646 mutex_unlock(&vs->dev.mutex); 1647 vhost_scsi_clear_endpoint(vs, &t); 1648 vhost_dev_stop(&vs->dev); 1649 vhost_dev_cleanup(&vs->dev); 1650 /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */ 1651 vhost_scsi_flush(vs); 1652 kfree(vs->dev.vqs); 1653 kvfree(vs); 1654 return 0; 1655 } 1656 1657 static long 1658 vhost_scsi_ioctl(struct file *f, 1659 unsigned int ioctl, 1660 unsigned long arg) 1661 { 1662 struct vhost_scsi *vs = f->private_data; 1663 struct vhost_scsi_target backend; 1664 void __user *argp = (void __user *)arg; 1665 u64 __user *featurep = argp; 1666 u32 __user *eventsp = argp; 1667 u32 events_missed; 1668 u64 features; 1669 int r, abi_version = VHOST_SCSI_ABI_VERSION; 1670 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; 1671 1672 switch (ioctl) { 1673 case VHOST_SCSI_SET_ENDPOINT: 1674 if (copy_from_user(&backend, argp, sizeof backend)) 1675 return -EFAULT; 1676 if (backend.reserved != 0) 1677 return -EOPNOTSUPP; 1678 1679 return vhost_scsi_set_endpoint(vs, &backend); 1680 case VHOST_SCSI_CLEAR_ENDPOINT: 1681 if (copy_from_user(&backend, argp, sizeof backend)) 1682 return -EFAULT; 1683 if (backend.reserved != 0) 1684 return -EOPNOTSUPP; 1685 1686 return vhost_scsi_clear_endpoint(vs, &backend); 1687 case VHOST_SCSI_GET_ABI_VERSION: 1688 if (copy_to_user(argp, &abi_version, sizeof abi_version)) 1689 return -EFAULT; 1690 return 0; 1691 case VHOST_SCSI_SET_EVENTS_MISSED: 1692 if (get_user(events_missed, eventsp)) 1693 return -EFAULT; 1694 mutex_lock(&vq->mutex); 1695 vs->vs_events_missed = events_missed; 1696 mutex_unlock(&vq->mutex); 1697 return 0; 1698 case VHOST_SCSI_GET_EVENTS_MISSED: 1699 mutex_lock(&vq->mutex); 1700 events_missed = vs->vs_events_missed; 1701 mutex_unlock(&vq->mutex); 1702 if (put_user(events_missed, eventsp)) 1703 return -EFAULT; 1704 return 0; 1705 case VHOST_GET_FEATURES: 1706 features = VHOST_SCSI_FEATURES; 1707 if (copy_to_user(featurep, &features, sizeof features)) 1708 return -EFAULT; 1709 return 0; 1710 case VHOST_SET_FEATURES: 1711 if (copy_from_user(&features, featurep, sizeof features)) 1712 return -EFAULT; 1713 return vhost_scsi_set_features(vs, features); 1714 default: 1715 mutex_lock(&vs->dev.mutex); 1716 r = vhost_dev_ioctl(&vs->dev, ioctl, argp); 1717 /* TODO: flush backend after dev ioctl. */ 1718 if (r == -ENOIOCTLCMD) 1719 r = vhost_vring_ioctl(&vs->dev, ioctl, argp); 1720 mutex_unlock(&vs->dev.mutex); 1721 return r; 1722 } 1723 } 1724 1725 #ifdef CONFIG_COMPAT 1726 static long vhost_scsi_compat_ioctl(struct file *f, unsigned int ioctl, 1727 unsigned long arg) 1728 { 1729 return vhost_scsi_ioctl(f, ioctl, (unsigned long)compat_ptr(arg)); 1730 } 1731 #endif 1732 1733 static const struct file_operations vhost_scsi_fops = { 1734 .owner = THIS_MODULE, 1735 .release = vhost_scsi_release, 1736 .unlocked_ioctl = vhost_scsi_ioctl, 1737 #ifdef CONFIG_COMPAT 1738 .compat_ioctl = vhost_scsi_compat_ioctl, 1739 #endif 1740 .open = vhost_scsi_open, 1741 .llseek = noop_llseek, 1742 }; 1743 1744 static struct miscdevice vhost_scsi_misc = { 1745 MISC_DYNAMIC_MINOR, 1746 "vhost-scsi", 1747 &vhost_scsi_fops, 1748 }; 1749 1750 static int __init vhost_scsi_register(void) 1751 { 1752 return misc_register(&vhost_scsi_misc); 1753 } 1754 1755 static void vhost_scsi_deregister(void) 1756 { 1757 misc_deregister(&vhost_scsi_misc); 1758 } 1759 1760 static char *vhost_scsi_dump_proto_id(struct vhost_scsi_tport *tport) 1761 { 1762 switch (tport->tport_proto_id) { 1763 case SCSI_PROTOCOL_SAS: 1764 return "SAS"; 1765 case SCSI_PROTOCOL_FCP: 1766 return "FCP"; 1767 case SCSI_PROTOCOL_ISCSI: 1768 return "iSCSI"; 1769 default: 1770 break; 1771 } 1772 1773 return "Unknown"; 1774 } 1775 1776 static void 1777 vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg, 1778 struct se_lun *lun, bool plug) 1779 { 1780 1781 struct vhost_scsi *vs = tpg->vhost_scsi; 1782 struct vhost_virtqueue *vq; 1783 u32 reason; 1784 1785 if (!vs) 1786 return; 1787 1788 mutex_lock(&vs->dev.mutex); 1789 1790 if (plug) 1791 reason = VIRTIO_SCSI_EVT_RESET_RESCAN; 1792 else 1793 reason = VIRTIO_SCSI_EVT_RESET_REMOVED; 1794 1795 vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; 1796 mutex_lock(&vq->mutex); 1797 if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG)) 1798 vhost_scsi_send_evt(vs, tpg, lun, 1799 VIRTIO_SCSI_T_TRANSPORT_RESET, reason); 1800 mutex_unlock(&vq->mutex); 1801 mutex_unlock(&vs->dev.mutex); 1802 } 1803 1804 static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun) 1805 { 1806 vhost_scsi_do_plug(tpg, lun, true); 1807 } 1808 1809 static void vhost_scsi_hotunplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun) 1810 { 1811 vhost_scsi_do_plug(tpg, lun, false); 1812 } 1813 1814 static int vhost_scsi_port_link(struct se_portal_group *se_tpg, 1815 struct se_lun *lun) 1816 { 1817 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 1818 struct vhost_scsi_tpg, se_tpg); 1819 1820 mutex_lock(&vhost_scsi_mutex); 1821 1822 mutex_lock(&tpg->tv_tpg_mutex); 1823 tpg->tv_tpg_port_count++; 1824 mutex_unlock(&tpg->tv_tpg_mutex); 1825 1826 vhost_scsi_hotplug(tpg, lun); 1827 1828 mutex_unlock(&vhost_scsi_mutex); 1829 1830 return 0; 1831 } 1832 1833 static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg, 1834 struct se_lun *lun) 1835 { 1836 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 1837 struct vhost_scsi_tpg, se_tpg); 1838 1839 mutex_lock(&vhost_scsi_mutex); 1840 1841 mutex_lock(&tpg->tv_tpg_mutex); 1842 tpg->tv_tpg_port_count--; 1843 mutex_unlock(&tpg->tv_tpg_mutex); 1844 1845 vhost_scsi_hotunplug(tpg, lun); 1846 1847 mutex_unlock(&vhost_scsi_mutex); 1848 } 1849 1850 static void vhost_scsi_free_cmd_map_res(struct se_session *se_sess) 1851 { 1852 struct vhost_scsi_cmd *tv_cmd; 1853 unsigned int i; 1854 1855 if (!se_sess->sess_cmd_map) 1856 return; 1857 1858 for (i = 0; i < VHOST_SCSI_DEFAULT_TAGS; i++) { 1859 tv_cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[i]; 1860 1861 kfree(tv_cmd->tvc_sgl); 1862 kfree(tv_cmd->tvc_prot_sgl); 1863 kfree(tv_cmd->tvc_upages); 1864 } 1865 } 1866 1867 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store( 1868 struct config_item *item, const char *page, size_t count) 1869 { 1870 struct se_portal_group *se_tpg = attrib_to_tpg(item); 1871 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 1872 struct vhost_scsi_tpg, se_tpg); 1873 unsigned long val; 1874 int ret = kstrtoul(page, 0, &val); 1875 1876 if (ret) { 1877 pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret); 1878 return ret; 1879 } 1880 if (val != 0 && val != 1 && val != 3) { 1881 pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val); 1882 return -EINVAL; 1883 } 1884 tpg->tv_fabric_prot_type = val; 1885 1886 return count; 1887 } 1888 1889 static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show( 1890 struct config_item *item, char *page) 1891 { 1892 struct se_portal_group *se_tpg = attrib_to_tpg(item); 1893 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 1894 struct vhost_scsi_tpg, se_tpg); 1895 1896 return sprintf(page, "%d\n", tpg->tv_fabric_prot_type); 1897 } 1898 1899 CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type); 1900 1901 static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = { 1902 &vhost_scsi_tpg_attrib_attr_fabric_prot_type, 1903 NULL, 1904 }; 1905 1906 static int vhost_scsi_nexus_cb(struct se_portal_group *se_tpg, 1907 struct se_session *se_sess, void *p) 1908 { 1909 struct vhost_scsi_cmd *tv_cmd; 1910 unsigned int i; 1911 1912 for (i = 0; i < VHOST_SCSI_DEFAULT_TAGS; i++) { 1913 tv_cmd = &((struct vhost_scsi_cmd *)se_sess->sess_cmd_map)[i]; 1914 1915 tv_cmd->tvc_sgl = kcalloc(VHOST_SCSI_PREALLOC_SGLS, 1916 sizeof(struct scatterlist), 1917 GFP_KERNEL); 1918 if (!tv_cmd->tvc_sgl) { 1919 pr_err("Unable to allocate tv_cmd->tvc_sgl\n"); 1920 goto out; 1921 } 1922 1923 tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES, 1924 sizeof(struct page *), 1925 GFP_KERNEL); 1926 if (!tv_cmd->tvc_upages) { 1927 pr_err("Unable to allocate tv_cmd->tvc_upages\n"); 1928 goto out; 1929 } 1930 1931 tv_cmd->tvc_prot_sgl = kcalloc(VHOST_SCSI_PREALLOC_PROT_SGLS, 1932 sizeof(struct scatterlist), 1933 GFP_KERNEL); 1934 if (!tv_cmd->tvc_prot_sgl) { 1935 pr_err("Unable to allocate tv_cmd->tvc_prot_sgl\n"); 1936 goto out; 1937 } 1938 } 1939 return 0; 1940 out: 1941 vhost_scsi_free_cmd_map_res(se_sess); 1942 return -ENOMEM; 1943 } 1944 1945 static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg, 1946 const char *name) 1947 { 1948 struct vhost_scsi_nexus *tv_nexus; 1949 1950 mutex_lock(&tpg->tv_tpg_mutex); 1951 if (tpg->tpg_nexus) { 1952 mutex_unlock(&tpg->tv_tpg_mutex); 1953 pr_debug("tpg->tpg_nexus already exists\n"); 1954 return -EEXIST; 1955 } 1956 1957 tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL); 1958 if (!tv_nexus) { 1959 mutex_unlock(&tpg->tv_tpg_mutex); 1960 pr_err("Unable to allocate struct vhost_scsi_nexus\n"); 1961 return -ENOMEM; 1962 } 1963 /* 1964 * Since we are running in 'demo mode' this call with generate a 1965 * struct se_node_acl for the vhost_scsi struct se_portal_group with 1966 * the SCSI Initiator port name of the passed configfs group 'name'. 1967 */ 1968 tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg, 1969 VHOST_SCSI_DEFAULT_TAGS, 1970 sizeof(struct vhost_scsi_cmd), 1971 TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS, 1972 (unsigned char *)name, tv_nexus, 1973 vhost_scsi_nexus_cb); 1974 if (IS_ERR(tv_nexus->tvn_se_sess)) { 1975 mutex_unlock(&tpg->tv_tpg_mutex); 1976 kfree(tv_nexus); 1977 return -ENOMEM; 1978 } 1979 tpg->tpg_nexus = tv_nexus; 1980 1981 mutex_unlock(&tpg->tv_tpg_mutex); 1982 return 0; 1983 } 1984 1985 static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg) 1986 { 1987 struct se_session *se_sess; 1988 struct vhost_scsi_nexus *tv_nexus; 1989 1990 mutex_lock(&tpg->tv_tpg_mutex); 1991 tv_nexus = tpg->tpg_nexus; 1992 if (!tv_nexus) { 1993 mutex_unlock(&tpg->tv_tpg_mutex); 1994 return -ENODEV; 1995 } 1996 1997 se_sess = tv_nexus->tvn_se_sess; 1998 if (!se_sess) { 1999 mutex_unlock(&tpg->tv_tpg_mutex); 2000 return -ENODEV; 2001 } 2002 2003 if (tpg->tv_tpg_port_count != 0) { 2004 mutex_unlock(&tpg->tv_tpg_mutex); 2005 pr_err("Unable to remove TCM_vhost I_T Nexus with" 2006 " active TPG port count: %d\n", 2007 tpg->tv_tpg_port_count); 2008 return -EBUSY; 2009 } 2010 2011 if (tpg->tv_tpg_vhost_count != 0) { 2012 mutex_unlock(&tpg->tv_tpg_mutex); 2013 pr_err("Unable to remove TCM_vhost I_T Nexus with" 2014 " active TPG vhost count: %d\n", 2015 tpg->tv_tpg_vhost_count); 2016 return -EBUSY; 2017 } 2018 2019 pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated" 2020 " %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport), 2021 tv_nexus->tvn_se_sess->se_node_acl->initiatorname); 2022 2023 vhost_scsi_free_cmd_map_res(se_sess); 2024 /* 2025 * Release the SCSI I_T Nexus to the emulated vhost Target Port 2026 */ 2027 target_remove_session(se_sess); 2028 tpg->tpg_nexus = NULL; 2029 mutex_unlock(&tpg->tv_tpg_mutex); 2030 2031 kfree(tv_nexus); 2032 return 0; 2033 } 2034 2035 static ssize_t vhost_scsi_tpg_nexus_show(struct config_item *item, char *page) 2036 { 2037 struct se_portal_group *se_tpg = to_tpg(item); 2038 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 2039 struct vhost_scsi_tpg, se_tpg); 2040 struct vhost_scsi_nexus *tv_nexus; 2041 ssize_t ret; 2042 2043 mutex_lock(&tpg->tv_tpg_mutex); 2044 tv_nexus = tpg->tpg_nexus; 2045 if (!tv_nexus) { 2046 mutex_unlock(&tpg->tv_tpg_mutex); 2047 return -ENODEV; 2048 } 2049 ret = snprintf(page, PAGE_SIZE, "%s\n", 2050 tv_nexus->tvn_se_sess->se_node_acl->initiatorname); 2051 mutex_unlock(&tpg->tv_tpg_mutex); 2052 2053 return ret; 2054 } 2055 2056 static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item, 2057 const char *page, size_t count) 2058 { 2059 struct se_portal_group *se_tpg = to_tpg(item); 2060 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 2061 struct vhost_scsi_tpg, se_tpg); 2062 struct vhost_scsi_tport *tport_wwn = tpg->tport; 2063 unsigned char i_port[VHOST_SCSI_NAMELEN], *ptr, *port_ptr; 2064 int ret; 2065 /* 2066 * Shutdown the active I_T nexus if 'NULL' is passed.. 2067 */ 2068 if (!strncmp(page, "NULL", 4)) { 2069 ret = vhost_scsi_drop_nexus(tpg); 2070 return (!ret) ? count : ret; 2071 } 2072 /* 2073 * Otherwise make sure the passed virtual Initiator port WWN matches 2074 * the fabric protocol_id set in vhost_scsi_make_tport(), and call 2075 * vhost_scsi_make_nexus(). 2076 */ 2077 if (strlen(page) >= VHOST_SCSI_NAMELEN) { 2078 pr_err("Emulated NAA Sas Address: %s, exceeds" 2079 " max: %d\n", page, VHOST_SCSI_NAMELEN); 2080 return -EINVAL; 2081 } 2082 snprintf(&i_port[0], VHOST_SCSI_NAMELEN, "%s", page); 2083 2084 ptr = strstr(i_port, "naa."); 2085 if (ptr) { 2086 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) { 2087 pr_err("Passed SAS Initiator Port %s does not" 2088 " match target port protoid: %s\n", i_port, 2089 vhost_scsi_dump_proto_id(tport_wwn)); 2090 return -EINVAL; 2091 } 2092 port_ptr = &i_port[0]; 2093 goto check_newline; 2094 } 2095 ptr = strstr(i_port, "fc."); 2096 if (ptr) { 2097 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) { 2098 pr_err("Passed FCP Initiator Port %s does not" 2099 " match target port protoid: %s\n", i_port, 2100 vhost_scsi_dump_proto_id(tport_wwn)); 2101 return -EINVAL; 2102 } 2103 port_ptr = &i_port[3]; /* Skip over "fc." */ 2104 goto check_newline; 2105 } 2106 ptr = strstr(i_port, "iqn."); 2107 if (ptr) { 2108 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) { 2109 pr_err("Passed iSCSI Initiator Port %s does not" 2110 " match target port protoid: %s\n", i_port, 2111 vhost_scsi_dump_proto_id(tport_wwn)); 2112 return -EINVAL; 2113 } 2114 port_ptr = &i_port[0]; 2115 goto check_newline; 2116 } 2117 pr_err("Unable to locate prefix for emulated Initiator Port:" 2118 " %s\n", i_port); 2119 return -EINVAL; 2120 /* 2121 * Clear any trailing newline for the NAA WWN 2122 */ 2123 check_newline: 2124 if (i_port[strlen(i_port)-1] == '\n') 2125 i_port[strlen(i_port)-1] = '\0'; 2126 2127 ret = vhost_scsi_make_nexus(tpg, port_ptr); 2128 if (ret < 0) 2129 return ret; 2130 2131 return count; 2132 } 2133 2134 CONFIGFS_ATTR(vhost_scsi_tpg_, nexus); 2135 2136 static struct configfs_attribute *vhost_scsi_tpg_attrs[] = { 2137 &vhost_scsi_tpg_attr_nexus, 2138 NULL, 2139 }; 2140 2141 static struct se_portal_group * 2142 vhost_scsi_make_tpg(struct se_wwn *wwn, const char *name) 2143 { 2144 struct vhost_scsi_tport *tport = container_of(wwn, 2145 struct vhost_scsi_tport, tport_wwn); 2146 2147 struct vhost_scsi_tpg *tpg; 2148 u16 tpgt; 2149 int ret; 2150 2151 if (strstr(name, "tpgt_") != name) 2152 return ERR_PTR(-EINVAL); 2153 if (kstrtou16(name + 5, 10, &tpgt) || tpgt >= VHOST_SCSI_MAX_TARGET) 2154 return ERR_PTR(-EINVAL); 2155 2156 tpg = kzalloc(sizeof(*tpg), GFP_KERNEL); 2157 if (!tpg) { 2158 pr_err("Unable to allocate struct vhost_scsi_tpg"); 2159 return ERR_PTR(-ENOMEM); 2160 } 2161 mutex_init(&tpg->tv_tpg_mutex); 2162 INIT_LIST_HEAD(&tpg->tv_tpg_list); 2163 tpg->tport = tport; 2164 tpg->tport_tpgt = tpgt; 2165 2166 ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id); 2167 if (ret < 0) { 2168 kfree(tpg); 2169 return NULL; 2170 } 2171 mutex_lock(&vhost_scsi_mutex); 2172 list_add_tail(&tpg->tv_tpg_list, &vhost_scsi_list); 2173 mutex_unlock(&vhost_scsi_mutex); 2174 2175 return &tpg->se_tpg; 2176 } 2177 2178 static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg) 2179 { 2180 struct vhost_scsi_tpg *tpg = container_of(se_tpg, 2181 struct vhost_scsi_tpg, se_tpg); 2182 2183 mutex_lock(&vhost_scsi_mutex); 2184 list_del(&tpg->tv_tpg_list); 2185 mutex_unlock(&vhost_scsi_mutex); 2186 /* 2187 * Release the virtual I_T Nexus for this vhost TPG 2188 */ 2189 vhost_scsi_drop_nexus(tpg); 2190 /* 2191 * Deregister the se_tpg from TCM.. 2192 */ 2193 core_tpg_deregister(se_tpg); 2194 kfree(tpg); 2195 } 2196 2197 static struct se_wwn * 2198 vhost_scsi_make_tport(struct target_fabric_configfs *tf, 2199 struct config_group *group, 2200 const char *name) 2201 { 2202 struct vhost_scsi_tport *tport; 2203 char *ptr; 2204 u64 wwpn = 0; 2205 int off = 0; 2206 2207 /* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0) 2208 return ERR_PTR(-EINVAL); */ 2209 2210 tport = kzalloc(sizeof(*tport), GFP_KERNEL); 2211 if (!tport) { 2212 pr_err("Unable to allocate struct vhost_scsi_tport"); 2213 return ERR_PTR(-ENOMEM); 2214 } 2215 tport->tport_wwpn = wwpn; 2216 /* 2217 * Determine the emulated Protocol Identifier and Target Port Name 2218 * based on the incoming configfs directory name. 2219 */ 2220 ptr = strstr(name, "naa."); 2221 if (ptr) { 2222 tport->tport_proto_id = SCSI_PROTOCOL_SAS; 2223 goto check_len; 2224 } 2225 ptr = strstr(name, "fc."); 2226 if (ptr) { 2227 tport->tport_proto_id = SCSI_PROTOCOL_FCP; 2228 off = 3; /* Skip over "fc." */ 2229 goto check_len; 2230 } 2231 ptr = strstr(name, "iqn."); 2232 if (ptr) { 2233 tport->tport_proto_id = SCSI_PROTOCOL_ISCSI; 2234 goto check_len; 2235 } 2236 2237 pr_err("Unable to locate prefix for emulated Target Port:" 2238 " %s\n", name); 2239 kfree(tport); 2240 return ERR_PTR(-EINVAL); 2241 2242 check_len: 2243 if (strlen(name) >= VHOST_SCSI_NAMELEN) { 2244 pr_err("Emulated %s Address: %s, exceeds" 2245 " max: %d\n", name, vhost_scsi_dump_proto_id(tport), 2246 VHOST_SCSI_NAMELEN); 2247 kfree(tport); 2248 return ERR_PTR(-EINVAL); 2249 } 2250 snprintf(&tport->tport_name[0], VHOST_SCSI_NAMELEN, "%s", &name[off]); 2251 2252 pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target" 2253 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name); 2254 2255 return &tport->tport_wwn; 2256 } 2257 2258 static void vhost_scsi_drop_tport(struct se_wwn *wwn) 2259 { 2260 struct vhost_scsi_tport *tport = container_of(wwn, 2261 struct vhost_scsi_tport, tport_wwn); 2262 2263 pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target" 2264 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport), 2265 tport->tport_name); 2266 2267 kfree(tport); 2268 } 2269 2270 static ssize_t 2271 vhost_scsi_wwn_version_show(struct config_item *item, char *page) 2272 { 2273 return sprintf(page, "TCM_VHOST fabric module %s on %s/%s" 2274 "on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname, 2275 utsname()->machine); 2276 } 2277 2278 CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version); 2279 2280 static struct configfs_attribute *vhost_scsi_wwn_attrs[] = { 2281 &vhost_scsi_wwn_attr_version, 2282 NULL, 2283 }; 2284 2285 static const struct target_core_fabric_ops vhost_scsi_ops = { 2286 .module = THIS_MODULE, 2287 .fabric_name = "vhost", 2288 .tpg_get_wwn = vhost_scsi_get_fabric_wwn, 2289 .tpg_get_tag = vhost_scsi_get_tpgt, 2290 .tpg_check_demo_mode = vhost_scsi_check_true, 2291 .tpg_check_demo_mode_cache = vhost_scsi_check_true, 2292 .tpg_check_demo_mode_write_protect = vhost_scsi_check_false, 2293 .tpg_check_prod_mode_write_protect = vhost_scsi_check_false, 2294 .tpg_check_prot_fabric_only = vhost_scsi_check_prot_fabric_only, 2295 .tpg_get_inst_index = vhost_scsi_tpg_get_inst_index, 2296 .release_cmd = vhost_scsi_release_cmd, 2297 .check_stop_free = vhost_scsi_check_stop_free, 2298 .sess_get_index = vhost_scsi_sess_get_index, 2299 .sess_get_initiator_sid = NULL, 2300 .write_pending = vhost_scsi_write_pending, 2301 .write_pending_status = vhost_scsi_write_pending_status, 2302 .set_default_node_attributes = vhost_scsi_set_default_node_attrs, 2303 .get_cmd_state = vhost_scsi_get_cmd_state, 2304 .queue_data_in = vhost_scsi_queue_data_in, 2305 .queue_status = vhost_scsi_queue_status, 2306 .queue_tm_rsp = vhost_scsi_queue_tm_rsp, 2307 .aborted_task = vhost_scsi_aborted_task, 2308 /* 2309 * Setup callers for generic logic in target_core_fabric_configfs.c 2310 */ 2311 .fabric_make_wwn = vhost_scsi_make_tport, 2312 .fabric_drop_wwn = vhost_scsi_drop_tport, 2313 .fabric_make_tpg = vhost_scsi_make_tpg, 2314 .fabric_drop_tpg = vhost_scsi_drop_tpg, 2315 .fabric_post_link = vhost_scsi_port_link, 2316 .fabric_pre_unlink = vhost_scsi_port_unlink, 2317 2318 .tfc_wwn_attrs = vhost_scsi_wwn_attrs, 2319 .tfc_tpg_base_attrs = vhost_scsi_tpg_attrs, 2320 .tfc_tpg_attrib_attrs = vhost_scsi_tpg_attrib_attrs, 2321 }; 2322 2323 static int __init vhost_scsi_init(void) 2324 { 2325 int ret = -ENOMEM; 2326 2327 pr_debug("TCM_VHOST fabric module %s on %s/%s" 2328 " on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname, 2329 utsname()->machine); 2330 2331 /* 2332 * Use our own dedicated workqueue for submitting I/O into 2333 * target core to avoid contention within system_wq. 2334 */ 2335 vhost_scsi_workqueue = alloc_workqueue("vhost_scsi", 0, 0); 2336 if (!vhost_scsi_workqueue) 2337 goto out; 2338 2339 ret = vhost_scsi_register(); 2340 if (ret < 0) 2341 goto out_destroy_workqueue; 2342 2343 ret = target_register_template(&vhost_scsi_ops); 2344 if (ret < 0) 2345 goto out_vhost_scsi_deregister; 2346 2347 return 0; 2348 2349 out_vhost_scsi_deregister: 2350 vhost_scsi_deregister(); 2351 out_destroy_workqueue: 2352 destroy_workqueue(vhost_scsi_workqueue); 2353 out: 2354 return ret; 2355 }; 2356 2357 static void vhost_scsi_exit(void) 2358 { 2359 target_unregister_template(&vhost_scsi_ops); 2360 vhost_scsi_deregister(); 2361 destroy_workqueue(vhost_scsi_workqueue); 2362 }; 2363 2364 MODULE_DESCRIPTION("VHOST_SCSI series fabric driver"); 2365 MODULE_ALIAS("tcm_vhost"); 2366 MODULE_LICENSE("GPL"); 2367 module_init(vhost_scsi_init); 2368 module_exit(vhost_scsi_exit); 2369