1 /* 2 * Copyright (c) 2016 Avago Technologies. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful. 9 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, 10 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A 11 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO 12 * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID. 13 * See the GNU General Public License for more details, a copy of which 14 * can be found in the file COPYING included with this package 15 * 16 */ 17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 18 #include <linux/module.h> 19 #include <linux/slab.h> 20 #include <linux/blk-mq.h> 21 #include <linux/parser.h> 22 #include <linux/random.h> 23 #include <uapi/scsi/fc/fc_fs.h> 24 #include <uapi/scsi/fc/fc_els.h> 25 26 #include "nvmet.h" 27 #include <linux/nvme-fc-driver.h> 28 #include <linux/nvme-fc.h> 29 30 31 /* *************************** Data Structures/Defines ****************** */ 32 33 34 #define NVMET_LS_CTX_COUNT 4 35 36 /* for this implementation, assume small single frame rqst/rsp */ 37 #define NVME_FC_MAX_LS_BUFFER_SIZE 2048 38 39 struct nvmet_fc_tgtport; 40 struct nvmet_fc_tgt_assoc; 41 42 struct nvmet_fc_ls_iod { 43 struct nvmefc_tgt_ls_req *lsreq; 44 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */ 45 46 struct list_head ls_list; /* tgtport->ls_list */ 47 48 struct nvmet_fc_tgtport *tgtport; 49 struct nvmet_fc_tgt_assoc *assoc; 50 51 u8 *rqstbuf; 52 u8 *rspbuf; 53 u16 rqstdatalen; 54 dma_addr_t rspdma; 55 56 struct scatterlist sg[2]; 57 58 struct work_struct work; 59 } __aligned(sizeof(unsigned long long)); 60 61 #define NVMET_FC_MAX_KB_PER_XFR 256 62 63 enum nvmet_fcp_datadir { 64 NVMET_FCP_NODATA, 65 NVMET_FCP_WRITE, 66 NVMET_FCP_READ, 67 NVMET_FCP_ABORTED, 68 }; 69 70 struct nvmet_fc_fcp_iod { 71 struct nvmefc_tgt_fcp_req *fcpreq; 72 73 struct nvme_fc_cmd_iu cmdiubuf; 74 struct nvme_fc_ersp_iu rspiubuf; 75 dma_addr_t rspdma; 76 struct scatterlist *data_sg; 77 struct scatterlist *next_sg; 78 int data_sg_cnt; 79 u32 next_sg_offset; 80 u32 total_length; 81 u32 offset; 82 enum nvmet_fcp_datadir io_dir; 83 bool active; 84 bool abort; 85 bool aborted; 86 bool writedataactive; 87 spinlock_t flock; 88 89 struct nvmet_req req; 90 struct work_struct work; 91 struct work_struct done_work; 92 93 struct nvmet_fc_tgtport *tgtport; 94 struct nvmet_fc_tgt_queue *queue; 95 96 struct list_head fcp_list; /* tgtport->fcp_list */ 97 }; 98 99 struct nvmet_fc_tgtport { 100 101 struct nvmet_fc_target_port fc_target_port; 102 103 struct list_head tgt_list; /* nvmet_fc_target_list */ 104 struct device *dev; /* dev for dma mapping */ 105 struct nvmet_fc_target_template *ops; 106 107 struct nvmet_fc_ls_iod *iod; 108 spinlock_t lock; 109 struct list_head ls_list; 110 struct list_head ls_busylist; 111 struct list_head assoc_list; 112 struct ida assoc_cnt; 113 struct nvmet_port *port; 114 struct kref ref; 115 }; 116 117 struct nvmet_fc_defer_fcp_req { 118 struct list_head req_list; 119 struct nvmefc_tgt_fcp_req *fcp_req; 120 }; 121 122 struct nvmet_fc_tgt_queue { 123 bool ninetypercent; 124 u16 qid; 125 u16 sqsize; 126 u16 ersp_ratio; 127 __le16 sqhd; 128 int cpu; 129 atomic_t connected; 130 atomic_t sqtail; 131 atomic_t zrspcnt; 132 atomic_t rsn; 133 spinlock_t qlock; 134 struct nvmet_port *port; 135 struct nvmet_cq nvme_cq; 136 struct nvmet_sq nvme_sq; 137 struct nvmet_fc_tgt_assoc *assoc; 138 struct nvmet_fc_fcp_iod *fod; /* array of fcp_iods */ 139 struct list_head fod_list; 140 struct list_head pending_cmd_list; 141 struct list_head avail_defer_list; 142 struct workqueue_struct *work_q; 143 struct kref ref; 144 } __aligned(sizeof(unsigned long long)); 145 146 struct nvmet_fc_tgt_assoc { 147 u64 association_id; 148 u32 a_id; 149 struct nvmet_fc_tgtport *tgtport; 150 struct list_head a_list; 151 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES]; 152 struct kref ref; 153 }; 154 155 156 static inline int 157 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr) 158 { 159 return (iodptr - iodptr->tgtport->iod); 160 } 161 162 static inline int 163 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr) 164 { 165 return (fodptr - fodptr->queue->fod); 166 } 167 168 169 /* 170 * Association and Connection IDs: 171 * 172 * Association ID will have random number in upper 6 bytes and zero 173 * in lower 2 bytes 174 * 175 * Connection IDs will be Association ID with QID or'd in lower 2 bytes 176 * 177 * note: Association ID = Connection ID for queue 0 178 */ 179 #define BYTES_FOR_QID sizeof(u16) 180 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8) 181 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1)) 182 183 static inline u64 184 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid) 185 { 186 return (assoc->association_id | qid); 187 } 188 189 static inline u64 190 nvmet_fc_getassociationid(u64 connectionid) 191 { 192 return connectionid & ~NVMET_FC_QUEUEID_MASK; 193 } 194 195 static inline u16 196 nvmet_fc_getqueueid(u64 connectionid) 197 { 198 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK); 199 } 200 201 static inline struct nvmet_fc_tgtport * 202 targetport_to_tgtport(struct nvmet_fc_target_port *targetport) 203 { 204 return container_of(targetport, struct nvmet_fc_tgtport, 205 fc_target_port); 206 } 207 208 static inline struct nvmet_fc_fcp_iod * 209 nvmet_req_to_fod(struct nvmet_req *nvme_req) 210 { 211 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req); 212 } 213 214 215 /* *************************** Globals **************************** */ 216 217 218 static DEFINE_SPINLOCK(nvmet_fc_tgtlock); 219 220 static LIST_HEAD(nvmet_fc_target_list); 221 static DEFINE_IDA(nvmet_fc_tgtport_cnt); 222 223 224 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work); 225 static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work); 226 static void nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work); 227 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc); 228 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc); 229 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue); 230 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue); 231 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport); 232 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport); 233 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, 234 struct nvmet_fc_fcp_iod *fod); 235 236 237 /* *********************** FC-NVME DMA Handling **************************** */ 238 239 /* 240 * The fcloop device passes in a NULL device pointer. Real LLD's will 241 * pass in a valid device pointer. If NULL is passed to the dma mapping 242 * routines, depending on the platform, it may or may not succeed, and 243 * may crash. 244 * 245 * As such: 246 * Wrapper all the dma routines and check the dev pointer. 247 * 248 * If simple mappings (return just a dma address, we'll noop them, 249 * returning a dma address of 0. 250 * 251 * On more complex mappings (dma_map_sg), a pseudo routine fills 252 * in the scatter list, setting all dma addresses to 0. 253 */ 254 255 static inline dma_addr_t 256 fc_dma_map_single(struct device *dev, void *ptr, size_t size, 257 enum dma_data_direction dir) 258 { 259 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L; 260 } 261 262 static inline int 263 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 264 { 265 return dev ? dma_mapping_error(dev, dma_addr) : 0; 266 } 267 268 static inline void 269 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, 270 enum dma_data_direction dir) 271 { 272 if (dev) 273 dma_unmap_single(dev, addr, size, dir); 274 } 275 276 static inline void 277 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 278 enum dma_data_direction dir) 279 { 280 if (dev) 281 dma_sync_single_for_cpu(dev, addr, size, dir); 282 } 283 284 static inline void 285 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, 286 enum dma_data_direction dir) 287 { 288 if (dev) 289 dma_sync_single_for_device(dev, addr, size, dir); 290 } 291 292 /* pseudo dma_map_sg call */ 293 static int 294 fc_map_sg(struct scatterlist *sg, int nents) 295 { 296 struct scatterlist *s; 297 int i; 298 299 WARN_ON(nents == 0 || sg[0].length == 0); 300 301 for_each_sg(sg, s, nents, i) { 302 s->dma_address = 0L; 303 #ifdef CONFIG_NEED_SG_DMA_LENGTH 304 s->dma_length = s->length; 305 #endif 306 } 307 return nents; 308 } 309 310 static inline int 311 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 312 enum dma_data_direction dir) 313 { 314 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents); 315 } 316 317 static inline void 318 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 319 enum dma_data_direction dir) 320 { 321 if (dev) 322 dma_unmap_sg(dev, sg, nents, dir); 323 } 324 325 326 /* *********************** FC-NVME Port Management ************************ */ 327 328 329 static int 330 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) 331 { 332 struct nvmet_fc_ls_iod *iod; 333 int i; 334 335 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod), 336 GFP_KERNEL); 337 if (!iod) 338 return -ENOMEM; 339 340 tgtport->iod = iod; 341 342 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { 343 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work); 344 iod->tgtport = tgtport; 345 list_add_tail(&iod->ls_list, &tgtport->ls_list); 346 347 iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE, 348 GFP_KERNEL); 349 if (!iod->rqstbuf) 350 goto out_fail; 351 352 iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE; 353 354 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf, 355 NVME_FC_MAX_LS_BUFFER_SIZE, 356 DMA_TO_DEVICE); 357 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma)) 358 goto out_fail; 359 } 360 361 return 0; 362 363 out_fail: 364 kfree(iod->rqstbuf); 365 list_del(&iod->ls_list); 366 for (iod--, i--; i >= 0; iod--, i--) { 367 fc_dma_unmap_single(tgtport->dev, iod->rspdma, 368 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); 369 kfree(iod->rqstbuf); 370 list_del(&iod->ls_list); 371 } 372 373 kfree(iod); 374 375 return -EFAULT; 376 } 377 378 static void 379 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport) 380 { 381 struct nvmet_fc_ls_iod *iod = tgtport->iod; 382 int i; 383 384 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { 385 fc_dma_unmap_single(tgtport->dev, 386 iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE, 387 DMA_TO_DEVICE); 388 kfree(iod->rqstbuf); 389 list_del(&iod->ls_list); 390 } 391 kfree(tgtport->iod); 392 } 393 394 static struct nvmet_fc_ls_iod * 395 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport) 396 { 397 struct nvmet_fc_ls_iod *iod; 398 unsigned long flags; 399 400 spin_lock_irqsave(&tgtport->lock, flags); 401 iod = list_first_entry_or_null(&tgtport->ls_list, 402 struct nvmet_fc_ls_iod, ls_list); 403 if (iod) 404 list_move_tail(&iod->ls_list, &tgtport->ls_busylist); 405 spin_unlock_irqrestore(&tgtport->lock, flags); 406 return iod; 407 } 408 409 410 static void 411 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport, 412 struct nvmet_fc_ls_iod *iod) 413 { 414 unsigned long flags; 415 416 spin_lock_irqsave(&tgtport->lock, flags); 417 list_move(&iod->ls_list, &tgtport->ls_list); 418 spin_unlock_irqrestore(&tgtport->lock, flags); 419 } 420 421 static void 422 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport, 423 struct nvmet_fc_tgt_queue *queue) 424 { 425 struct nvmet_fc_fcp_iod *fod = queue->fod; 426 int i; 427 428 for (i = 0; i < queue->sqsize; fod++, i++) { 429 INIT_WORK(&fod->work, nvmet_fc_handle_fcp_rqst_work); 430 INIT_WORK(&fod->done_work, nvmet_fc_fcp_rqst_op_done_work); 431 fod->tgtport = tgtport; 432 fod->queue = queue; 433 fod->active = false; 434 fod->abort = false; 435 fod->aborted = false; 436 fod->fcpreq = NULL; 437 list_add_tail(&fod->fcp_list, &queue->fod_list); 438 spin_lock_init(&fod->flock); 439 440 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf, 441 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 442 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) { 443 list_del(&fod->fcp_list); 444 for (fod--, i--; i >= 0; fod--, i--) { 445 fc_dma_unmap_single(tgtport->dev, fod->rspdma, 446 sizeof(fod->rspiubuf), 447 DMA_TO_DEVICE); 448 fod->rspdma = 0L; 449 list_del(&fod->fcp_list); 450 } 451 452 return; 453 } 454 } 455 } 456 457 static void 458 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport, 459 struct nvmet_fc_tgt_queue *queue) 460 { 461 struct nvmet_fc_fcp_iod *fod = queue->fod; 462 int i; 463 464 for (i = 0; i < queue->sqsize; fod++, i++) { 465 if (fod->rspdma) 466 fc_dma_unmap_single(tgtport->dev, fod->rspdma, 467 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 468 } 469 } 470 471 static struct nvmet_fc_fcp_iod * 472 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue) 473 { 474 struct nvmet_fc_fcp_iod *fod; 475 476 lockdep_assert_held(&queue->qlock); 477 478 fod = list_first_entry_or_null(&queue->fod_list, 479 struct nvmet_fc_fcp_iod, fcp_list); 480 if (fod) { 481 list_del(&fod->fcp_list); 482 fod->active = true; 483 /* 484 * no queue reference is taken, as it was taken by the 485 * queue lookup just prior to the allocation. The iod 486 * will "inherit" that reference. 487 */ 488 } 489 return fod; 490 } 491 492 493 static void 494 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport, 495 struct nvmet_fc_tgt_queue *queue, 496 struct nvmefc_tgt_fcp_req *fcpreq) 497 { 498 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 499 500 /* 501 * put all admin cmds on hw queue id 0. All io commands go to 502 * the respective hw queue based on a modulo basis 503 */ 504 fcpreq->hwqid = queue->qid ? 505 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0; 506 507 if (tgtport->ops->target_features & NVMET_FCTGTFEAT_CMD_IN_ISR) 508 queue_work_on(queue->cpu, queue->work_q, &fod->work); 509 else 510 nvmet_fc_handle_fcp_rqst(tgtport, fod); 511 } 512 513 static void 514 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue, 515 struct nvmet_fc_fcp_iod *fod) 516 { 517 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 518 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 519 struct nvmet_fc_defer_fcp_req *deferfcp; 520 unsigned long flags; 521 522 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma, 523 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 524 525 fcpreq->nvmet_fc_private = NULL; 526 527 fod->active = false; 528 fod->abort = false; 529 fod->aborted = false; 530 fod->writedataactive = false; 531 fod->fcpreq = NULL; 532 533 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq); 534 535 spin_lock_irqsave(&queue->qlock, flags); 536 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list, 537 struct nvmet_fc_defer_fcp_req, req_list); 538 if (!deferfcp) { 539 list_add_tail(&fod->fcp_list, &fod->queue->fod_list); 540 spin_unlock_irqrestore(&queue->qlock, flags); 541 542 /* Release reference taken at queue lookup and fod allocation */ 543 nvmet_fc_tgt_q_put(queue); 544 return; 545 } 546 547 /* Re-use the fod for the next pending cmd that was deferred */ 548 list_del(&deferfcp->req_list); 549 550 fcpreq = deferfcp->fcp_req; 551 552 /* deferfcp can be reused for another IO at a later date */ 553 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list); 554 555 spin_unlock_irqrestore(&queue->qlock, flags); 556 557 /* Save NVME CMD IO in fod */ 558 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen); 559 560 /* Setup new fcpreq to be processed */ 561 fcpreq->rspaddr = NULL; 562 fcpreq->rsplen = 0; 563 fcpreq->nvmet_fc_private = fod; 564 fod->fcpreq = fcpreq; 565 fod->active = true; 566 567 /* inform LLDD IO is now being processed */ 568 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq); 569 570 /* Submit deferred IO for processing */ 571 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq); 572 573 /* 574 * Leave the queue lookup get reference taken when 575 * fod was originally allocated. 576 */ 577 } 578 579 static int 580 nvmet_fc_queue_to_cpu(struct nvmet_fc_tgtport *tgtport, int qid) 581 { 582 int cpu, idx, cnt; 583 584 if (tgtport->ops->max_hw_queues == 1) 585 return WORK_CPU_UNBOUND; 586 587 /* Simple cpu selection based on qid modulo active cpu count */ 588 idx = !qid ? 0 : (qid - 1) % num_active_cpus(); 589 590 /* find the n'th active cpu */ 591 for (cpu = 0, cnt = 0; ; ) { 592 if (cpu_active(cpu)) { 593 if (cnt == idx) 594 break; 595 cnt++; 596 } 597 cpu = (cpu + 1) % num_possible_cpus(); 598 } 599 600 return cpu; 601 } 602 603 static struct nvmet_fc_tgt_queue * 604 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc, 605 u16 qid, u16 sqsize) 606 { 607 struct nvmet_fc_tgt_queue *queue; 608 unsigned long flags; 609 int ret; 610 611 if (qid >= NVMET_NR_QUEUES) 612 return NULL; 613 614 queue = kzalloc((sizeof(*queue) + 615 (sizeof(struct nvmet_fc_fcp_iod) * sqsize)), 616 GFP_KERNEL); 617 if (!queue) 618 return NULL; 619 620 if (!nvmet_fc_tgt_a_get(assoc)) 621 goto out_free_queue; 622 623 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0, 624 assoc->tgtport->fc_target_port.port_num, 625 assoc->a_id, qid); 626 if (!queue->work_q) 627 goto out_a_put; 628 629 queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1]; 630 queue->qid = qid; 631 queue->sqsize = sqsize; 632 queue->assoc = assoc; 633 queue->port = assoc->tgtport->port; 634 queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid); 635 INIT_LIST_HEAD(&queue->fod_list); 636 INIT_LIST_HEAD(&queue->avail_defer_list); 637 INIT_LIST_HEAD(&queue->pending_cmd_list); 638 atomic_set(&queue->connected, 0); 639 atomic_set(&queue->sqtail, 0); 640 atomic_set(&queue->rsn, 1); 641 atomic_set(&queue->zrspcnt, 0); 642 spin_lock_init(&queue->qlock); 643 kref_init(&queue->ref); 644 645 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue); 646 647 ret = nvmet_sq_init(&queue->nvme_sq); 648 if (ret) 649 goto out_fail_iodlist; 650 651 WARN_ON(assoc->queues[qid]); 652 spin_lock_irqsave(&assoc->tgtport->lock, flags); 653 assoc->queues[qid] = queue; 654 spin_unlock_irqrestore(&assoc->tgtport->lock, flags); 655 656 return queue; 657 658 out_fail_iodlist: 659 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue); 660 destroy_workqueue(queue->work_q); 661 out_a_put: 662 nvmet_fc_tgt_a_put(assoc); 663 out_free_queue: 664 kfree(queue); 665 return NULL; 666 } 667 668 669 static void 670 nvmet_fc_tgt_queue_free(struct kref *ref) 671 { 672 struct nvmet_fc_tgt_queue *queue = 673 container_of(ref, struct nvmet_fc_tgt_queue, ref); 674 unsigned long flags; 675 676 spin_lock_irqsave(&queue->assoc->tgtport->lock, flags); 677 queue->assoc->queues[queue->qid] = NULL; 678 spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags); 679 680 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue); 681 682 nvmet_fc_tgt_a_put(queue->assoc); 683 684 destroy_workqueue(queue->work_q); 685 686 kfree(queue); 687 } 688 689 static void 690 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue) 691 { 692 kref_put(&queue->ref, nvmet_fc_tgt_queue_free); 693 } 694 695 static int 696 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue) 697 { 698 return kref_get_unless_zero(&queue->ref); 699 } 700 701 702 static void 703 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue) 704 { 705 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport; 706 struct nvmet_fc_fcp_iod *fod = queue->fod; 707 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr; 708 unsigned long flags; 709 int i, writedataactive; 710 bool disconnect; 711 712 disconnect = atomic_xchg(&queue->connected, 0); 713 714 spin_lock_irqsave(&queue->qlock, flags); 715 /* about outstanding io's */ 716 for (i = 0; i < queue->sqsize; fod++, i++) { 717 if (fod->active) { 718 spin_lock(&fod->flock); 719 fod->abort = true; 720 writedataactive = fod->writedataactive; 721 spin_unlock(&fod->flock); 722 /* 723 * only call lldd abort routine if waiting for 724 * writedata. other outstanding ops should finish 725 * on their own. 726 */ 727 if (writedataactive) { 728 spin_lock(&fod->flock); 729 fod->aborted = true; 730 spin_unlock(&fod->flock); 731 tgtport->ops->fcp_abort( 732 &tgtport->fc_target_port, fod->fcpreq); 733 } 734 } 735 } 736 737 /* Cleanup defer'ed IOs in queue */ 738 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list, 739 req_list) { 740 list_del(&deferfcp->req_list); 741 kfree(deferfcp); 742 } 743 744 for (;;) { 745 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list, 746 struct nvmet_fc_defer_fcp_req, req_list); 747 if (!deferfcp) 748 break; 749 750 list_del(&deferfcp->req_list); 751 spin_unlock_irqrestore(&queue->qlock, flags); 752 753 tgtport->ops->defer_rcv(&tgtport->fc_target_port, 754 deferfcp->fcp_req); 755 756 tgtport->ops->fcp_abort(&tgtport->fc_target_port, 757 deferfcp->fcp_req); 758 759 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, 760 deferfcp->fcp_req); 761 762 kfree(deferfcp); 763 764 spin_lock_irqsave(&queue->qlock, flags); 765 } 766 spin_unlock_irqrestore(&queue->qlock, flags); 767 768 flush_workqueue(queue->work_q); 769 770 if (disconnect) 771 nvmet_sq_destroy(&queue->nvme_sq); 772 773 nvmet_fc_tgt_q_put(queue); 774 } 775 776 static struct nvmet_fc_tgt_queue * 777 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport, 778 u64 connection_id) 779 { 780 struct nvmet_fc_tgt_assoc *assoc; 781 struct nvmet_fc_tgt_queue *queue; 782 u64 association_id = nvmet_fc_getassociationid(connection_id); 783 u16 qid = nvmet_fc_getqueueid(connection_id); 784 unsigned long flags; 785 786 spin_lock_irqsave(&tgtport->lock, flags); 787 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) { 788 if (association_id == assoc->association_id) { 789 queue = assoc->queues[qid]; 790 if (queue && 791 (!atomic_read(&queue->connected) || 792 !nvmet_fc_tgt_q_get(queue))) 793 queue = NULL; 794 spin_unlock_irqrestore(&tgtport->lock, flags); 795 return queue; 796 } 797 } 798 spin_unlock_irqrestore(&tgtport->lock, flags); 799 return NULL; 800 } 801 802 static struct nvmet_fc_tgt_assoc * 803 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport) 804 { 805 struct nvmet_fc_tgt_assoc *assoc, *tmpassoc; 806 unsigned long flags; 807 u64 ran; 808 int idx; 809 bool needrandom = true; 810 811 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL); 812 if (!assoc) 813 return NULL; 814 815 idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL); 816 if (idx < 0) 817 goto out_free_assoc; 818 819 if (!nvmet_fc_tgtport_get(tgtport)) 820 goto out_ida_put; 821 822 assoc->tgtport = tgtport; 823 assoc->a_id = idx; 824 INIT_LIST_HEAD(&assoc->a_list); 825 kref_init(&assoc->ref); 826 827 while (needrandom) { 828 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID); 829 ran = ran << BYTES_FOR_QID_SHIFT; 830 831 spin_lock_irqsave(&tgtport->lock, flags); 832 needrandom = false; 833 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list) 834 if (ran == tmpassoc->association_id) { 835 needrandom = true; 836 break; 837 } 838 if (!needrandom) { 839 assoc->association_id = ran; 840 list_add_tail(&assoc->a_list, &tgtport->assoc_list); 841 } 842 spin_unlock_irqrestore(&tgtport->lock, flags); 843 } 844 845 return assoc; 846 847 out_ida_put: 848 ida_simple_remove(&tgtport->assoc_cnt, idx); 849 out_free_assoc: 850 kfree(assoc); 851 return NULL; 852 } 853 854 static void 855 nvmet_fc_target_assoc_free(struct kref *ref) 856 { 857 struct nvmet_fc_tgt_assoc *assoc = 858 container_of(ref, struct nvmet_fc_tgt_assoc, ref); 859 struct nvmet_fc_tgtport *tgtport = assoc->tgtport; 860 unsigned long flags; 861 862 spin_lock_irqsave(&tgtport->lock, flags); 863 list_del(&assoc->a_list); 864 spin_unlock_irqrestore(&tgtport->lock, flags); 865 ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id); 866 kfree(assoc); 867 nvmet_fc_tgtport_put(tgtport); 868 } 869 870 static void 871 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc) 872 { 873 kref_put(&assoc->ref, nvmet_fc_target_assoc_free); 874 } 875 876 static int 877 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc) 878 { 879 return kref_get_unless_zero(&assoc->ref); 880 } 881 882 static void 883 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc) 884 { 885 struct nvmet_fc_tgtport *tgtport = assoc->tgtport; 886 struct nvmet_fc_tgt_queue *queue; 887 unsigned long flags; 888 int i; 889 890 spin_lock_irqsave(&tgtport->lock, flags); 891 for (i = NVMET_NR_QUEUES - 1; i >= 0; i--) { 892 queue = assoc->queues[i]; 893 if (queue) { 894 if (!nvmet_fc_tgt_q_get(queue)) 895 continue; 896 spin_unlock_irqrestore(&tgtport->lock, flags); 897 nvmet_fc_delete_target_queue(queue); 898 nvmet_fc_tgt_q_put(queue); 899 spin_lock_irqsave(&tgtport->lock, flags); 900 } 901 } 902 spin_unlock_irqrestore(&tgtport->lock, flags); 903 904 nvmet_fc_tgt_a_put(assoc); 905 } 906 907 static struct nvmet_fc_tgt_assoc * 908 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport, 909 u64 association_id) 910 { 911 struct nvmet_fc_tgt_assoc *assoc; 912 struct nvmet_fc_tgt_assoc *ret = NULL; 913 unsigned long flags; 914 915 spin_lock_irqsave(&tgtport->lock, flags); 916 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) { 917 if (association_id == assoc->association_id) { 918 ret = assoc; 919 nvmet_fc_tgt_a_get(assoc); 920 break; 921 } 922 } 923 spin_unlock_irqrestore(&tgtport->lock, flags); 924 925 return ret; 926 } 927 928 929 /** 930 * nvme_fc_register_targetport - transport entry point called by an 931 * LLDD to register the existence of a local 932 * NVME subystem FC port. 933 * @pinfo: pointer to information about the port to be registered 934 * @template: LLDD entrypoints and operational parameters for the port 935 * @dev: physical hardware device node port corresponds to. Will be 936 * used for DMA mappings 937 * @portptr: pointer to a local port pointer. Upon success, the routine 938 * will allocate a nvme_fc_local_port structure and place its 939 * address in the local port pointer. Upon failure, local port 940 * pointer will be set to NULL. 941 * 942 * Returns: 943 * a completion status. Must be 0 upon success; a negative errno 944 * (ex: -ENXIO) upon failure. 945 */ 946 int 947 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, 948 struct nvmet_fc_target_template *template, 949 struct device *dev, 950 struct nvmet_fc_target_port **portptr) 951 { 952 struct nvmet_fc_tgtport *newrec; 953 unsigned long flags; 954 int ret, idx; 955 956 if (!template->xmt_ls_rsp || !template->fcp_op || 957 !template->fcp_abort || 958 !template->fcp_req_release || !template->targetport_delete || 959 !template->max_hw_queues || !template->max_sgl_segments || 960 !template->max_dif_sgl_segments || !template->dma_boundary) { 961 ret = -EINVAL; 962 goto out_regtgt_failed; 963 } 964 965 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz), 966 GFP_KERNEL); 967 if (!newrec) { 968 ret = -ENOMEM; 969 goto out_regtgt_failed; 970 } 971 972 idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL); 973 if (idx < 0) { 974 ret = -ENOSPC; 975 goto out_fail_kfree; 976 } 977 978 if (!get_device(dev) && dev) { 979 ret = -ENODEV; 980 goto out_ida_put; 981 } 982 983 newrec->fc_target_port.node_name = pinfo->node_name; 984 newrec->fc_target_port.port_name = pinfo->port_name; 985 newrec->fc_target_port.private = &newrec[1]; 986 newrec->fc_target_port.port_id = pinfo->port_id; 987 newrec->fc_target_port.port_num = idx; 988 INIT_LIST_HEAD(&newrec->tgt_list); 989 newrec->dev = dev; 990 newrec->ops = template; 991 spin_lock_init(&newrec->lock); 992 INIT_LIST_HEAD(&newrec->ls_list); 993 INIT_LIST_HEAD(&newrec->ls_busylist); 994 INIT_LIST_HEAD(&newrec->assoc_list); 995 kref_init(&newrec->ref); 996 ida_init(&newrec->assoc_cnt); 997 998 ret = nvmet_fc_alloc_ls_iodlist(newrec); 999 if (ret) { 1000 ret = -ENOMEM; 1001 goto out_free_newrec; 1002 } 1003 1004 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1005 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list); 1006 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1007 1008 *portptr = &newrec->fc_target_port; 1009 return 0; 1010 1011 out_free_newrec: 1012 put_device(dev); 1013 out_ida_put: 1014 ida_simple_remove(&nvmet_fc_tgtport_cnt, idx); 1015 out_fail_kfree: 1016 kfree(newrec); 1017 out_regtgt_failed: 1018 *portptr = NULL; 1019 return ret; 1020 } 1021 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport); 1022 1023 1024 static void 1025 nvmet_fc_free_tgtport(struct kref *ref) 1026 { 1027 struct nvmet_fc_tgtport *tgtport = 1028 container_of(ref, struct nvmet_fc_tgtport, ref); 1029 struct device *dev = tgtport->dev; 1030 unsigned long flags; 1031 1032 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1033 list_del(&tgtport->tgt_list); 1034 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1035 1036 nvmet_fc_free_ls_iodlist(tgtport); 1037 1038 /* let the LLDD know we've finished tearing it down */ 1039 tgtport->ops->targetport_delete(&tgtport->fc_target_port); 1040 1041 ida_simple_remove(&nvmet_fc_tgtport_cnt, 1042 tgtport->fc_target_port.port_num); 1043 1044 ida_destroy(&tgtport->assoc_cnt); 1045 1046 kfree(tgtport); 1047 1048 put_device(dev); 1049 } 1050 1051 static void 1052 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport) 1053 { 1054 kref_put(&tgtport->ref, nvmet_fc_free_tgtport); 1055 } 1056 1057 static int 1058 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport) 1059 { 1060 return kref_get_unless_zero(&tgtport->ref); 1061 } 1062 1063 static void 1064 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport) 1065 { 1066 struct nvmet_fc_tgt_assoc *assoc, *next; 1067 unsigned long flags; 1068 1069 spin_lock_irqsave(&tgtport->lock, flags); 1070 list_for_each_entry_safe(assoc, next, 1071 &tgtport->assoc_list, a_list) { 1072 if (!nvmet_fc_tgt_a_get(assoc)) 1073 continue; 1074 spin_unlock_irqrestore(&tgtport->lock, flags); 1075 nvmet_fc_delete_target_assoc(assoc); 1076 nvmet_fc_tgt_a_put(assoc); 1077 spin_lock_irqsave(&tgtport->lock, flags); 1078 } 1079 spin_unlock_irqrestore(&tgtport->lock, flags); 1080 } 1081 1082 /* 1083 * nvmet layer has called to terminate an association 1084 */ 1085 static void 1086 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) 1087 { 1088 struct nvmet_fc_tgtport *tgtport, *next; 1089 struct nvmet_fc_tgt_assoc *assoc; 1090 struct nvmet_fc_tgt_queue *queue; 1091 unsigned long flags; 1092 bool found_ctrl = false; 1093 1094 /* this is a bit ugly, but don't want to make locks layered */ 1095 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1096 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list, 1097 tgt_list) { 1098 if (!nvmet_fc_tgtport_get(tgtport)) 1099 continue; 1100 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1101 1102 spin_lock_irqsave(&tgtport->lock, flags); 1103 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) { 1104 queue = assoc->queues[0]; 1105 if (queue && queue->nvme_sq.ctrl == ctrl) { 1106 if (nvmet_fc_tgt_a_get(assoc)) 1107 found_ctrl = true; 1108 break; 1109 } 1110 } 1111 spin_unlock_irqrestore(&tgtport->lock, flags); 1112 1113 nvmet_fc_tgtport_put(tgtport); 1114 1115 if (found_ctrl) { 1116 nvmet_fc_delete_target_assoc(assoc); 1117 nvmet_fc_tgt_a_put(assoc); 1118 return; 1119 } 1120 1121 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1122 } 1123 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1124 } 1125 1126 /** 1127 * nvme_fc_unregister_targetport - transport entry point called by an 1128 * LLDD to deregister/remove a previously 1129 * registered a local NVME subsystem FC port. 1130 * @tgtport: pointer to the (registered) target port that is to be 1131 * deregistered. 1132 * 1133 * Returns: 1134 * a completion status. Must be 0 upon success; a negative errno 1135 * (ex: -ENXIO) upon failure. 1136 */ 1137 int 1138 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port) 1139 { 1140 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 1141 1142 /* terminate any outstanding associations */ 1143 __nvmet_fc_free_assocs(tgtport); 1144 1145 nvmet_fc_tgtport_put(tgtport); 1146 1147 return 0; 1148 } 1149 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport); 1150 1151 1152 /* *********************** FC-NVME LS Handling **************************** */ 1153 1154 1155 static void 1156 nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd) 1157 { 1158 struct fcnvme_ls_acc_hdr *acc = buf; 1159 1160 acc->w0.ls_cmd = ls_cmd; 1161 acc->desc_list_len = desc_len; 1162 acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST); 1163 acc->rqst.desc_len = 1164 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)); 1165 acc->rqst.w0.ls_cmd = rqst_ls_cmd; 1166 } 1167 1168 static int 1169 nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd, 1170 u8 reason, u8 explanation, u8 vendor) 1171 { 1172 struct fcnvme_ls_rjt *rjt = buf; 1173 1174 nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST, 1175 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)), 1176 ls_cmd); 1177 rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT); 1178 rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt)); 1179 rjt->rjt.reason_code = reason; 1180 rjt->rjt.reason_explanation = explanation; 1181 rjt->rjt.vendor = vendor; 1182 1183 return sizeof(struct fcnvme_ls_rjt); 1184 } 1185 1186 /* Validation Error indexes into the string table below */ 1187 enum { 1188 VERR_NO_ERROR = 0, 1189 VERR_CR_ASSOC_LEN = 1, 1190 VERR_CR_ASSOC_RQST_LEN = 2, 1191 VERR_CR_ASSOC_CMD = 3, 1192 VERR_CR_ASSOC_CMD_LEN = 4, 1193 VERR_ERSP_RATIO = 5, 1194 VERR_ASSOC_ALLOC_FAIL = 6, 1195 VERR_QUEUE_ALLOC_FAIL = 7, 1196 VERR_CR_CONN_LEN = 8, 1197 VERR_CR_CONN_RQST_LEN = 9, 1198 VERR_ASSOC_ID = 10, 1199 VERR_ASSOC_ID_LEN = 11, 1200 VERR_NO_ASSOC = 12, 1201 VERR_CONN_ID = 13, 1202 VERR_CONN_ID_LEN = 14, 1203 VERR_NO_CONN = 15, 1204 VERR_CR_CONN_CMD = 16, 1205 VERR_CR_CONN_CMD_LEN = 17, 1206 VERR_DISCONN_LEN = 18, 1207 VERR_DISCONN_RQST_LEN = 19, 1208 VERR_DISCONN_CMD = 20, 1209 VERR_DISCONN_CMD_LEN = 21, 1210 VERR_DISCONN_SCOPE = 22, 1211 VERR_RS_LEN = 23, 1212 VERR_RS_RQST_LEN = 24, 1213 VERR_RS_CMD = 25, 1214 VERR_RS_CMD_LEN = 26, 1215 VERR_RS_RCTL = 27, 1216 VERR_RS_RO = 28, 1217 }; 1218 1219 static char *validation_errors[] = { 1220 "OK", 1221 "Bad CR_ASSOC Length", 1222 "Bad CR_ASSOC Rqst Length", 1223 "Not CR_ASSOC Cmd", 1224 "Bad CR_ASSOC Cmd Length", 1225 "Bad Ersp Ratio", 1226 "Association Allocation Failed", 1227 "Queue Allocation Failed", 1228 "Bad CR_CONN Length", 1229 "Bad CR_CONN Rqst Length", 1230 "Not Association ID", 1231 "Bad Association ID Length", 1232 "No Association", 1233 "Not Connection ID", 1234 "Bad Connection ID Length", 1235 "No Connection", 1236 "Not CR_CONN Cmd", 1237 "Bad CR_CONN Cmd Length", 1238 "Bad DISCONN Length", 1239 "Bad DISCONN Rqst Length", 1240 "Not DISCONN Cmd", 1241 "Bad DISCONN Cmd Length", 1242 "Bad Disconnect Scope", 1243 "Bad RS Length", 1244 "Bad RS Rqst Length", 1245 "Not RS Cmd", 1246 "Bad RS Cmd Length", 1247 "Bad RS R_CTL", 1248 "Bad RS Relative Offset", 1249 }; 1250 1251 static void 1252 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, 1253 struct nvmet_fc_ls_iod *iod) 1254 { 1255 struct fcnvme_ls_cr_assoc_rqst *rqst = 1256 (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf; 1257 struct fcnvme_ls_cr_assoc_acc *acc = 1258 (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf; 1259 struct nvmet_fc_tgt_queue *queue; 1260 int ret = 0; 1261 1262 memset(acc, 0, sizeof(*acc)); 1263 1264 /* 1265 * FC-NVME spec changes. There are initiators sending different 1266 * lengths as padding sizes for Create Association Cmd descriptor 1267 * was incorrect. 1268 * Accept anything of "minimum" length. Assume format per 1.15 1269 * spec (with HOSTID reduced to 16 bytes), ignore how long the 1270 * trailing pad length is. 1271 */ 1272 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN) 1273 ret = VERR_CR_ASSOC_LEN; 1274 else if (be32_to_cpu(rqst->desc_list_len) < 1275 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN) 1276 ret = VERR_CR_ASSOC_RQST_LEN; 1277 else if (rqst->assoc_cmd.desc_tag != 1278 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD)) 1279 ret = VERR_CR_ASSOC_CMD; 1280 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) < 1281 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN) 1282 ret = VERR_CR_ASSOC_CMD_LEN; 1283 else if (!rqst->assoc_cmd.ersp_ratio || 1284 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >= 1285 be16_to_cpu(rqst->assoc_cmd.sqsize))) 1286 ret = VERR_ERSP_RATIO; 1287 1288 else { 1289 /* new association w/ admin queue */ 1290 iod->assoc = nvmet_fc_alloc_target_assoc(tgtport); 1291 if (!iod->assoc) 1292 ret = VERR_ASSOC_ALLOC_FAIL; 1293 else { 1294 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0, 1295 be16_to_cpu(rqst->assoc_cmd.sqsize)); 1296 if (!queue) 1297 ret = VERR_QUEUE_ALLOC_FAIL; 1298 } 1299 } 1300 1301 if (ret) { 1302 dev_err(tgtport->dev, 1303 "Create Association LS failed: %s\n", 1304 validation_errors[ret]); 1305 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, 1306 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, 1307 FCNVME_RJT_RC_LOGIC, 1308 FCNVME_RJT_EXP_NONE, 0); 1309 return; 1310 } 1311 1312 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio); 1313 atomic_set(&queue->connected, 1); 1314 queue->sqhd = 0; /* best place to init value */ 1315 1316 /* format a response */ 1317 1318 iod->lsreq->rsplen = sizeof(*acc); 1319 1320 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1321 fcnvme_lsdesc_len( 1322 sizeof(struct fcnvme_ls_cr_assoc_acc)), 1323 FCNVME_LS_CREATE_ASSOCIATION); 1324 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); 1325 acc->associd.desc_len = 1326 fcnvme_lsdesc_len( 1327 sizeof(struct fcnvme_lsdesc_assoc_id)); 1328 acc->associd.association_id = 1329 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0)); 1330 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); 1331 acc->connectid.desc_len = 1332 fcnvme_lsdesc_len( 1333 sizeof(struct fcnvme_lsdesc_conn_id)); 1334 acc->connectid.connection_id = acc->associd.association_id; 1335 } 1336 1337 static void 1338 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, 1339 struct nvmet_fc_ls_iod *iod) 1340 { 1341 struct fcnvme_ls_cr_conn_rqst *rqst = 1342 (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf; 1343 struct fcnvme_ls_cr_conn_acc *acc = 1344 (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf; 1345 struct nvmet_fc_tgt_queue *queue; 1346 int ret = 0; 1347 1348 memset(acc, 0, sizeof(*acc)); 1349 1350 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst)) 1351 ret = VERR_CR_CONN_LEN; 1352 else if (rqst->desc_list_len != 1353 fcnvme_lsdesc_len( 1354 sizeof(struct fcnvme_ls_cr_conn_rqst))) 1355 ret = VERR_CR_CONN_RQST_LEN; 1356 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) 1357 ret = VERR_ASSOC_ID; 1358 else if (rqst->associd.desc_len != 1359 fcnvme_lsdesc_len( 1360 sizeof(struct fcnvme_lsdesc_assoc_id))) 1361 ret = VERR_ASSOC_ID_LEN; 1362 else if (rqst->connect_cmd.desc_tag != 1363 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD)) 1364 ret = VERR_CR_CONN_CMD; 1365 else if (rqst->connect_cmd.desc_len != 1366 fcnvme_lsdesc_len( 1367 sizeof(struct fcnvme_lsdesc_cr_conn_cmd))) 1368 ret = VERR_CR_CONN_CMD_LEN; 1369 else if (!rqst->connect_cmd.ersp_ratio || 1370 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >= 1371 be16_to_cpu(rqst->connect_cmd.sqsize))) 1372 ret = VERR_ERSP_RATIO; 1373 1374 else { 1375 /* new io queue */ 1376 iod->assoc = nvmet_fc_find_target_assoc(tgtport, 1377 be64_to_cpu(rqst->associd.association_id)); 1378 if (!iod->assoc) 1379 ret = VERR_NO_ASSOC; 1380 else { 1381 queue = nvmet_fc_alloc_target_queue(iod->assoc, 1382 be16_to_cpu(rqst->connect_cmd.qid), 1383 be16_to_cpu(rqst->connect_cmd.sqsize)); 1384 if (!queue) 1385 ret = VERR_QUEUE_ALLOC_FAIL; 1386 1387 /* release get taken in nvmet_fc_find_target_assoc */ 1388 nvmet_fc_tgt_a_put(iod->assoc); 1389 } 1390 } 1391 1392 if (ret) { 1393 dev_err(tgtport->dev, 1394 "Create Connection LS failed: %s\n", 1395 validation_errors[ret]); 1396 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, 1397 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, 1398 (ret == VERR_NO_ASSOC) ? 1399 FCNVME_RJT_RC_INV_ASSOC : 1400 FCNVME_RJT_RC_LOGIC, 1401 FCNVME_RJT_EXP_NONE, 0); 1402 return; 1403 } 1404 1405 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio); 1406 atomic_set(&queue->connected, 1); 1407 queue->sqhd = 0; /* best place to init value */ 1408 1409 /* format a response */ 1410 1411 iod->lsreq->rsplen = sizeof(*acc); 1412 1413 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1414 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)), 1415 FCNVME_LS_CREATE_CONNECTION); 1416 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); 1417 acc->connectid.desc_len = 1418 fcnvme_lsdesc_len( 1419 sizeof(struct fcnvme_lsdesc_conn_id)); 1420 acc->connectid.connection_id = 1421 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 1422 be16_to_cpu(rqst->connect_cmd.qid))); 1423 } 1424 1425 static void 1426 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, 1427 struct nvmet_fc_ls_iod *iod) 1428 { 1429 struct fcnvme_ls_disconnect_rqst *rqst = 1430 (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf; 1431 struct fcnvme_ls_disconnect_acc *acc = 1432 (struct fcnvme_ls_disconnect_acc *)iod->rspbuf; 1433 struct nvmet_fc_tgt_queue *queue = NULL; 1434 struct nvmet_fc_tgt_assoc *assoc; 1435 int ret = 0; 1436 bool del_assoc = false; 1437 1438 memset(acc, 0, sizeof(*acc)); 1439 1440 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst)) 1441 ret = VERR_DISCONN_LEN; 1442 else if (rqst->desc_list_len != 1443 fcnvme_lsdesc_len( 1444 sizeof(struct fcnvme_ls_disconnect_rqst))) 1445 ret = VERR_DISCONN_RQST_LEN; 1446 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) 1447 ret = VERR_ASSOC_ID; 1448 else if (rqst->associd.desc_len != 1449 fcnvme_lsdesc_len( 1450 sizeof(struct fcnvme_lsdesc_assoc_id))) 1451 ret = VERR_ASSOC_ID_LEN; 1452 else if (rqst->discon_cmd.desc_tag != 1453 cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD)) 1454 ret = VERR_DISCONN_CMD; 1455 else if (rqst->discon_cmd.desc_len != 1456 fcnvme_lsdesc_len( 1457 sizeof(struct fcnvme_lsdesc_disconn_cmd))) 1458 ret = VERR_DISCONN_CMD_LEN; 1459 else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) && 1460 (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION)) 1461 ret = VERR_DISCONN_SCOPE; 1462 else { 1463 /* match an active association */ 1464 assoc = nvmet_fc_find_target_assoc(tgtport, 1465 be64_to_cpu(rqst->associd.association_id)); 1466 iod->assoc = assoc; 1467 if (assoc) { 1468 if (rqst->discon_cmd.scope == 1469 FCNVME_DISCONN_CONNECTION) { 1470 queue = nvmet_fc_find_target_queue(tgtport, 1471 be64_to_cpu( 1472 rqst->discon_cmd.id)); 1473 if (!queue) { 1474 nvmet_fc_tgt_a_put(assoc); 1475 ret = VERR_NO_CONN; 1476 } 1477 } 1478 } else 1479 ret = VERR_NO_ASSOC; 1480 } 1481 1482 if (ret) { 1483 dev_err(tgtport->dev, 1484 "Disconnect LS failed: %s\n", 1485 validation_errors[ret]); 1486 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, 1487 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, 1488 (ret == VERR_NO_ASSOC) ? 1489 FCNVME_RJT_RC_INV_ASSOC : 1490 (ret == VERR_NO_CONN) ? 1491 FCNVME_RJT_RC_INV_CONN : 1492 FCNVME_RJT_RC_LOGIC, 1493 FCNVME_RJT_EXP_NONE, 0); 1494 return; 1495 } 1496 1497 /* format a response */ 1498 1499 iod->lsreq->rsplen = sizeof(*acc); 1500 1501 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1502 fcnvme_lsdesc_len( 1503 sizeof(struct fcnvme_ls_disconnect_acc)), 1504 FCNVME_LS_DISCONNECT); 1505 1506 1507 /* are we to delete a Connection ID (queue) */ 1508 if (queue) { 1509 int qid = queue->qid; 1510 1511 nvmet_fc_delete_target_queue(queue); 1512 1513 /* release the get taken by find_target_queue */ 1514 nvmet_fc_tgt_q_put(queue); 1515 1516 /* tear association down if io queue terminated */ 1517 if (!qid) 1518 del_assoc = true; 1519 } 1520 1521 /* release get taken in nvmet_fc_find_target_assoc */ 1522 nvmet_fc_tgt_a_put(iod->assoc); 1523 1524 if (del_assoc) 1525 nvmet_fc_delete_target_assoc(iod->assoc); 1526 } 1527 1528 1529 /* *********************** NVME Ctrl Routines **************************** */ 1530 1531 1532 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req); 1533 1534 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops; 1535 1536 static void 1537 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq) 1538 { 1539 struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private; 1540 struct nvmet_fc_tgtport *tgtport = iod->tgtport; 1541 1542 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma, 1543 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); 1544 nvmet_fc_free_ls_iod(tgtport, iod); 1545 nvmet_fc_tgtport_put(tgtport); 1546 } 1547 1548 static void 1549 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, 1550 struct nvmet_fc_ls_iod *iod) 1551 { 1552 int ret; 1553 1554 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma, 1555 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); 1556 1557 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq); 1558 if (ret) 1559 nvmet_fc_xmt_ls_rsp_done(iod->lsreq); 1560 } 1561 1562 /* 1563 * Actual processing routine for received FC-NVME LS Requests from the LLD 1564 */ 1565 static void 1566 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, 1567 struct nvmet_fc_ls_iod *iod) 1568 { 1569 struct fcnvme_ls_rqst_w0 *w0 = 1570 (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf; 1571 1572 iod->lsreq->nvmet_fc_private = iod; 1573 iod->lsreq->rspbuf = iod->rspbuf; 1574 iod->lsreq->rspdma = iod->rspdma; 1575 iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done; 1576 /* Be preventative. handlers will later set to valid length */ 1577 iod->lsreq->rsplen = 0; 1578 1579 iod->assoc = NULL; 1580 1581 /* 1582 * handlers: 1583 * parse request input, execute the request, and format the 1584 * LS response 1585 */ 1586 switch (w0->ls_cmd) { 1587 case FCNVME_LS_CREATE_ASSOCIATION: 1588 /* Creates Association and initial Admin Queue/Connection */ 1589 nvmet_fc_ls_create_association(tgtport, iod); 1590 break; 1591 case FCNVME_LS_CREATE_CONNECTION: 1592 /* Creates an IO Queue/Connection */ 1593 nvmet_fc_ls_create_connection(tgtport, iod); 1594 break; 1595 case FCNVME_LS_DISCONNECT: 1596 /* Terminate a Queue/Connection or the Association */ 1597 nvmet_fc_ls_disconnect(tgtport, iod); 1598 break; 1599 default: 1600 iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf, 1601 NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd, 1602 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); 1603 } 1604 1605 nvmet_fc_xmt_ls_rsp(tgtport, iod); 1606 } 1607 1608 /* 1609 * Actual processing routine for received FC-NVME LS Requests from the LLD 1610 */ 1611 static void 1612 nvmet_fc_handle_ls_rqst_work(struct work_struct *work) 1613 { 1614 struct nvmet_fc_ls_iod *iod = 1615 container_of(work, struct nvmet_fc_ls_iod, work); 1616 struct nvmet_fc_tgtport *tgtport = iod->tgtport; 1617 1618 nvmet_fc_handle_ls_rqst(tgtport, iod); 1619 } 1620 1621 1622 /** 1623 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD 1624 * upon the reception of a NVME LS request. 1625 * 1626 * The nvmet-fc layer will copy payload to an internal structure for 1627 * processing. As such, upon completion of the routine, the LLDD may 1628 * immediately free/reuse the LS request buffer passed in the call. 1629 * 1630 * If this routine returns error, the LLDD should abort the exchange. 1631 * 1632 * @tgtport: pointer to the (registered) target port the LS was 1633 * received on. 1634 * @lsreq: pointer to a lsreq request structure to be used to reference 1635 * the exchange corresponding to the LS. 1636 * @lsreqbuf: pointer to the buffer containing the LS Request 1637 * @lsreqbuf_len: length, in bytes, of the received LS request 1638 */ 1639 int 1640 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, 1641 struct nvmefc_tgt_ls_req *lsreq, 1642 void *lsreqbuf, u32 lsreqbuf_len) 1643 { 1644 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 1645 struct nvmet_fc_ls_iod *iod; 1646 1647 if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE) 1648 return -E2BIG; 1649 1650 if (!nvmet_fc_tgtport_get(tgtport)) 1651 return -ESHUTDOWN; 1652 1653 iod = nvmet_fc_alloc_ls_iod(tgtport); 1654 if (!iod) { 1655 nvmet_fc_tgtport_put(tgtport); 1656 return -ENOENT; 1657 } 1658 1659 iod->lsreq = lsreq; 1660 iod->fcpreq = NULL; 1661 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len); 1662 iod->rqstdatalen = lsreqbuf_len; 1663 1664 schedule_work(&iod->work); 1665 1666 return 0; 1667 } 1668 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req); 1669 1670 1671 /* 1672 * ********************** 1673 * Start of FCP handling 1674 * ********************** 1675 */ 1676 1677 static int 1678 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod) 1679 { 1680 struct scatterlist *sg; 1681 struct page *page; 1682 unsigned int nent; 1683 u32 page_len, length; 1684 int i = 0; 1685 1686 length = fod->total_length; 1687 nent = DIV_ROUND_UP(length, PAGE_SIZE); 1688 sg = kmalloc_array(nent, sizeof(struct scatterlist), GFP_KERNEL); 1689 if (!sg) 1690 goto out; 1691 1692 sg_init_table(sg, nent); 1693 1694 while (length) { 1695 page_len = min_t(u32, length, PAGE_SIZE); 1696 1697 page = alloc_page(GFP_KERNEL); 1698 if (!page) 1699 goto out_free_pages; 1700 1701 sg_set_page(&sg[i], page, page_len, 0); 1702 length -= page_len; 1703 i++; 1704 } 1705 1706 fod->data_sg = sg; 1707 fod->data_sg_cnt = nent; 1708 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent, 1709 ((fod->io_dir == NVMET_FCP_WRITE) ? 1710 DMA_FROM_DEVICE : DMA_TO_DEVICE)); 1711 /* note: write from initiator perspective */ 1712 1713 return 0; 1714 1715 out_free_pages: 1716 while (i > 0) { 1717 i--; 1718 __free_page(sg_page(&sg[i])); 1719 } 1720 kfree(sg); 1721 fod->data_sg = NULL; 1722 fod->data_sg_cnt = 0; 1723 out: 1724 return NVME_SC_INTERNAL; 1725 } 1726 1727 static void 1728 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod) 1729 { 1730 struct scatterlist *sg; 1731 int count; 1732 1733 if (!fod->data_sg || !fod->data_sg_cnt) 1734 return; 1735 1736 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt, 1737 ((fod->io_dir == NVMET_FCP_WRITE) ? 1738 DMA_FROM_DEVICE : DMA_TO_DEVICE)); 1739 for_each_sg(fod->data_sg, sg, fod->data_sg_cnt, count) 1740 __free_page(sg_page(sg)); 1741 kfree(fod->data_sg); 1742 fod->data_sg = NULL; 1743 fod->data_sg_cnt = 0; 1744 } 1745 1746 1747 static bool 1748 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd) 1749 { 1750 u32 sqtail, used; 1751 1752 /* egad, this is ugly. And sqtail is just a best guess */ 1753 sqtail = atomic_read(&q->sqtail) % q->sqsize; 1754 1755 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd); 1756 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9))); 1757 } 1758 1759 /* 1760 * Prep RSP payload. 1761 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op 1762 */ 1763 static void 1764 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport, 1765 struct nvmet_fc_fcp_iod *fod) 1766 { 1767 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf; 1768 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common; 1769 struct nvme_completion *cqe = &ersp->cqe; 1770 u32 *cqewd = (u32 *)cqe; 1771 bool send_ersp = false; 1772 u32 rsn, rspcnt, xfr_length; 1773 1774 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP) 1775 xfr_length = fod->total_length; 1776 else 1777 xfr_length = fod->offset; 1778 1779 /* 1780 * check to see if we can send a 0's rsp. 1781 * Note: to send a 0's response, the NVME-FC host transport will 1782 * recreate the CQE. The host transport knows: sq id, SQHD (last 1783 * seen in an ersp), and command_id. Thus it will create a 1784 * zero-filled CQE with those known fields filled in. Transport 1785 * must send an ersp for any condition where the cqe won't match 1786 * this. 1787 * 1788 * Here are the FC-NVME mandated cases where we must send an ersp: 1789 * every N responses, where N=ersp_ratio 1790 * force fabric commands to send ersp's (not in FC-NVME but good 1791 * practice) 1792 * normal cmds: any time status is non-zero, or status is zero 1793 * but words 0 or 1 are non-zero. 1794 * the SQ is 90% or more full 1795 * the cmd is a fused command 1796 * transferred data length not equal to cmd iu length 1797 */ 1798 rspcnt = atomic_inc_return(&fod->queue->zrspcnt); 1799 if (!(rspcnt % fod->queue->ersp_ratio) || 1800 sqe->opcode == nvme_fabrics_command || 1801 xfr_length != fod->total_length || 1802 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] || 1803 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) || 1804 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head))) 1805 send_ersp = true; 1806 1807 /* re-set the fields */ 1808 fod->fcpreq->rspaddr = ersp; 1809 fod->fcpreq->rspdma = fod->rspdma; 1810 1811 if (!send_ersp) { 1812 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP); 1813 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP; 1814 } else { 1815 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32)); 1816 rsn = atomic_inc_return(&fod->queue->rsn); 1817 ersp->rsn = cpu_to_be32(rsn); 1818 ersp->xfrd_len = cpu_to_be32(xfr_length); 1819 fod->fcpreq->rsplen = sizeof(*ersp); 1820 } 1821 1822 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma, 1823 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 1824 } 1825 1826 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq); 1827 1828 static void 1829 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport, 1830 struct nvmet_fc_fcp_iod *fod) 1831 { 1832 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 1833 1834 /* data no longer needed */ 1835 nvmet_fc_free_tgt_pgs(fod); 1836 1837 /* 1838 * if an ABTS was received or we issued the fcp_abort early 1839 * don't call abort routine again. 1840 */ 1841 /* no need to take lock - lock was taken earlier to get here */ 1842 if (!fod->aborted) 1843 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq); 1844 1845 nvmet_fc_free_fcp_iod(fod->queue, fod); 1846 } 1847 1848 static void 1849 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport, 1850 struct nvmet_fc_fcp_iod *fod) 1851 { 1852 int ret; 1853 1854 fod->fcpreq->op = NVMET_FCOP_RSP; 1855 fod->fcpreq->timeout = 0; 1856 1857 nvmet_fc_prep_fcp_rsp(tgtport, fod); 1858 1859 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); 1860 if (ret) 1861 nvmet_fc_abort_op(tgtport, fod); 1862 } 1863 1864 static void 1865 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport, 1866 struct nvmet_fc_fcp_iod *fod, u8 op) 1867 { 1868 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 1869 struct scatterlist *sg, *datasg; 1870 unsigned long flags; 1871 u32 tlen, sg_off; 1872 int ret; 1873 1874 fcpreq->op = op; 1875 fcpreq->offset = fod->offset; 1876 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC; 1877 tlen = min_t(u32, (NVMET_FC_MAX_KB_PER_XFR * 1024), 1878 (fod->total_length - fod->offset)); 1879 tlen = min_t(u32, tlen, NVME_FC_MAX_SEGMENTS * PAGE_SIZE); 1880 tlen = min_t(u32, tlen, fod->tgtport->ops->max_sgl_segments 1881 * PAGE_SIZE); 1882 fcpreq->transfer_length = tlen; 1883 fcpreq->transferred_length = 0; 1884 fcpreq->fcp_error = 0; 1885 fcpreq->rsplen = 0; 1886 1887 fcpreq->sg_cnt = 0; 1888 1889 datasg = fod->next_sg; 1890 sg_off = fod->next_sg_offset; 1891 1892 for (sg = fcpreq->sg ; tlen; sg++) { 1893 *sg = *datasg; 1894 if (sg_off) { 1895 sg->offset += sg_off; 1896 sg->length -= sg_off; 1897 sg->dma_address += sg_off; 1898 sg_off = 0; 1899 } 1900 if (tlen < sg->length) { 1901 sg->length = tlen; 1902 fod->next_sg = datasg; 1903 fod->next_sg_offset += tlen; 1904 } else if (tlen == sg->length) { 1905 fod->next_sg_offset = 0; 1906 fod->next_sg = sg_next(datasg); 1907 } else { 1908 fod->next_sg_offset = 0; 1909 datasg = sg_next(datasg); 1910 } 1911 tlen -= sg->length; 1912 fcpreq->sg_cnt++; 1913 } 1914 1915 /* 1916 * If the last READDATA request: check if LLDD supports 1917 * combined xfr with response. 1918 */ 1919 if ((op == NVMET_FCOP_READDATA) && 1920 ((fod->offset + fcpreq->transfer_length) == fod->total_length) && 1921 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) { 1922 fcpreq->op = NVMET_FCOP_READDATA_RSP; 1923 nvmet_fc_prep_fcp_rsp(tgtport, fod); 1924 } 1925 1926 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); 1927 if (ret) { 1928 /* 1929 * should be ok to set w/o lock as its in the thread of 1930 * execution (not an async timer routine) and doesn't 1931 * contend with any clearing action 1932 */ 1933 fod->abort = true; 1934 1935 if (op == NVMET_FCOP_WRITEDATA) { 1936 spin_lock_irqsave(&fod->flock, flags); 1937 fod->writedataactive = false; 1938 spin_unlock_irqrestore(&fod->flock, flags); 1939 nvmet_req_complete(&fod->req, 1940 NVME_SC_FC_TRANSPORT_ERROR); 1941 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ { 1942 fcpreq->fcp_error = ret; 1943 fcpreq->transferred_length = 0; 1944 nvmet_fc_xmt_fcp_op_done(fod->fcpreq); 1945 } 1946 } 1947 } 1948 1949 static inline bool 1950 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort) 1951 { 1952 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 1953 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 1954 1955 /* if in the middle of an io and we need to tear down */ 1956 if (abort) { 1957 if (fcpreq->op == NVMET_FCOP_WRITEDATA) { 1958 nvmet_req_complete(&fod->req, 1959 NVME_SC_FC_TRANSPORT_ERROR); 1960 return true; 1961 } 1962 1963 nvmet_fc_abort_op(tgtport, fod); 1964 return true; 1965 } 1966 1967 return false; 1968 } 1969 1970 /* 1971 * actual done handler for FCP operations when completed by the lldd 1972 */ 1973 static void 1974 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod) 1975 { 1976 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 1977 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 1978 unsigned long flags; 1979 bool abort; 1980 1981 spin_lock_irqsave(&fod->flock, flags); 1982 abort = fod->abort; 1983 fod->writedataactive = false; 1984 spin_unlock_irqrestore(&fod->flock, flags); 1985 1986 switch (fcpreq->op) { 1987 1988 case NVMET_FCOP_WRITEDATA: 1989 if (__nvmet_fc_fod_op_abort(fod, abort)) 1990 return; 1991 if (fcpreq->fcp_error || 1992 fcpreq->transferred_length != fcpreq->transfer_length) { 1993 spin_lock(&fod->flock); 1994 fod->abort = true; 1995 spin_unlock(&fod->flock); 1996 1997 nvmet_req_complete(&fod->req, 1998 NVME_SC_FC_TRANSPORT_ERROR); 1999 return; 2000 } 2001 2002 fod->offset += fcpreq->transferred_length; 2003 if (fod->offset != fod->total_length) { 2004 spin_lock_irqsave(&fod->flock, flags); 2005 fod->writedataactive = true; 2006 spin_unlock_irqrestore(&fod->flock, flags); 2007 2008 /* transfer the next chunk */ 2009 nvmet_fc_transfer_fcp_data(tgtport, fod, 2010 NVMET_FCOP_WRITEDATA); 2011 return; 2012 } 2013 2014 /* data transfer complete, resume with nvmet layer */ 2015 2016 fod->req.execute(&fod->req); 2017 2018 break; 2019 2020 case NVMET_FCOP_READDATA: 2021 case NVMET_FCOP_READDATA_RSP: 2022 if (__nvmet_fc_fod_op_abort(fod, abort)) 2023 return; 2024 if (fcpreq->fcp_error || 2025 fcpreq->transferred_length != fcpreq->transfer_length) { 2026 nvmet_fc_abort_op(tgtport, fod); 2027 return; 2028 } 2029 2030 /* success */ 2031 2032 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) { 2033 /* data no longer needed */ 2034 nvmet_fc_free_tgt_pgs(fod); 2035 nvmet_fc_free_fcp_iod(fod->queue, fod); 2036 return; 2037 } 2038 2039 fod->offset += fcpreq->transferred_length; 2040 if (fod->offset != fod->total_length) { 2041 /* transfer the next chunk */ 2042 nvmet_fc_transfer_fcp_data(tgtport, fod, 2043 NVMET_FCOP_READDATA); 2044 return; 2045 } 2046 2047 /* data transfer complete, send response */ 2048 2049 /* data no longer needed */ 2050 nvmet_fc_free_tgt_pgs(fod); 2051 2052 nvmet_fc_xmt_fcp_rsp(tgtport, fod); 2053 2054 break; 2055 2056 case NVMET_FCOP_RSP: 2057 if (__nvmet_fc_fod_op_abort(fod, abort)) 2058 return; 2059 nvmet_fc_free_fcp_iod(fod->queue, fod); 2060 break; 2061 2062 default: 2063 break; 2064 } 2065 } 2066 2067 static void 2068 nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work) 2069 { 2070 struct nvmet_fc_fcp_iod *fod = 2071 container_of(work, struct nvmet_fc_fcp_iod, done_work); 2072 2073 nvmet_fc_fod_op_done(fod); 2074 } 2075 2076 static void 2077 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq) 2078 { 2079 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 2080 struct nvmet_fc_tgt_queue *queue = fod->queue; 2081 2082 if (fod->tgtport->ops->target_features & NVMET_FCTGTFEAT_OPDONE_IN_ISR) 2083 /* context switch so completion is not in ISR context */ 2084 queue_work_on(queue->cpu, queue->work_q, &fod->done_work); 2085 else 2086 nvmet_fc_fod_op_done(fod); 2087 } 2088 2089 /* 2090 * actual completion handler after execution by the nvmet layer 2091 */ 2092 static void 2093 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport, 2094 struct nvmet_fc_fcp_iod *fod, int status) 2095 { 2096 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common; 2097 struct nvme_completion *cqe = &fod->rspiubuf.cqe; 2098 unsigned long flags; 2099 bool abort; 2100 2101 spin_lock_irqsave(&fod->flock, flags); 2102 abort = fod->abort; 2103 spin_unlock_irqrestore(&fod->flock, flags); 2104 2105 /* if we have a CQE, snoop the last sq_head value */ 2106 if (!status) 2107 fod->queue->sqhd = cqe->sq_head; 2108 2109 if (abort) { 2110 nvmet_fc_abort_op(tgtport, fod); 2111 return; 2112 } 2113 2114 /* if an error handling the cmd post initial parsing */ 2115 if (status) { 2116 /* fudge up a failed CQE status for our transport error */ 2117 memset(cqe, 0, sizeof(*cqe)); 2118 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */ 2119 cqe->sq_id = cpu_to_le16(fod->queue->qid); 2120 cqe->command_id = sqe->command_id; 2121 cqe->status = cpu_to_le16(status); 2122 } else { 2123 2124 /* 2125 * try to push the data even if the SQE status is non-zero. 2126 * There may be a status where data still was intended to 2127 * be moved 2128 */ 2129 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) { 2130 /* push the data over before sending rsp */ 2131 nvmet_fc_transfer_fcp_data(tgtport, fod, 2132 NVMET_FCOP_READDATA); 2133 return; 2134 } 2135 2136 /* writes & no data - fall thru */ 2137 } 2138 2139 /* data no longer needed */ 2140 nvmet_fc_free_tgt_pgs(fod); 2141 2142 nvmet_fc_xmt_fcp_rsp(tgtport, fod); 2143 } 2144 2145 2146 static void 2147 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req) 2148 { 2149 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req); 2150 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 2151 2152 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0); 2153 } 2154 2155 2156 /* 2157 * Actual processing routine for received FC-NVME LS Requests from the LLD 2158 */ 2159 static void 2160 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, 2161 struct nvmet_fc_fcp_iod *fod) 2162 { 2163 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf; 2164 int ret; 2165 2166 /* 2167 * Fused commands are currently not supported in the linux 2168 * implementation. 2169 * 2170 * As such, the implementation of the FC transport does not 2171 * look at the fused commands and order delivery to the upper 2172 * layer until we have both based on csn. 2173 */ 2174 2175 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done; 2176 2177 fod->total_length = be32_to_cpu(cmdiu->data_len); 2178 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) { 2179 fod->io_dir = NVMET_FCP_WRITE; 2180 if (!nvme_is_write(&cmdiu->sqe)) 2181 goto transport_error; 2182 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) { 2183 fod->io_dir = NVMET_FCP_READ; 2184 if (nvme_is_write(&cmdiu->sqe)) 2185 goto transport_error; 2186 } else { 2187 fod->io_dir = NVMET_FCP_NODATA; 2188 if (fod->total_length) 2189 goto transport_error; 2190 } 2191 2192 fod->req.cmd = &fod->cmdiubuf.sqe; 2193 fod->req.rsp = &fod->rspiubuf.cqe; 2194 fod->req.port = fod->queue->port; 2195 2196 /* ensure nvmet handlers will set cmd handler callback */ 2197 fod->req.execute = NULL; 2198 2199 /* clear any response payload */ 2200 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf)); 2201 2202 fod->data_sg = NULL; 2203 fod->data_sg_cnt = 0; 2204 2205 ret = nvmet_req_init(&fod->req, 2206 &fod->queue->nvme_cq, 2207 &fod->queue->nvme_sq, 2208 &nvmet_fc_tgt_fcp_ops); 2209 if (!ret) { 2210 /* bad SQE content or invalid ctrl state */ 2211 /* nvmet layer has already called op done to send rsp. */ 2212 return; 2213 } 2214 2215 /* keep a running counter of tail position */ 2216 atomic_inc(&fod->queue->sqtail); 2217 2218 if (fod->total_length) { 2219 ret = nvmet_fc_alloc_tgt_pgs(fod); 2220 if (ret) { 2221 nvmet_req_complete(&fod->req, ret); 2222 return; 2223 } 2224 } 2225 fod->req.sg = fod->data_sg; 2226 fod->req.sg_cnt = fod->data_sg_cnt; 2227 fod->offset = 0; 2228 fod->next_sg = fod->data_sg; 2229 fod->next_sg_offset = 0; 2230 2231 if (fod->io_dir == NVMET_FCP_WRITE) { 2232 /* pull the data over before invoking nvmet layer */ 2233 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA); 2234 return; 2235 } 2236 2237 /* 2238 * Reads or no data: 2239 * 2240 * can invoke the nvmet_layer now. If read data, cmd completion will 2241 * push the data 2242 */ 2243 2244 fod->req.execute(&fod->req); 2245 2246 return; 2247 2248 transport_error: 2249 nvmet_fc_abort_op(tgtport, fod); 2250 } 2251 2252 /* 2253 * Actual processing routine for received FC-NVME LS Requests from the LLD 2254 */ 2255 static void 2256 nvmet_fc_handle_fcp_rqst_work(struct work_struct *work) 2257 { 2258 struct nvmet_fc_fcp_iod *fod = 2259 container_of(work, struct nvmet_fc_fcp_iod, work); 2260 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 2261 2262 nvmet_fc_handle_fcp_rqst(tgtport, fod); 2263 } 2264 2265 /** 2266 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD 2267 * upon the reception of a NVME FCP CMD IU. 2268 * 2269 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc 2270 * layer for processing. 2271 * 2272 * The nvmet_fc layer allocates a local job structure (struct 2273 * nvmet_fc_fcp_iod) from the queue for the io and copies the 2274 * CMD IU buffer to the job structure. As such, on a successful 2275 * completion (returns 0), the LLDD may immediately free/reuse 2276 * the CMD IU buffer passed in the call. 2277 * 2278 * However, in some circumstances, due to the packetized nature of FC 2279 * and the api of the FC LLDD which may issue a hw command to send the 2280 * response, but the LLDD may not get the hw completion for that command 2281 * and upcall the nvmet_fc layer before a new command may be 2282 * asynchronously received - its possible for a command to be received 2283 * before the LLDD and nvmet_fc have recycled the job structure. It gives 2284 * the appearance of more commands received than fits in the sq. 2285 * To alleviate this scenario, a temporary queue is maintained in the 2286 * transport for pending LLDD requests waiting for a queue job structure. 2287 * In these "overrun" cases, a temporary queue element is allocated 2288 * the LLDD request and CMD iu buffer information remembered, and the 2289 * routine returns a -EOVERFLOW status. Subsequently, when a queue job 2290 * structure is freed, it is immediately reallocated for anything on the 2291 * pending request list. The LLDDs defer_rcv() callback is called, 2292 * informing the LLDD that it may reuse the CMD IU buffer, and the io 2293 * is then started normally with the transport. 2294 * 2295 * The LLDD, when receiving an -EOVERFLOW completion status, is to treat 2296 * the completion as successful but must not reuse the CMD IU buffer 2297 * until the LLDD's defer_rcv() callback has been called for the 2298 * corresponding struct nvmefc_tgt_fcp_req pointer. 2299 * 2300 * If there is any other condition in which an error occurs, the 2301 * transport will return a non-zero status indicating the error. 2302 * In all cases other than -EOVERFLOW, the transport has not accepted the 2303 * request and the LLDD should abort the exchange. 2304 * 2305 * @target_port: pointer to the (registered) target port the FCP CMD IU 2306 * was received on. 2307 * @fcpreq: pointer to a fcpreq request structure to be used to reference 2308 * the exchange corresponding to the FCP Exchange. 2309 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU 2310 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU 2311 */ 2312 int 2313 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port, 2314 struct nvmefc_tgt_fcp_req *fcpreq, 2315 void *cmdiubuf, u32 cmdiubuf_len) 2316 { 2317 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 2318 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf; 2319 struct nvmet_fc_tgt_queue *queue; 2320 struct nvmet_fc_fcp_iod *fod; 2321 struct nvmet_fc_defer_fcp_req *deferfcp; 2322 unsigned long flags; 2323 2324 /* validate iu, so the connection id can be used to find the queue */ 2325 if ((cmdiubuf_len != sizeof(*cmdiu)) || 2326 (cmdiu->scsi_id != NVME_CMD_SCSI_ID) || 2327 (cmdiu->fc_id != NVME_CMD_FC_ID) || 2328 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4))) 2329 return -EIO; 2330 2331 queue = nvmet_fc_find_target_queue(tgtport, 2332 be64_to_cpu(cmdiu->connection_id)); 2333 if (!queue) 2334 return -ENOTCONN; 2335 2336 /* 2337 * note: reference taken by find_target_queue 2338 * After successful fod allocation, the fod will inherit the 2339 * ownership of that reference and will remove the reference 2340 * when the fod is freed. 2341 */ 2342 2343 spin_lock_irqsave(&queue->qlock, flags); 2344 2345 fod = nvmet_fc_alloc_fcp_iod(queue); 2346 if (fod) { 2347 spin_unlock_irqrestore(&queue->qlock, flags); 2348 2349 fcpreq->nvmet_fc_private = fod; 2350 fod->fcpreq = fcpreq; 2351 2352 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len); 2353 2354 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq); 2355 2356 return 0; 2357 } 2358 2359 if (!tgtport->ops->defer_rcv) { 2360 spin_unlock_irqrestore(&queue->qlock, flags); 2361 /* release the queue lookup reference */ 2362 nvmet_fc_tgt_q_put(queue); 2363 return -ENOENT; 2364 } 2365 2366 deferfcp = list_first_entry_or_null(&queue->avail_defer_list, 2367 struct nvmet_fc_defer_fcp_req, req_list); 2368 if (deferfcp) { 2369 /* Just re-use one that was previously allocated */ 2370 list_del(&deferfcp->req_list); 2371 } else { 2372 spin_unlock_irqrestore(&queue->qlock, flags); 2373 2374 /* Now we need to dynamically allocate one */ 2375 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL); 2376 if (!deferfcp) { 2377 /* release the queue lookup reference */ 2378 nvmet_fc_tgt_q_put(queue); 2379 return -ENOMEM; 2380 } 2381 spin_lock_irqsave(&queue->qlock, flags); 2382 } 2383 2384 /* For now, use rspaddr / rsplen to save payload information */ 2385 fcpreq->rspaddr = cmdiubuf; 2386 fcpreq->rsplen = cmdiubuf_len; 2387 deferfcp->fcp_req = fcpreq; 2388 2389 /* defer processing till a fod becomes available */ 2390 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list); 2391 2392 /* NOTE: the queue lookup reference is still valid */ 2393 2394 spin_unlock_irqrestore(&queue->qlock, flags); 2395 2396 return -EOVERFLOW; 2397 } 2398 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req); 2399 2400 /** 2401 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD 2402 * upon the reception of an ABTS for a FCP command 2403 * 2404 * Notify the transport that an ABTS has been received for a FCP command 2405 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The 2406 * LLDD believes the command is still being worked on 2407 * (template_ops->fcp_req_release() has not been called). 2408 * 2409 * The transport will wait for any outstanding work (an op to the LLDD, 2410 * which the lldd should complete with error due to the ABTS; or the 2411 * completion from the nvmet layer of the nvme command), then will 2412 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to 2413 * return the i/o context to the LLDD. The LLDD may send the BA_ACC 2414 * to the ABTS either after return from this function (assuming any 2415 * outstanding op work has been terminated) or upon the callback being 2416 * called. 2417 * 2418 * @target_port: pointer to the (registered) target port the FCP CMD IU 2419 * was received on. 2420 * @fcpreq: pointer to the fcpreq request structure that corresponds 2421 * to the exchange that received the ABTS. 2422 */ 2423 void 2424 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port, 2425 struct nvmefc_tgt_fcp_req *fcpreq) 2426 { 2427 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 2428 struct nvmet_fc_tgt_queue *queue; 2429 unsigned long flags; 2430 2431 if (!fod || fod->fcpreq != fcpreq) 2432 /* job appears to have already completed, ignore abort */ 2433 return; 2434 2435 queue = fod->queue; 2436 2437 spin_lock_irqsave(&queue->qlock, flags); 2438 if (fod->active) { 2439 /* 2440 * mark as abort. The abort handler, invoked upon completion 2441 * of any work, will detect the aborted status and do the 2442 * callback. 2443 */ 2444 spin_lock(&fod->flock); 2445 fod->abort = true; 2446 fod->aborted = true; 2447 spin_unlock(&fod->flock); 2448 } 2449 spin_unlock_irqrestore(&queue->qlock, flags); 2450 } 2451 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort); 2452 2453 2454 struct nvmet_fc_traddr { 2455 u64 nn; 2456 u64 pn; 2457 }; 2458 2459 static int 2460 __nvme_fc_parse_u64(substring_t *sstr, u64 *val) 2461 { 2462 u64 token64; 2463 2464 if (match_u64(sstr, &token64)) 2465 return -EINVAL; 2466 *val = token64; 2467 2468 return 0; 2469 } 2470 2471 /* 2472 * This routine validates and extracts the WWN's from the TRADDR string. 2473 * As kernel parsers need the 0x to determine number base, universally 2474 * build string to parse with 0x prefix before parsing name strings. 2475 */ 2476 static int 2477 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen) 2478 { 2479 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1]; 2480 substring_t wwn = { name, &name[sizeof(name)-1] }; 2481 int nnoffset, pnoffset; 2482 2483 /* validate it string one of the 2 allowed formats */ 2484 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH && 2485 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) && 2486 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET], 2487 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) { 2488 nnoffset = NVME_FC_TRADDR_OXNNLEN; 2489 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET + 2490 NVME_FC_TRADDR_OXNNLEN; 2491 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH && 2492 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) && 2493 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET], 2494 "pn-", NVME_FC_TRADDR_NNLEN))) { 2495 nnoffset = NVME_FC_TRADDR_NNLEN; 2496 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN; 2497 } else 2498 goto out_einval; 2499 2500 name[0] = '0'; 2501 name[1] = 'x'; 2502 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0; 2503 2504 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN); 2505 if (__nvme_fc_parse_u64(&wwn, &traddr->nn)) 2506 goto out_einval; 2507 2508 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN); 2509 if (__nvme_fc_parse_u64(&wwn, &traddr->pn)) 2510 goto out_einval; 2511 2512 return 0; 2513 2514 out_einval: 2515 pr_warn("%s: bad traddr string\n", __func__); 2516 return -EINVAL; 2517 } 2518 2519 static int 2520 nvmet_fc_add_port(struct nvmet_port *port) 2521 { 2522 struct nvmet_fc_tgtport *tgtport; 2523 struct nvmet_fc_traddr traddr = { 0L, 0L }; 2524 unsigned long flags; 2525 int ret; 2526 2527 /* validate the address info */ 2528 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) || 2529 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC)) 2530 return -EINVAL; 2531 2532 /* map the traddr address info to a target port */ 2533 2534 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr, 2535 sizeof(port->disc_addr.traddr)); 2536 if (ret) 2537 return ret; 2538 2539 ret = -ENXIO; 2540 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 2541 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) { 2542 if ((tgtport->fc_target_port.node_name == traddr.nn) && 2543 (tgtport->fc_target_port.port_name == traddr.pn)) { 2544 /* a FC port can only be 1 nvmet port id */ 2545 if (!tgtport->port) { 2546 tgtport->port = port; 2547 port->priv = tgtport; 2548 nvmet_fc_tgtport_get(tgtport); 2549 ret = 0; 2550 } else 2551 ret = -EALREADY; 2552 break; 2553 } 2554 } 2555 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 2556 return ret; 2557 } 2558 2559 static void 2560 nvmet_fc_remove_port(struct nvmet_port *port) 2561 { 2562 struct nvmet_fc_tgtport *tgtport = port->priv; 2563 unsigned long flags; 2564 2565 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 2566 if (tgtport->port == port) { 2567 nvmet_fc_tgtport_put(tgtport); 2568 tgtport->port = NULL; 2569 } 2570 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 2571 } 2572 2573 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = { 2574 .owner = THIS_MODULE, 2575 .type = NVMF_TRTYPE_FC, 2576 .msdbd = 1, 2577 .add_port = nvmet_fc_add_port, 2578 .remove_port = nvmet_fc_remove_port, 2579 .queue_response = nvmet_fc_fcp_nvme_cmd_done, 2580 .delete_ctrl = nvmet_fc_delete_ctrl, 2581 }; 2582 2583 static int __init nvmet_fc_init_module(void) 2584 { 2585 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops); 2586 } 2587 2588 static void __exit nvmet_fc_exit_module(void) 2589 { 2590 /* sanity check - all lports should be removed */ 2591 if (!list_empty(&nvmet_fc_target_list)) 2592 pr_warn("%s: targetport list not empty\n", __func__); 2593 2594 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops); 2595 2596 ida_destroy(&nvmet_fc_tgtport_cnt); 2597 } 2598 2599 module_init(nvmet_fc_init_module); 2600 module_exit(nvmet_fc_exit_module); 2601 2602 MODULE_LICENSE("GPL v2"); 2603