1 /* 2 * Copyright (c) 2016 Avago Technologies. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of version 2 of the GNU General Public License as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful. 9 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, 10 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A 11 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO 12 * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID. 13 * See the GNU General Public License for more details, a copy of which 14 * can be found in the file COPYING included with this package 15 * 16 */ 17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 18 #include <linux/module.h> 19 #include <linux/slab.h> 20 #include <linux/blk-mq.h> 21 #include <linux/parser.h> 22 #include <linux/random.h> 23 #include <uapi/scsi/fc/fc_fs.h> 24 #include <uapi/scsi/fc/fc_els.h> 25 26 #include "nvmet.h" 27 #include <linux/nvme-fc-driver.h> 28 #include <linux/nvme-fc.h> 29 30 31 /* *************************** Data Structures/Defines ****************** */ 32 33 34 #define NVMET_LS_CTX_COUNT 4 35 36 /* for this implementation, assume small single frame rqst/rsp */ 37 #define NVME_FC_MAX_LS_BUFFER_SIZE 2048 38 39 struct nvmet_fc_tgtport; 40 struct nvmet_fc_tgt_assoc; 41 42 struct nvmet_fc_ls_iod { 43 struct nvmefc_tgt_ls_req *lsreq; 44 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */ 45 46 struct list_head ls_list; /* tgtport->ls_list */ 47 48 struct nvmet_fc_tgtport *tgtport; 49 struct nvmet_fc_tgt_assoc *assoc; 50 51 u8 *rqstbuf; 52 u8 *rspbuf; 53 u16 rqstdatalen; 54 dma_addr_t rspdma; 55 56 struct scatterlist sg[2]; 57 58 struct work_struct work; 59 } __aligned(sizeof(unsigned long long)); 60 61 #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024) 62 #define NVMET_FC_MAX_XFR_SGENTS (NVMET_FC_MAX_SEQ_LENGTH / PAGE_SIZE) 63 64 enum nvmet_fcp_datadir { 65 NVMET_FCP_NODATA, 66 NVMET_FCP_WRITE, 67 NVMET_FCP_READ, 68 NVMET_FCP_ABORTED, 69 }; 70 71 struct nvmet_fc_fcp_iod { 72 struct nvmefc_tgt_fcp_req *fcpreq; 73 74 struct nvme_fc_cmd_iu cmdiubuf; 75 struct nvme_fc_ersp_iu rspiubuf; 76 dma_addr_t rspdma; 77 struct scatterlist *data_sg; 78 int data_sg_cnt; 79 u32 offset; 80 enum nvmet_fcp_datadir io_dir; 81 bool active; 82 bool abort; 83 bool aborted; 84 bool writedataactive; 85 spinlock_t flock; 86 87 struct nvmet_req req; 88 struct work_struct work; 89 struct work_struct done_work; 90 91 struct nvmet_fc_tgtport *tgtport; 92 struct nvmet_fc_tgt_queue *queue; 93 94 struct list_head fcp_list; /* tgtport->fcp_list */ 95 }; 96 97 struct nvmet_fc_tgtport { 98 99 struct nvmet_fc_target_port fc_target_port; 100 101 struct list_head tgt_list; /* nvmet_fc_target_list */ 102 struct device *dev; /* dev for dma mapping */ 103 struct nvmet_fc_target_template *ops; 104 105 struct nvmet_fc_ls_iod *iod; 106 spinlock_t lock; 107 struct list_head ls_list; 108 struct list_head ls_busylist; 109 struct list_head assoc_list; 110 struct ida assoc_cnt; 111 struct nvmet_port *port; 112 struct kref ref; 113 u32 max_sg_cnt; 114 }; 115 116 struct nvmet_fc_defer_fcp_req { 117 struct list_head req_list; 118 struct nvmefc_tgt_fcp_req *fcp_req; 119 }; 120 121 struct nvmet_fc_tgt_queue { 122 bool ninetypercent; 123 u16 qid; 124 u16 sqsize; 125 u16 ersp_ratio; 126 __le16 sqhd; 127 int cpu; 128 atomic_t connected; 129 atomic_t sqtail; 130 atomic_t zrspcnt; 131 atomic_t rsn; 132 spinlock_t qlock; 133 struct nvmet_port *port; 134 struct nvmet_cq nvme_cq; 135 struct nvmet_sq nvme_sq; 136 struct nvmet_fc_tgt_assoc *assoc; 137 struct nvmet_fc_fcp_iod *fod; /* array of fcp_iods */ 138 struct list_head fod_list; 139 struct list_head pending_cmd_list; 140 struct list_head avail_defer_list; 141 struct workqueue_struct *work_q; 142 struct kref ref; 143 } __aligned(sizeof(unsigned long long)); 144 145 struct nvmet_fc_tgt_assoc { 146 u64 association_id; 147 u32 a_id; 148 struct nvmet_fc_tgtport *tgtport; 149 struct list_head a_list; 150 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1]; 151 struct kref ref; 152 struct work_struct del_work; 153 }; 154 155 156 static inline int 157 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr) 158 { 159 return (iodptr - iodptr->tgtport->iod); 160 } 161 162 static inline int 163 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr) 164 { 165 return (fodptr - fodptr->queue->fod); 166 } 167 168 169 /* 170 * Association and Connection IDs: 171 * 172 * Association ID will have random number in upper 6 bytes and zero 173 * in lower 2 bytes 174 * 175 * Connection IDs will be Association ID with QID or'd in lower 2 bytes 176 * 177 * note: Association ID = Connection ID for queue 0 178 */ 179 #define BYTES_FOR_QID sizeof(u16) 180 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8) 181 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1)) 182 183 static inline u64 184 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid) 185 { 186 return (assoc->association_id | qid); 187 } 188 189 static inline u64 190 nvmet_fc_getassociationid(u64 connectionid) 191 { 192 return connectionid & ~NVMET_FC_QUEUEID_MASK; 193 } 194 195 static inline u16 196 nvmet_fc_getqueueid(u64 connectionid) 197 { 198 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK); 199 } 200 201 static inline struct nvmet_fc_tgtport * 202 targetport_to_tgtport(struct nvmet_fc_target_port *targetport) 203 { 204 return container_of(targetport, struct nvmet_fc_tgtport, 205 fc_target_port); 206 } 207 208 static inline struct nvmet_fc_fcp_iod * 209 nvmet_req_to_fod(struct nvmet_req *nvme_req) 210 { 211 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req); 212 } 213 214 215 /* *************************** Globals **************************** */ 216 217 218 static DEFINE_SPINLOCK(nvmet_fc_tgtlock); 219 220 static LIST_HEAD(nvmet_fc_target_list); 221 static DEFINE_IDA(nvmet_fc_tgtport_cnt); 222 223 224 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work); 225 static void nvmet_fc_handle_fcp_rqst_work(struct work_struct *work); 226 static void nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work); 227 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc); 228 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc); 229 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue); 230 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue); 231 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport); 232 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport); 233 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, 234 struct nvmet_fc_fcp_iod *fod); 235 static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc); 236 237 238 /* *********************** FC-NVME DMA Handling **************************** */ 239 240 /* 241 * The fcloop device passes in a NULL device pointer. Real LLD's will 242 * pass in a valid device pointer. If NULL is passed to the dma mapping 243 * routines, depending on the platform, it may or may not succeed, and 244 * may crash. 245 * 246 * As such: 247 * Wrapper all the dma routines and check the dev pointer. 248 * 249 * If simple mappings (return just a dma address, we'll noop them, 250 * returning a dma address of 0. 251 * 252 * On more complex mappings (dma_map_sg), a pseudo routine fills 253 * in the scatter list, setting all dma addresses to 0. 254 */ 255 256 static inline dma_addr_t 257 fc_dma_map_single(struct device *dev, void *ptr, size_t size, 258 enum dma_data_direction dir) 259 { 260 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L; 261 } 262 263 static inline int 264 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 265 { 266 return dev ? dma_mapping_error(dev, dma_addr) : 0; 267 } 268 269 static inline void 270 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, 271 enum dma_data_direction dir) 272 { 273 if (dev) 274 dma_unmap_single(dev, addr, size, dir); 275 } 276 277 static inline void 278 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 279 enum dma_data_direction dir) 280 { 281 if (dev) 282 dma_sync_single_for_cpu(dev, addr, size, dir); 283 } 284 285 static inline void 286 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size, 287 enum dma_data_direction dir) 288 { 289 if (dev) 290 dma_sync_single_for_device(dev, addr, size, dir); 291 } 292 293 /* pseudo dma_map_sg call */ 294 static int 295 fc_map_sg(struct scatterlist *sg, int nents) 296 { 297 struct scatterlist *s; 298 int i; 299 300 WARN_ON(nents == 0 || sg[0].length == 0); 301 302 for_each_sg(sg, s, nents, i) { 303 s->dma_address = 0L; 304 #ifdef CONFIG_NEED_SG_DMA_LENGTH 305 s->dma_length = s->length; 306 #endif 307 } 308 return nents; 309 } 310 311 static inline int 312 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 313 enum dma_data_direction dir) 314 { 315 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents); 316 } 317 318 static inline void 319 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 320 enum dma_data_direction dir) 321 { 322 if (dev) 323 dma_unmap_sg(dev, sg, nents, dir); 324 } 325 326 327 /* *********************** FC-NVME Port Management ************************ */ 328 329 330 static int 331 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) 332 { 333 struct nvmet_fc_ls_iod *iod; 334 int i; 335 336 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod), 337 GFP_KERNEL); 338 if (!iod) 339 return -ENOMEM; 340 341 tgtport->iod = iod; 342 343 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { 344 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work); 345 iod->tgtport = tgtport; 346 list_add_tail(&iod->ls_list, &tgtport->ls_list); 347 348 iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE, 349 GFP_KERNEL); 350 if (!iod->rqstbuf) 351 goto out_fail; 352 353 iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE; 354 355 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf, 356 NVME_FC_MAX_LS_BUFFER_SIZE, 357 DMA_TO_DEVICE); 358 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma)) 359 goto out_fail; 360 } 361 362 return 0; 363 364 out_fail: 365 kfree(iod->rqstbuf); 366 list_del(&iod->ls_list); 367 for (iod--, i--; i >= 0; iod--, i--) { 368 fc_dma_unmap_single(tgtport->dev, iod->rspdma, 369 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); 370 kfree(iod->rqstbuf); 371 list_del(&iod->ls_list); 372 } 373 374 kfree(iod); 375 376 return -EFAULT; 377 } 378 379 static void 380 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport) 381 { 382 struct nvmet_fc_ls_iod *iod = tgtport->iod; 383 int i; 384 385 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) { 386 fc_dma_unmap_single(tgtport->dev, 387 iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE, 388 DMA_TO_DEVICE); 389 kfree(iod->rqstbuf); 390 list_del(&iod->ls_list); 391 } 392 kfree(tgtport->iod); 393 } 394 395 static struct nvmet_fc_ls_iod * 396 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport) 397 { 398 struct nvmet_fc_ls_iod *iod; 399 unsigned long flags; 400 401 spin_lock_irqsave(&tgtport->lock, flags); 402 iod = list_first_entry_or_null(&tgtport->ls_list, 403 struct nvmet_fc_ls_iod, ls_list); 404 if (iod) 405 list_move_tail(&iod->ls_list, &tgtport->ls_busylist); 406 spin_unlock_irqrestore(&tgtport->lock, flags); 407 return iod; 408 } 409 410 411 static void 412 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport, 413 struct nvmet_fc_ls_iod *iod) 414 { 415 unsigned long flags; 416 417 spin_lock_irqsave(&tgtport->lock, flags); 418 list_move(&iod->ls_list, &tgtport->ls_list); 419 spin_unlock_irqrestore(&tgtport->lock, flags); 420 } 421 422 static void 423 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport, 424 struct nvmet_fc_tgt_queue *queue) 425 { 426 struct nvmet_fc_fcp_iod *fod = queue->fod; 427 int i; 428 429 for (i = 0; i < queue->sqsize; fod++, i++) { 430 INIT_WORK(&fod->work, nvmet_fc_handle_fcp_rqst_work); 431 INIT_WORK(&fod->done_work, nvmet_fc_fcp_rqst_op_done_work); 432 fod->tgtport = tgtport; 433 fod->queue = queue; 434 fod->active = false; 435 fod->abort = false; 436 fod->aborted = false; 437 fod->fcpreq = NULL; 438 list_add_tail(&fod->fcp_list, &queue->fod_list); 439 spin_lock_init(&fod->flock); 440 441 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf, 442 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 443 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) { 444 list_del(&fod->fcp_list); 445 for (fod--, i--; i >= 0; fod--, i--) { 446 fc_dma_unmap_single(tgtport->dev, fod->rspdma, 447 sizeof(fod->rspiubuf), 448 DMA_TO_DEVICE); 449 fod->rspdma = 0L; 450 list_del(&fod->fcp_list); 451 } 452 453 return; 454 } 455 } 456 } 457 458 static void 459 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport, 460 struct nvmet_fc_tgt_queue *queue) 461 { 462 struct nvmet_fc_fcp_iod *fod = queue->fod; 463 int i; 464 465 for (i = 0; i < queue->sqsize; fod++, i++) { 466 if (fod->rspdma) 467 fc_dma_unmap_single(tgtport->dev, fod->rspdma, 468 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 469 } 470 } 471 472 static struct nvmet_fc_fcp_iod * 473 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue) 474 { 475 struct nvmet_fc_fcp_iod *fod; 476 477 lockdep_assert_held(&queue->qlock); 478 479 fod = list_first_entry_or_null(&queue->fod_list, 480 struct nvmet_fc_fcp_iod, fcp_list); 481 if (fod) { 482 list_del(&fod->fcp_list); 483 fod->active = true; 484 /* 485 * no queue reference is taken, as it was taken by the 486 * queue lookup just prior to the allocation. The iod 487 * will "inherit" that reference. 488 */ 489 } 490 return fod; 491 } 492 493 494 static void 495 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport, 496 struct nvmet_fc_tgt_queue *queue, 497 struct nvmefc_tgt_fcp_req *fcpreq) 498 { 499 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 500 501 /* 502 * put all admin cmds on hw queue id 0. All io commands go to 503 * the respective hw queue based on a modulo basis 504 */ 505 fcpreq->hwqid = queue->qid ? 506 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0; 507 508 if (tgtport->ops->target_features & NVMET_FCTGTFEAT_CMD_IN_ISR) 509 queue_work_on(queue->cpu, queue->work_q, &fod->work); 510 else 511 nvmet_fc_handle_fcp_rqst(tgtport, fod); 512 } 513 514 static void 515 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue, 516 struct nvmet_fc_fcp_iod *fod) 517 { 518 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 519 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 520 struct nvmet_fc_defer_fcp_req *deferfcp; 521 unsigned long flags; 522 523 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma, 524 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 525 526 fcpreq->nvmet_fc_private = NULL; 527 528 fod->active = false; 529 fod->abort = false; 530 fod->aborted = false; 531 fod->writedataactive = false; 532 fod->fcpreq = NULL; 533 534 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq); 535 536 spin_lock_irqsave(&queue->qlock, flags); 537 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list, 538 struct nvmet_fc_defer_fcp_req, req_list); 539 if (!deferfcp) { 540 list_add_tail(&fod->fcp_list, &fod->queue->fod_list); 541 spin_unlock_irqrestore(&queue->qlock, flags); 542 543 /* Release reference taken at queue lookup and fod allocation */ 544 nvmet_fc_tgt_q_put(queue); 545 return; 546 } 547 548 /* Re-use the fod for the next pending cmd that was deferred */ 549 list_del(&deferfcp->req_list); 550 551 fcpreq = deferfcp->fcp_req; 552 553 /* deferfcp can be reused for another IO at a later date */ 554 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list); 555 556 spin_unlock_irqrestore(&queue->qlock, flags); 557 558 /* Save NVME CMD IO in fod */ 559 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen); 560 561 /* Setup new fcpreq to be processed */ 562 fcpreq->rspaddr = NULL; 563 fcpreq->rsplen = 0; 564 fcpreq->nvmet_fc_private = fod; 565 fod->fcpreq = fcpreq; 566 fod->active = true; 567 568 /* inform LLDD IO is now being processed */ 569 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq); 570 571 /* Submit deferred IO for processing */ 572 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq); 573 574 /* 575 * Leave the queue lookup get reference taken when 576 * fod was originally allocated. 577 */ 578 } 579 580 static int 581 nvmet_fc_queue_to_cpu(struct nvmet_fc_tgtport *tgtport, int qid) 582 { 583 int cpu, idx, cnt; 584 585 if (tgtport->ops->max_hw_queues == 1) 586 return WORK_CPU_UNBOUND; 587 588 /* Simple cpu selection based on qid modulo active cpu count */ 589 idx = !qid ? 0 : (qid - 1) % num_active_cpus(); 590 591 /* find the n'th active cpu */ 592 for (cpu = 0, cnt = 0; ; ) { 593 if (cpu_active(cpu)) { 594 if (cnt == idx) 595 break; 596 cnt++; 597 } 598 cpu = (cpu + 1) % num_possible_cpus(); 599 } 600 601 return cpu; 602 } 603 604 static struct nvmet_fc_tgt_queue * 605 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc, 606 u16 qid, u16 sqsize) 607 { 608 struct nvmet_fc_tgt_queue *queue; 609 unsigned long flags; 610 int ret; 611 612 if (qid > NVMET_NR_QUEUES) 613 return NULL; 614 615 queue = kzalloc((sizeof(*queue) + 616 (sizeof(struct nvmet_fc_fcp_iod) * sqsize)), 617 GFP_KERNEL); 618 if (!queue) 619 return NULL; 620 621 if (!nvmet_fc_tgt_a_get(assoc)) 622 goto out_free_queue; 623 624 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0, 625 assoc->tgtport->fc_target_port.port_num, 626 assoc->a_id, qid); 627 if (!queue->work_q) 628 goto out_a_put; 629 630 queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1]; 631 queue->qid = qid; 632 queue->sqsize = sqsize; 633 queue->assoc = assoc; 634 queue->port = assoc->tgtport->port; 635 queue->cpu = nvmet_fc_queue_to_cpu(assoc->tgtport, qid); 636 INIT_LIST_HEAD(&queue->fod_list); 637 INIT_LIST_HEAD(&queue->avail_defer_list); 638 INIT_LIST_HEAD(&queue->pending_cmd_list); 639 atomic_set(&queue->connected, 0); 640 atomic_set(&queue->sqtail, 0); 641 atomic_set(&queue->rsn, 1); 642 atomic_set(&queue->zrspcnt, 0); 643 spin_lock_init(&queue->qlock); 644 kref_init(&queue->ref); 645 646 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue); 647 648 ret = nvmet_sq_init(&queue->nvme_sq); 649 if (ret) 650 goto out_fail_iodlist; 651 652 WARN_ON(assoc->queues[qid]); 653 spin_lock_irqsave(&assoc->tgtport->lock, flags); 654 assoc->queues[qid] = queue; 655 spin_unlock_irqrestore(&assoc->tgtport->lock, flags); 656 657 return queue; 658 659 out_fail_iodlist: 660 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue); 661 destroy_workqueue(queue->work_q); 662 out_a_put: 663 nvmet_fc_tgt_a_put(assoc); 664 out_free_queue: 665 kfree(queue); 666 return NULL; 667 } 668 669 670 static void 671 nvmet_fc_tgt_queue_free(struct kref *ref) 672 { 673 struct nvmet_fc_tgt_queue *queue = 674 container_of(ref, struct nvmet_fc_tgt_queue, ref); 675 unsigned long flags; 676 677 spin_lock_irqsave(&queue->assoc->tgtport->lock, flags); 678 queue->assoc->queues[queue->qid] = NULL; 679 spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags); 680 681 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue); 682 683 nvmet_fc_tgt_a_put(queue->assoc); 684 685 destroy_workqueue(queue->work_q); 686 687 kfree(queue); 688 } 689 690 static void 691 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue) 692 { 693 kref_put(&queue->ref, nvmet_fc_tgt_queue_free); 694 } 695 696 static int 697 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue) 698 { 699 return kref_get_unless_zero(&queue->ref); 700 } 701 702 703 static void 704 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue) 705 { 706 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport; 707 struct nvmet_fc_fcp_iod *fod = queue->fod; 708 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr; 709 unsigned long flags; 710 int i, writedataactive; 711 bool disconnect; 712 713 disconnect = atomic_xchg(&queue->connected, 0); 714 715 spin_lock_irqsave(&queue->qlock, flags); 716 /* about outstanding io's */ 717 for (i = 0; i < queue->sqsize; fod++, i++) { 718 if (fod->active) { 719 spin_lock(&fod->flock); 720 fod->abort = true; 721 writedataactive = fod->writedataactive; 722 spin_unlock(&fod->flock); 723 /* 724 * only call lldd abort routine if waiting for 725 * writedata. other outstanding ops should finish 726 * on their own. 727 */ 728 if (writedataactive) { 729 spin_lock(&fod->flock); 730 fod->aborted = true; 731 spin_unlock(&fod->flock); 732 tgtport->ops->fcp_abort( 733 &tgtport->fc_target_port, fod->fcpreq); 734 } 735 } 736 } 737 738 /* Cleanup defer'ed IOs in queue */ 739 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list, 740 req_list) { 741 list_del(&deferfcp->req_list); 742 kfree(deferfcp); 743 } 744 745 for (;;) { 746 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list, 747 struct nvmet_fc_defer_fcp_req, req_list); 748 if (!deferfcp) 749 break; 750 751 list_del(&deferfcp->req_list); 752 spin_unlock_irqrestore(&queue->qlock, flags); 753 754 tgtport->ops->defer_rcv(&tgtport->fc_target_port, 755 deferfcp->fcp_req); 756 757 tgtport->ops->fcp_abort(&tgtport->fc_target_port, 758 deferfcp->fcp_req); 759 760 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, 761 deferfcp->fcp_req); 762 763 kfree(deferfcp); 764 765 spin_lock_irqsave(&queue->qlock, flags); 766 } 767 spin_unlock_irqrestore(&queue->qlock, flags); 768 769 flush_workqueue(queue->work_q); 770 771 if (disconnect) 772 nvmet_sq_destroy(&queue->nvme_sq); 773 774 nvmet_fc_tgt_q_put(queue); 775 } 776 777 static struct nvmet_fc_tgt_queue * 778 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport, 779 u64 connection_id) 780 { 781 struct nvmet_fc_tgt_assoc *assoc; 782 struct nvmet_fc_tgt_queue *queue; 783 u64 association_id = nvmet_fc_getassociationid(connection_id); 784 u16 qid = nvmet_fc_getqueueid(connection_id); 785 unsigned long flags; 786 787 if (qid > NVMET_NR_QUEUES) 788 return NULL; 789 790 spin_lock_irqsave(&tgtport->lock, flags); 791 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) { 792 if (association_id == assoc->association_id) { 793 queue = assoc->queues[qid]; 794 if (queue && 795 (!atomic_read(&queue->connected) || 796 !nvmet_fc_tgt_q_get(queue))) 797 queue = NULL; 798 spin_unlock_irqrestore(&tgtport->lock, flags); 799 return queue; 800 } 801 } 802 spin_unlock_irqrestore(&tgtport->lock, flags); 803 return NULL; 804 } 805 806 static void 807 nvmet_fc_delete_assoc(struct work_struct *work) 808 { 809 struct nvmet_fc_tgt_assoc *assoc = 810 container_of(work, struct nvmet_fc_tgt_assoc, del_work); 811 812 nvmet_fc_delete_target_assoc(assoc); 813 nvmet_fc_tgt_a_put(assoc); 814 } 815 816 static struct nvmet_fc_tgt_assoc * 817 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport) 818 { 819 struct nvmet_fc_tgt_assoc *assoc, *tmpassoc; 820 unsigned long flags; 821 u64 ran; 822 int idx; 823 bool needrandom = true; 824 825 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL); 826 if (!assoc) 827 return NULL; 828 829 idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL); 830 if (idx < 0) 831 goto out_free_assoc; 832 833 if (!nvmet_fc_tgtport_get(tgtport)) 834 goto out_ida_put; 835 836 assoc->tgtport = tgtport; 837 assoc->a_id = idx; 838 INIT_LIST_HEAD(&assoc->a_list); 839 kref_init(&assoc->ref); 840 INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc); 841 842 while (needrandom) { 843 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID); 844 ran = ran << BYTES_FOR_QID_SHIFT; 845 846 spin_lock_irqsave(&tgtport->lock, flags); 847 needrandom = false; 848 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list) 849 if (ran == tmpassoc->association_id) { 850 needrandom = true; 851 break; 852 } 853 if (!needrandom) { 854 assoc->association_id = ran; 855 list_add_tail(&assoc->a_list, &tgtport->assoc_list); 856 } 857 spin_unlock_irqrestore(&tgtport->lock, flags); 858 } 859 860 return assoc; 861 862 out_ida_put: 863 ida_simple_remove(&tgtport->assoc_cnt, idx); 864 out_free_assoc: 865 kfree(assoc); 866 return NULL; 867 } 868 869 static void 870 nvmet_fc_target_assoc_free(struct kref *ref) 871 { 872 struct nvmet_fc_tgt_assoc *assoc = 873 container_of(ref, struct nvmet_fc_tgt_assoc, ref); 874 struct nvmet_fc_tgtport *tgtport = assoc->tgtport; 875 unsigned long flags; 876 877 spin_lock_irqsave(&tgtport->lock, flags); 878 list_del(&assoc->a_list); 879 spin_unlock_irqrestore(&tgtport->lock, flags); 880 ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id); 881 kfree(assoc); 882 nvmet_fc_tgtport_put(tgtport); 883 } 884 885 static void 886 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc) 887 { 888 kref_put(&assoc->ref, nvmet_fc_target_assoc_free); 889 } 890 891 static int 892 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc) 893 { 894 return kref_get_unless_zero(&assoc->ref); 895 } 896 897 static void 898 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc) 899 { 900 struct nvmet_fc_tgtport *tgtport = assoc->tgtport; 901 struct nvmet_fc_tgt_queue *queue; 902 unsigned long flags; 903 int i; 904 905 spin_lock_irqsave(&tgtport->lock, flags); 906 for (i = NVMET_NR_QUEUES; i >= 0; i--) { 907 queue = assoc->queues[i]; 908 if (queue) { 909 if (!nvmet_fc_tgt_q_get(queue)) 910 continue; 911 spin_unlock_irqrestore(&tgtport->lock, flags); 912 nvmet_fc_delete_target_queue(queue); 913 nvmet_fc_tgt_q_put(queue); 914 spin_lock_irqsave(&tgtport->lock, flags); 915 } 916 } 917 spin_unlock_irqrestore(&tgtport->lock, flags); 918 919 nvmet_fc_tgt_a_put(assoc); 920 } 921 922 static struct nvmet_fc_tgt_assoc * 923 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport, 924 u64 association_id) 925 { 926 struct nvmet_fc_tgt_assoc *assoc; 927 struct nvmet_fc_tgt_assoc *ret = NULL; 928 unsigned long flags; 929 930 spin_lock_irqsave(&tgtport->lock, flags); 931 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) { 932 if (association_id == assoc->association_id) { 933 ret = assoc; 934 nvmet_fc_tgt_a_get(assoc); 935 break; 936 } 937 } 938 spin_unlock_irqrestore(&tgtport->lock, flags); 939 940 return ret; 941 } 942 943 944 /** 945 * nvme_fc_register_targetport - transport entry point called by an 946 * LLDD to register the existence of a local 947 * NVME subystem FC port. 948 * @pinfo: pointer to information about the port to be registered 949 * @template: LLDD entrypoints and operational parameters for the port 950 * @dev: physical hardware device node port corresponds to. Will be 951 * used for DMA mappings 952 * @portptr: pointer to a local port pointer. Upon success, the routine 953 * will allocate a nvme_fc_local_port structure and place its 954 * address in the local port pointer. Upon failure, local port 955 * pointer will be set to NULL. 956 * 957 * Returns: 958 * a completion status. Must be 0 upon success; a negative errno 959 * (ex: -ENXIO) upon failure. 960 */ 961 int 962 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo, 963 struct nvmet_fc_target_template *template, 964 struct device *dev, 965 struct nvmet_fc_target_port **portptr) 966 { 967 struct nvmet_fc_tgtport *newrec; 968 unsigned long flags; 969 int ret, idx; 970 971 if (!template->xmt_ls_rsp || !template->fcp_op || 972 !template->fcp_abort || 973 !template->fcp_req_release || !template->targetport_delete || 974 !template->max_hw_queues || !template->max_sgl_segments || 975 !template->max_dif_sgl_segments || !template->dma_boundary) { 976 ret = -EINVAL; 977 goto out_regtgt_failed; 978 } 979 980 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz), 981 GFP_KERNEL); 982 if (!newrec) { 983 ret = -ENOMEM; 984 goto out_regtgt_failed; 985 } 986 987 idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL); 988 if (idx < 0) { 989 ret = -ENOSPC; 990 goto out_fail_kfree; 991 } 992 993 if (!get_device(dev) && dev) { 994 ret = -ENODEV; 995 goto out_ida_put; 996 } 997 998 newrec->fc_target_port.node_name = pinfo->node_name; 999 newrec->fc_target_port.port_name = pinfo->port_name; 1000 newrec->fc_target_port.private = &newrec[1]; 1001 newrec->fc_target_port.port_id = pinfo->port_id; 1002 newrec->fc_target_port.port_num = idx; 1003 INIT_LIST_HEAD(&newrec->tgt_list); 1004 newrec->dev = dev; 1005 newrec->ops = template; 1006 spin_lock_init(&newrec->lock); 1007 INIT_LIST_HEAD(&newrec->ls_list); 1008 INIT_LIST_HEAD(&newrec->ls_busylist); 1009 INIT_LIST_HEAD(&newrec->assoc_list); 1010 kref_init(&newrec->ref); 1011 ida_init(&newrec->assoc_cnt); 1012 newrec->max_sg_cnt = min_t(u32, NVMET_FC_MAX_XFR_SGENTS, 1013 template->max_sgl_segments); 1014 1015 ret = nvmet_fc_alloc_ls_iodlist(newrec); 1016 if (ret) { 1017 ret = -ENOMEM; 1018 goto out_free_newrec; 1019 } 1020 1021 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1022 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list); 1023 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1024 1025 *portptr = &newrec->fc_target_port; 1026 return 0; 1027 1028 out_free_newrec: 1029 put_device(dev); 1030 out_ida_put: 1031 ida_simple_remove(&nvmet_fc_tgtport_cnt, idx); 1032 out_fail_kfree: 1033 kfree(newrec); 1034 out_regtgt_failed: 1035 *portptr = NULL; 1036 return ret; 1037 } 1038 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport); 1039 1040 1041 static void 1042 nvmet_fc_free_tgtport(struct kref *ref) 1043 { 1044 struct nvmet_fc_tgtport *tgtport = 1045 container_of(ref, struct nvmet_fc_tgtport, ref); 1046 struct device *dev = tgtport->dev; 1047 unsigned long flags; 1048 1049 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1050 list_del(&tgtport->tgt_list); 1051 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1052 1053 nvmet_fc_free_ls_iodlist(tgtport); 1054 1055 /* let the LLDD know we've finished tearing it down */ 1056 tgtport->ops->targetport_delete(&tgtport->fc_target_port); 1057 1058 ida_simple_remove(&nvmet_fc_tgtport_cnt, 1059 tgtport->fc_target_port.port_num); 1060 1061 ida_destroy(&tgtport->assoc_cnt); 1062 1063 kfree(tgtport); 1064 1065 put_device(dev); 1066 } 1067 1068 static void 1069 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport) 1070 { 1071 kref_put(&tgtport->ref, nvmet_fc_free_tgtport); 1072 } 1073 1074 static int 1075 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport) 1076 { 1077 return kref_get_unless_zero(&tgtport->ref); 1078 } 1079 1080 static void 1081 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport) 1082 { 1083 struct nvmet_fc_tgt_assoc *assoc, *next; 1084 unsigned long flags; 1085 1086 spin_lock_irqsave(&tgtport->lock, flags); 1087 list_for_each_entry_safe(assoc, next, 1088 &tgtport->assoc_list, a_list) { 1089 if (!nvmet_fc_tgt_a_get(assoc)) 1090 continue; 1091 spin_unlock_irqrestore(&tgtport->lock, flags); 1092 nvmet_fc_delete_target_assoc(assoc); 1093 nvmet_fc_tgt_a_put(assoc); 1094 spin_lock_irqsave(&tgtport->lock, flags); 1095 } 1096 spin_unlock_irqrestore(&tgtport->lock, flags); 1097 } 1098 1099 /* 1100 * nvmet layer has called to terminate an association 1101 */ 1102 static void 1103 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl) 1104 { 1105 struct nvmet_fc_tgtport *tgtport, *next; 1106 struct nvmet_fc_tgt_assoc *assoc; 1107 struct nvmet_fc_tgt_queue *queue; 1108 unsigned long flags; 1109 bool found_ctrl = false; 1110 1111 /* this is a bit ugly, but don't want to make locks layered */ 1112 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1113 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list, 1114 tgt_list) { 1115 if (!nvmet_fc_tgtport_get(tgtport)) 1116 continue; 1117 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1118 1119 spin_lock_irqsave(&tgtport->lock, flags); 1120 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) { 1121 queue = assoc->queues[0]; 1122 if (queue && queue->nvme_sq.ctrl == ctrl) { 1123 if (nvmet_fc_tgt_a_get(assoc)) 1124 found_ctrl = true; 1125 break; 1126 } 1127 } 1128 spin_unlock_irqrestore(&tgtport->lock, flags); 1129 1130 nvmet_fc_tgtport_put(tgtport); 1131 1132 if (found_ctrl) { 1133 schedule_work(&assoc->del_work); 1134 return; 1135 } 1136 1137 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 1138 } 1139 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 1140 } 1141 1142 /** 1143 * nvme_fc_unregister_targetport - transport entry point called by an 1144 * LLDD to deregister/remove a previously 1145 * registered a local NVME subsystem FC port. 1146 * @tgtport: pointer to the (registered) target port that is to be 1147 * deregistered. 1148 * 1149 * Returns: 1150 * a completion status. Must be 0 upon success; a negative errno 1151 * (ex: -ENXIO) upon failure. 1152 */ 1153 int 1154 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port) 1155 { 1156 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 1157 1158 /* terminate any outstanding associations */ 1159 __nvmet_fc_free_assocs(tgtport); 1160 1161 nvmet_fc_tgtport_put(tgtport); 1162 1163 return 0; 1164 } 1165 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport); 1166 1167 1168 /* *********************** FC-NVME LS Handling **************************** */ 1169 1170 1171 static void 1172 nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd) 1173 { 1174 struct fcnvme_ls_acc_hdr *acc = buf; 1175 1176 acc->w0.ls_cmd = ls_cmd; 1177 acc->desc_list_len = desc_len; 1178 acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST); 1179 acc->rqst.desc_len = 1180 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst)); 1181 acc->rqst.w0.ls_cmd = rqst_ls_cmd; 1182 } 1183 1184 static int 1185 nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd, 1186 u8 reason, u8 explanation, u8 vendor) 1187 { 1188 struct fcnvme_ls_rjt *rjt = buf; 1189 1190 nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST, 1191 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)), 1192 ls_cmd); 1193 rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT); 1194 rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt)); 1195 rjt->rjt.reason_code = reason; 1196 rjt->rjt.reason_explanation = explanation; 1197 rjt->rjt.vendor = vendor; 1198 1199 return sizeof(struct fcnvme_ls_rjt); 1200 } 1201 1202 /* Validation Error indexes into the string table below */ 1203 enum { 1204 VERR_NO_ERROR = 0, 1205 VERR_CR_ASSOC_LEN = 1, 1206 VERR_CR_ASSOC_RQST_LEN = 2, 1207 VERR_CR_ASSOC_CMD = 3, 1208 VERR_CR_ASSOC_CMD_LEN = 4, 1209 VERR_ERSP_RATIO = 5, 1210 VERR_ASSOC_ALLOC_FAIL = 6, 1211 VERR_QUEUE_ALLOC_FAIL = 7, 1212 VERR_CR_CONN_LEN = 8, 1213 VERR_CR_CONN_RQST_LEN = 9, 1214 VERR_ASSOC_ID = 10, 1215 VERR_ASSOC_ID_LEN = 11, 1216 VERR_NO_ASSOC = 12, 1217 VERR_CONN_ID = 13, 1218 VERR_CONN_ID_LEN = 14, 1219 VERR_NO_CONN = 15, 1220 VERR_CR_CONN_CMD = 16, 1221 VERR_CR_CONN_CMD_LEN = 17, 1222 VERR_DISCONN_LEN = 18, 1223 VERR_DISCONN_RQST_LEN = 19, 1224 VERR_DISCONN_CMD = 20, 1225 VERR_DISCONN_CMD_LEN = 21, 1226 VERR_DISCONN_SCOPE = 22, 1227 VERR_RS_LEN = 23, 1228 VERR_RS_RQST_LEN = 24, 1229 VERR_RS_CMD = 25, 1230 VERR_RS_CMD_LEN = 26, 1231 VERR_RS_RCTL = 27, 1232 VERR_RS_RO = 28, 1233 }; 1234 1235 static char *validation_errors[] = { 1236 "OK", 1237 "Bad CR_ASSOC Length", 1238 "Bad CR_ASSOC Rqst Length", 1239 "Not CR_ASSOC Cmd", 1240 "Bad CR_ASSOC Cmd Length", 1241 "Bad Ersp Ratio", 1242 "Association Allocation Failed", 1243 "Queue Allocation Failed", 1244 "Bad CR_CONN Length", 1245 "Bad CR_CONN Rqst Length", 1246 "Not Association ID", 1247 "Bad Association ID Length", 1248 "No Association", 1249 "Not Connection ID", 1250 "Bad Connection ID Length", 1251 "No Connection", 1252 "Not CR_CONN Cmd", 1253 "Bad CR_CONN Cmd Length", 1254 "Bad DISCONN Length", 1255 "Bad DISCONN Rqst Length", 1256 "Not DISCONN Cmd", 1257 "Bad DISCONN Cmd Length", 1258 "Bad Disconnect Scope", 1259 "Bad RS Length", 1260 "Bad RS Rqst Length", 1261 "Not RS Cmd", 1262 "Bad RS Cmd Length", 1263 "Bad RS R_CTL", 1264 "Bad RS Relative Offset", 1265 }; 1266 1267 static void 1268 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport, 1269 struct nvmet_fc_ls_iod *iod) 1270 { 1271 struct fcnvme_ls_cr_assoc_rqst *rqst = 1272 (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf; 1273 struct fcnvme_ls_cr_assoc_acc *acc = 1274 (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf; 1275 struct nvmet_fc_tgt_queue *queue; 1276 int ret = 0; 1277 1278 memset(acc, 0, sizeof(*acc)); 1279 1280 /* 1281 * FC-NVME spec changes. There are initiators sending different 1282 * lengths as padding sizes for Create Association Cmd descriptor 1283 * was incorrect. 1284 * Accept anything of "minimum" length. Assume format per 1.15 1285 * spec (with HOSTID reduced to 16 bytes), ignore how long the 1286 * trailing pad length is. 1287 */ 1288 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN) 1289 ret = VERR_CR_ASSOC_LEN; 1290 else if (be32_to_cpu(rqst->desc_list_len) < 1291 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN) 1292 ret = VERR_CR_ASSOC_RQST_LEN; 1293 else if (rqst->assoc_cmd.desc_tag != 1294 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD)) 1295 ret = VERR_CR_ASSOC_CMD; 1296 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) < 1297 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN) 1298 ret = VERR_CR_ASSOC_CMD_LEN; 1299 else if (!rqst->assoc_cmd.ersp_ratio || 1300 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >= 1301 be16_to_cpu(rqst->assoc_cmd.sqsize))) 1302 ret = VERR_ERSP_RATIO; 1303 1304 else { 1305 /* new association w/ admin queue */ 1306 iod->assoc = nvmet_fc_alloc_target_assoc(tgtport); 1307 if (!iod->assoc) 1308 ret = VERR_ASSOC_ALLOC_FAIL; 1309 else { 1310 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0, 1311 be16_to_cpu(rqst->assoc_cmd.sqsize)); 1312 if (!queue) 1313 ret = VERR_QUEUE_ALLOC_FAIL; 1314 } 1315 } 1316 1317 if (ret) { 1318 dev_err(tgtport->dev, 1319 "Create Association LS failed: %s\n", 1320 validation_errors[ret]); 1321 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, 1322 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, 1323 FCNVME_RJT_RC_LOGIC, 1324 FCNVME_RJT_EXP_NONE, 0); 1325 return; 1326 } 1327 1328 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio); 1329 atomic_set(&queue->connected, 1); 1330 queue->sqhd = 0; /* best place to init value */ 1331 1332 /* format a response */ 1333 1334 iod->lsreq->rsplen = sizeof(*acc); 1335 1336 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1337 fcnvme_lsdesc_len( 1338 sizeof(struct fcnvme_ls_cr_assoc_acc)), 1339 FCNVME_LS_CREATE_ASSOCIATION); 1340 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID); 1341 acc->associd.desc_len = 1342 fcnvme_lsdesc_len( 1343 sizeof(struct fcnvme_lsdesc_assoc_id)); 1344 acc->associd.association_id = 1345 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0)); 1346 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); 1347 acc->connectid.desc_len = 1348 fcnvme_lsdesc_len( 1349 sizeof(struct fcnvme_lsdesc_conn_id)); 1350 acc->connectid.connection_id = acc->associd.association_id; 1351 } 1352 1353 static void 1354 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport, 1355 struct nvmet_fc_ls_iod *iod) 1356 { 1357 struct fcnvme_ls_cr_conn_rqst *rqst = 1358 (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf; 1359 struct fcnvme_ls_cr_conn_acc *acc = 1360 (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf; 1361 struct nvmet_fc_tgt_queue *queue; 1362 int ret = 0; 1363 1364 memset(acc, 0, sizeof(*acc)); 1365 1366 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst)) 1367 ret = VERR_CR_CONN_LEN; 1368 else if (rqst->desc_list_len != 1369 fcnvme_lsdesc_len( 1370 sizeof(struct fcnvme_ls_cr_conn_rqst))) 1371 ret = VERR_CR_CONN_RQST_LEN; 1372 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) 1373 ret = VERR_ASSOC_ID; 1374 else if (rqst->associd.desc_len != 1375 fcnvme_lsdesc_len( 1376 sizeof(struct fcnvme_lsdesc_assoc_id))) 1377 ret = VERR_ASSOC_ID_LEN; 1378 else if (rqst->connect_cmd.desc_tag != 1379 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD)) 1380 ret = VERR_CR_CONN_CMD; 1381 else if (rqst->connect_cmd.desc_len != 1382 fcnvme_lsdesc_len( 1383 sizeof(struct fcnvme_lsdesc_cr_conn_cmd))) 1384 ret = VERR_CR_CONN_CMD_LEN; 1385 else if (!rqst->connect_cmd.ersp_ratio || 1386 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >= 1387 be16_to_cpu(rqst->connect_cmd.sqsize))) 1388 ret = VERR_ERSP_RATIO; 1389 1390 else { 1391 /* new io queue */ 1392 iod->assoc = nvmet_fc_find_target_assoc(tgtport, 1393 be64_to_cpu(rqst->associd.association_id)); 1394 if (!iod->assoc) 1395 ret = VERR_NO_ASSOC; 1396 else { 1397 queue = nvmet_fc_alloc_target_queue(iod->assoc, 1398 be16_to_cpu(rqst->connect_cmd.qid), 1399 be16_to_cpu(rqst->connect_cmd.sqsize)); 1400 if (!queue) 1401 ret = VERR_QUEUE_ALLOC_FAIL; 1402 1403 /* release get taken in nvmet_fc_find_target_assoc */ 1404 nvmet_fc_tgt_a_put(iod->assoc); 1405 } 1406 } 1407 1408 if (ret) { 1409 dev_err(tgtport->dev, 1410 "Create Connection LS failed: %s\n", 1411 validation_errors[ret]); 1412 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, 1413 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, 1414 (ret == VERR_NO_ASSOC) ? 1415 FCNVME_RJT_RC_INV_ASSOC : 1416 FCNVME_RJT_RC_LOGIC, 1417 FCNVME_RJT_EXP_NONE, 0); 1418 return; 1419 } 1420 1421 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio); 1422 atomic_set(&queue->connected, 1); 1423 queue->sqhd = 0; /* best place to init value */ 1424 1425 /* format a response */ 1426 1427 iod->lsreq->rsplen = sizeof(*acc); 1428 1429 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1430 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)), 1431 FCNVME_LS_CREATE_CONNECTION); 1432 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID); 1433 acc->connectid.desc_len = 1434 fcnvme_lsdesc_len( 1435 sizeof(struct fcnvme_lsdesc_conn_id)); 1436 acc->connectid.connection_id = 1437 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 1438 be16_to_cpu(rqst->connect_cmd.qid))); 1439 } 1440 1441 static void 1442 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport, 1443 struct nvmet_fc_ls_iod *iod) 1444 { 1445 struct fcnvme_ls_disconnect_rqst *rqst = 1446 (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf; 1447 struct fcnvme_ls_disconnect_acc *acc = 1448 (struct fcnvme_ls_disconnect_acc *)iod->rspbuf; 1449 struct nvmet_fc_tgt_queue *queue = NULL; 1450 struct nvmet_fc_tgt_assoc *assoc; 1451 int ret = 0; 1452 bool del_assoc = false; 1453 1454 memset(acc, 0, sizeof(*acc)); 1455 1456 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst)) 1457 ret = VERR_DISCONN_LEN; 1458 else if (rqst->desc_list_len != 1459 fcnvme_lsdesc_len( 1460 sizeof(struct fcnvme_ls_disconnect_rqst))) 1461 ret = VERR_DISCONN_RQST_LEN; 1462 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID)) 1463 ret = VERR_ASSOC_ID; 1464 else if (rqst->associd.desc_len != 1465 fcnvme_lsdesc_len( 1466 sizeof(struct fcnvme_lsdesc_assoc_id))) 1467 ret = VERR_ASSOC_ID_LEN; 1468 else if (rqst->discon_cmd.desc_tag != 1469 cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD)) 1470 ret = VERR_DISCONN_CMD; 1471 else if (rqst->discon_cmd.desc_len != 1472 fcnvme_lsdesc_len( 1473 sizeof(struct fcnvme_lsdesc_disconn_cmd))) 1474 ret = VERR_DISCONN_CMD_LEN; 1475 else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) && 1476 (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION)) 1477 ret = VERR_DISCONN_SCOPE; 1478 else { 1479 /* match an active association */ 1480 assoc = nvmet_fc_find_target_assoc(tgtport, 1481 be64_to_cpu(rqst->associd.association_id)); 1482 iod->assoc = assoc; 1483 if (assoc) { 1484 if (rqst->discon_cmd.scope == 1485 FCNVME_DISCONN_CONNECTION) { 1486 queue = nvmet_fc_find_target_queue(tgtport, 1487 be64_to_cpu( 1488 rqst->discon_cmd.id)); 1489 if (!queue) { 1490 nvmet_fc_tgt_a_put(assoc); 1491 ret = VERR_NO_CONN; 1492 } 1493 } 1494 } else 1495 ret = VERR_NO_ASSOC; 1496 } 1497 1498 if (ret) { 1499 dev_err(tgtport->dev, 1500 "Disconnect LS failed: %s\n", 1501 validation_errors[ret]); 1502 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc, 1503 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd, 1504 (ret == VERR_NO_ASSOC) ? 1505 FCNVME_RJT_RC_INV_ASSOC : 1506 (ret == VERR_NO_CONN) ? 1507 FCNVME_RJT_RC_INV_CONN : 1508 FCNVME_RJT_RC_LOGIC, 1509 FCNVME_RJT_EXP_NONE, 0); 1510 return; 1511 } 1512 1513 /* format a response */ 1514 1515 iod->lsreq->rsplen = sizeof(*acc); 1516 1517 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC, 1518 fcnvme_lsdesc_len( 1519 sizeof(struct fcnvme_ls_disconnect_acc)), 1520 FCNVME_LS_DISCONNECT); 1521 1522 1523 /* are we to delete a Connection ID (queue) */ 1524 if (queue) { 1525 int qid = queue->qid; 1526 1527 nvmet_fc_delete_target_queue(queue); 1528 1529 /* release the get taken by find_target_queue */ 1530 nvmet_fc_tgt_q_put(queue); 1531 1532 /* tear association down if io queue terminated */ 1533 if (!qid) 1534 del_assoc = true; 1535 } 1536 1537 /* release get taken in nvmet_fc_find_target_assoc */ 1538 nvmet_fc_tgt_a_put(iod->assoc); 1539 1540 if (del_assoc) 1541 nvmet_fc_delete_target_assoc(iod->assoc); 1542 } 1543 1544 1545 /* *********************** NVME Ctrl Routines **************************** */ 1546 1547 1548 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req); 1549 1550 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops; 1551 1552 static void 1553 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq) 1554 { 1555 struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private; 1556 struct nvmet_fc_tgtport *tgtport = iod->tgtport; 1557 1558 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma, 1559 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); 1560 nvmet_fc_free_ls_iod(tgtport, iod); 1561 nvmet_fc_tgtport_put(tgtport); 1562 } 1563 1564 static void 1565 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport, 1566 struct nvmet_fc_ls_iod *iod) 1567 { 1568 int ret; 1569 1570 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma, 1571 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE); 1572 1573 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq); 1574 if (ret) 1575 nvmet_fc_xmt_ls_rsp_done(iod->lsreq); 1576 } 1577 1578 /* 1579 * Actual processing routine for received FC-NVME LS Requests from the LLD 1580 */ 1581 static void 1582 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport, 1583 struct nvmet_fc_ls_iod *iod) 1584 { 1585 struct fcnvme_ls_rqst_w0 *w0 = 1586 (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf; 1587 1588 iod->lsreq->nvmet_fc_private = iod; 1589 iod->lsreq->rspbuf = iod->rspbuf; 1590 iod->lsreq->rspdma = iod->rspdma; 1591 iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done; 1592 /* Be preventative. handlers will later set to valid length */ 1593 iod->lsreq->rsplen = 0; 1594 1595 iod->assoc = NULL; 1596 1597 /* 1598 * handlers: 1599 * parse request input, execute the request, and format the 1600 * LS response 1601 */ 1602 switch (w0->ls_cmd) { 1603 case FCNVME_LS_CREATE_ASSOCIATION: 1604 /* Creates Association and initial Admin Queue/Connection */ 1605 nvmet_fc_ls_create_association(tgtport, iod); 1606 break; 1607 case FCNVME_LS_CREATE_CONNECTION: 1608 /* Creates an IO Queue/Connection */ 1609 nvmet_fc_ls_create_connection(tgtport, iod); 1610 break; 1611 case FCNVME_LS_DISCONNECT: 1612 /* Terminate a Queue/Connection or the Association */ 1613 nvmet_fc_ls_disconnect(tgtport, iod); 1614 break; 1615 default: 1616 iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf, 1617 NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd, 1618 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0); 1619 } 1620 1621 nvmet_fc_xmt_ls_rsp(tgtport, iod); 1622 } 1623 1624 /* 1625 * Actual processing routine for received FC-NVME LS Requests from the LLD 1626 */ 1627 static void 1628 nvmet_fc_handle_ls_rqst_work(struct work_struct *work) 1629 { 1630 struct nvmet_fc_ls_iod *iod = 1631 container_of(work, struct nvmet_fc_ls_iod, work); 1632 struct nvmet_fc_tgtport *tgtport = iod->tgtport; 1633 1634 nvmet_fc_handle_ls_rqst(tgtport, iod); 1635 } 1636 1637 1638 /** 1639 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD 1640 * upon the reception of a NVME LS request. 1641 * 1642 * The nvmet-fc layer will copy payload to an internal structure for 1643 * processing. As such, upon completion of the routine, the LLDD may 1644 * immediately free/reuse the LS request buffer passed in the call. 1645 * 1646 * If this routine returns error, the LLDD should abort the exchange. 1647 * 1648 * @tgtport: pointer to the (registered) target port the LS was 1649 * received on. 1650 * @lsreq: pointer to a lsreq request structure to be used to reference 1651 * the exchange corresponding to the LS. 1652 * @lsreqbuf: pointer to the buffer containing the LS Request 1653 * @lsreqbuf_len: length, in bytes, of the received LS request 1654 */ 1655 int 1656 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port, 1657 struct nvmefc_tgt_ls_req *lsreq, 1658 void *lsreqbuf, u32 lsreqbuf_len) 1659 { 1660 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 1661 struct nvmet_fc_ls_iod *iod; 1662 1663 if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE) 1664 return -E2BIG; 1665 1666 if (!nvmet_fc_tgtport_get(tgtport)) 1667 return -ESHUTDOWN; 1668 1669 iod = nvmet_fc_alloc_ls_iod(tgtport); 1670 if (!iod) { 1671 nvmet_fc_tgtport_put(tgtport); 1672 return -ENOENT; 1673 } 1674 1675 iod->lsreq = lsreq; 1676 iod->fcpreq = NULL; 1677 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len); 1678 iod->rqstdatalen = lsreqbuf_len; 1679 1680 schedule_work(&iod->work); 1681 1682 return 0; 1683 } 1684 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req); 1685 1686 1687 /* 1688 * ********************** 1689 * Start of FCP handling 1690 * ********************** 1691 */ 1692 1693 static int 1694 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod) 1695 { 1696 struct scatterlist *sg; 1697 struct page *page; 1698 unsigned int nent; 1699 u32 page_len, length; 1700 int i = 0; 1701 1702 length = fod->req.transfer_len; 1703 nent = DIV_ROUND_UP(length, PAGE_SIZE); 1704 sg = kmalloc_array(nent, sizeof(struct scatterlist), GFP_KERNEL); 1705 if (!sg) 1706 goto out; 1707 1708 sg_init_table(sg, nent); 1709 1710 while (length) { 1711 page_len = min_t(u32, length, PAGE_SIZE); 1712 1713 page = alloc_page(GFP_KERNEL); 1714 if (!page) 1715 goto out_free_pages; 1716 1717 sg_set_page(&sg[i], page, page_len, 0); 1718 length -= page_len; 1719 i++; 1720 } 1721 1722 fod->data_sg = sg; 1723 fod->data_sg_cnt = nent; 1724 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent, 1725 ((fod->io_dir == NVMET_FCP_WRITE) ? 1726 DMA_FROM_DEVICE : DMA_TO_DEVICE)); 1727 /* note: write from initiator perspective */ 1728 1729 return 0; 1730 1731 out_free_pages: 1732 while (i > 0) { 1733 i--; 1734 __free_page(sg_page(&sg[i])); 1735 } 1736 kfree(sg); 1737 fod->data_sg = NULL; 1738 fod->data_sg_cnt = 0; 1739 out: 1740 return NVME_SC_INTERNAL; 1741 } 1742 1743 static void 1744 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod) 1745 { 1746 struct scatterlist *sg; 1747 int count; 1748 1749 if (!fod->data_sg || !fod->data_sg_cnt) 1750 return; 1751 1752 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt, 1753 ((fod->io_dir == NVMET_FCP_WRITE) ? 1754 DMA_FROM_DEVICE : DMA_TO_DEVICE)); 1755 for_each_sg(fod->data_sg, sg, fod->data_sg_cnt, count) 1756 __free_page(sg_page(sg)); 1757 kfree(fod->data_sg); 1758 fod->data_sg = NULL; 1759 fod->data_sg_cnt = 0; 1760 } 1761 1762 1763 static bool 1764 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd) 1765 { 1766 u32 sqtail, used; 1767 1768 /* egad, this is ugly. And sqtail is just a best guess */ 1769 sqtail = atomic_read(&q->sqtail) % q->sqsize; 1770 1771 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd); 1772 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9))); 1773 } 1774 1775 /* 1776 * Prep RSP payload. 1777 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op 1778 */ 1779 static void 1780 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport, 1781 struct nvmet_fc_fcp_iod *fod) 1782 { 1783 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf; 1784 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common; 1785 struct nvme_completion *cqe = &ersp->cqe; 1786 u32 *cqewd = (u32 *)cqe; 1787 bool send_ersp = false; 1788 u32 rsn, rspcnt, xfr_length; 1789 1790 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP) 1791 xfr_length = fod->req.transfer_len; 1792 else 1793 xfr_length = fod->offset; 1794 1795 /* 1796 * check to see if we can send a 0's rsp. 1797 * Note: to send a 0's response, the NVME-FC host transport will 1798 * recreate the CQE. The host transport knows: sq id, SQHD (last 1799 * seen in an ersp), and command_id. Thus it will create a 1800 * zero-filled CQE with those known fields filled in. Transport 1801 * must send an ersp for any condition where the cqe won't match 1802 * this. 1803 * 1804 * Here are the FC-NVME mandated cases where we must send an ersp: 1805 * every N responses, where N=ersp_ratio 1806 * force fabric commands to send ersp's (not in FC-NVME but good 1807 * practice) 1808 * normal cmds: any time status is non-zero, or status is zero 1809 * but words 0 or 1 are non-zero. 1810 * the SQ is 90% or more full 1811 * the cmd is a fused command 1812 * transferred data length not equal to cmd iu length 1813 */ 1814 rspcnt = atomic_inc_return(&fod->queue->zrspcnt); 1815 if (!(rspcnt % fod->queue->ersp_ratio) || 1816 sqe->opcode == nvme_fabrics_command || 1817 xfr_length != fod->req.transfer_len || 1818 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] || 1819 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) || 1820 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head))) 1821 send_ersp = true; 1822 1823 /* re-set the fields */ 1824 fod->fcpreq->rspaddr = ersp; 1825 fod->fcpreq->rspdma = fod->rspdma; 1826 1827 if (!send_ersp) { 1828 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP); 1829 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP; 1830 } else { 1831 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32)); 1832 rsn = atomic_inc_return(&fod->queue->rsn); 1833 ersp->rsn = cpu_to_be32(rsn); 1834 ersp->xfrd_len = cpu_to_be32(xfr_length); 1835 fod->fcpreq->rsplen = sizeof(*ersp); 1836 } 1837 1838 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma, 1839 sizeof(fod->rspiubuf), DMA_TO_DEVICE); 1840 } 1841 1842 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq); 1843 1844 static void 1845 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport, 1846 struct nvmet_fc_fcp_iod *fod) 1847 { 1848 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 1849 1850 /* data no longer needed */ 1851 nvmet_fc_free_tgt_pgs(fod); 1852 1853 /* 1854 * if an ABTS was received or we issued the fcp_abort early 1855 * don't call abort routine again. 1856 */ 1857 /* no need to take lock - lock was taken earlier to get here */ 1858 if (!fod->aborted) 1859 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq); 1860 1861 nvmet_fc_free_fcp_iod(fod->queue, fod); 1862 } 1863 1864 static void 1865 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport, 1866 struct nvmet_fc_fcp_iod *fod) 1867 { 1868 int ret; 1869 1870 fod->fcpreq->op = NVMET_FCOP_RSP; 1871 fod->fcpreq->timeout = 0; 1872 1873 nvmet_fc_prep_fcp_rsp(tgtport, fod); 1874 1875 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); 1876 if (ret) 1877 nvmet_fc_abort_op(tgtport, fod); 1878 } 1879 1880 static void 1881 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport, 1882 struct nvmet_fc_fcp_iod *fod, u8 op) 1883 { 1884 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 1885 unsigned long flags; 1886 u32 tlen; 1887 int ret; 1888 1889 fcpreq->op = op; 1890 fcpreq->offset = fod->offset; 1891 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC; 1892 1893 tlen = min_t(u32, tgtport->max_sg_cnt * PAGE_SIZE, 1894 (fod->req.transfer_len - fod->offset)); 1895 fcpreq->transfer_length = tlen; 1896 fcpreq->transferred_length = 0; 1897 fcpreq->fcp_error = 0; 1898 fcpreq->rsplen = 0; 1899 1900 fcpreq->sg = &fod->data_sg[fod->offset / PAGE_SIZE]; 1901 fcpreq->sg_cnt = DIV_ROUND_UP(tlen, PAGE_SIZE); 1902 1903 /* 1904 * If the last READDATA request: check if LLDD supports 1905 * combined xfr with response. 1906 */ 1907 if ((op == NVMET_FCOP_READDATA) && 1908 ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) && 1909 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) { 1910 fcpreq->op = NVMET_FCOP_READDATA_RSP; 1911 nvmet_fc_prep_fcp_rsp(tgtport, fod); 1912 } 1913 1914 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); 1915 if (ret) { 1916 /* 1917 * should be ok to set w/o lock as its in the thread of 1918 * execution (not an async timer routine) and doesn't 1919 * contend with any clearing action 1920 */ 1921 fod->abort = true; 1922 1923 if (op == NVMET_FCOP_WRITEDATA) { 1924 spin_lock_irqsave(&fod->flock, flags); 1925 fod->writedataactive = false; 1926 spin_unlock_irqrestore(&fod->flock, flags); 1927 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); 1928 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ { 1929 fcpreq->fcp_error = ret; 1930 fcpreq->transferred_length = 0; 1931 nvmet_fc_xmt_fcp_op_done(fod->fcpreq); 1932 } 1933 } 1934 } 1935 1936 static inline bool 1937 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort) 1938 { 1939 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 1940 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 1941 1942 /* if in the middle of an io and we need to tear down */ 1943 if (abort) { 1944 if (fcpreq->op == NVMET_FCOP_WRITEDATA) { 1945 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); 1946 return true; 1947 } 1948 1949 nvmet_fc_abort_op(tgtport, fod); 1950 return true; 1951 } 1952 1953 return false; 1954 } 1955 1956 /* 1957 * actual done handler for FCP operations when completed by the lldd 1958 */ 1959 static void 1960 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod) 1961 { 1962 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq; 1963 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 1964 unsigned long flags; 1965 bool abort; 1966 1967 spin_lock_irqsave(&fod->flock, flags); 1968 abort = fod->abort; 1969 fod->writedataactive = false; 1970 spin_unlock_irqrestore(&fod->flock, flags); 1971 1972 switch (fcpreq->op) { 1973 1974 case NVMET_FCOP_WRITEDATA: 1975 if (__nvmet_fc_fod_op_abort(fod, abort)) 1976 return; 1977 if (fcpreq->fcp_error || 1978 fcpreq->transferred_length != fcpreq->transfer_length) { 1979 spin_lock(&fod->flock); 1980 fod->abort = true; 1981 spin_unlock(&fod->flock); 1982 1983 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL); 1984 return; 1985 } 1986 1987 fod->offset += fcpreq->transferred_length; 1988 if (fod->offset != fod->req.transfer_len) { 1989 spin_lock_irqsave(&fod->flock, flags); 1990 fod->writedataactive = true; 1991 spin_unlock_irqrestore(&fod->flock, flags); 1992 1993 /* transfer the next chunk */ 1994 nvmet_fc_transfer_fcp_data(tgtport, fod, 1995 NVMET_FCOP_WRITEDATA); 1996 return; 1997 } 1998 1999 /* data transfer complete, resume with nvmet layer */ 2000 nvmet_req_execute(&fod->req); 2001 break; 2002 2003 case NVMET_FCOP_READDATA: 2004 case NVMET_FCOP_READDATA_RSP: 2005 if (__nvmet_fc_fod_op_abort(fod, abort)) 2006 return; 2007 if (fcpreq->fcp_error || 2008 fcpreq->transferred_length != fcpreq->transfer_length) { 2009 nvmet_fc_abort_op(tgtport, fod); 2010 return; 2011 } 2012 2013 /* success */ 2014 2015 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) { 2016 /* data no longer needed */ 2017 nvmet_fc_free_tgt_pgs(fod); 2018 nvmet_fc_free_fcp_iod(fod->queue, fod); 2019 return; 2020 } 2021 2022 fod->offset += fcpreq->transferred_length; 2023 if (fod->offset != fod->req.transfer_len) { 2024 /* transfer the next chunk */ 2025 nvmet_fc_transfer_fcp_data(tgtport, fod, 2026 NVMET_FCOP_READDATA); 2027 return; 2028 } 2029 2030 /* data transfer complete, send response */ 2031 2032 /* data no longer needed */ 2033 nvmet_fc_free_tgt_pgs(fod); 2034 2035 nvmet_fc_xmt_fcp_rsp(tgtport, fod); 2036 2037 break; 2038 2039 case NVMET_FCOP_RSP: 2040 if (__nvmet_fc_fod_op_abort(fod, abort)) 2041 return; 2042 nvmet_fc_free_fcp_iod(fod->queue, fod); 2043 break; 2044 2045 default: 2046 break; 2047 } 2048 } 2049 2050 static void 2051 nvmet_fc_fcp_rqst_op_done_work(struct work_struct *work) 2052 { 2053 struct nvmet_fc_fcp_iod *fod = 2054 container_of(work, struct nvmet_fc_fcp_iod, done_work); 2055 2056 nvmet_fc_fod_op_done(fod); 2057 } 2058 2059 static void 2060 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq) 2061 { 2062 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 2063 struct nvmet_fc_tgt_queue *queue = fod->queue; 2064 2065 if (fod->tgtport->ops->target_features & NVMET_FCTGTFEAT_OPDONE_IN_ISR) 2066 /* context switch so completion is not in ISR context */ 2067 queue_work_on(queue->cpu, queue->work_q, &fod->done_work); 2068 else 2069 nvmet_fc_fod_op_done(fod); 2070 } 2071 2072 /* 2073 * actual completion handler after execution by the nvmet layer 2074 */ 2075 static void 2076 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport, 2077 struct nvmet_fc_fcp_iod *fod, int status) 2078 { 2079 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common; 2080 struct nvme_completion *cqe = &fod->rspiubuf.cqe; 2081 unsigned long flags; 2082 bool abort; 2083 2084 spin_lock_irqsave(&fod->flock, flags); 2085 abort = fod->abort; 2086 spin_unlock_irqrestore(&fod->flock, flags); 2087 2088 /* if we have a CQE, snoop the last sq_head value */ 2089 if (!status) 2090 fod->queue->sqhd = cqe->sq_head; 2091 2092 if (abort) { 2093 nvmet_fc_abort_op(tgtport, fod); 2094 return; 2095 } 2096 2097 /* if an error handling the cmd post initial parsing */ 2098 if (status) { 2099 /* fudge up a failed CQE status for our transport error */ 2100 memset(cqe, 0, sizeof(*cqe)); 2101 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */ 2102 cqe->sq_id = cpu_to_le16(fod->queue->qid); 2103 cqe->command_id = sqe->command_id; 2104 cqe->status = cpu_to_le16(status); 2105 } else { 2106 2107 /* 2108 * try to push the data even if the SQE status is non-zero. 2109 * There may be a status where data still was intended to 2110 * be moved 2111 */ 2112 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) { 2113 /* push the data over before sending rsp */ 2114 nvmet_fc_transfer_fcp_data(tgtport, fod, 2115 NVMET_FCOP_READDATA); 2116 return; 2117 } 2118 2119 /* writes & no data - fall thru */ 2120 } 2121 2122 /* data no longer needed */ 2123 nvmet_fc_free_tgt_pgs(fod); 2124 2125 nvmet_fc_xmt_fcp_rsp(tgtport, fod); 2126 } 2127 2128 2129 static void 2130 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req) 2131 { 2132 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req); 2133 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 2134 2135 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0); 2136 } 2137 2138 2139 /* 2140 * Actual processing routine for received FC-NVME LS Requests from the LLD 2141 */ 2142 static void 2143 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport, 2144 struct nvmet_fc_fcp_iod *fod) 2145 { 2146 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf; 2147 u32 xfrlen = be32_to_cpu(cmdiu->data_len); 2148 int ret; 2149 2150 /* 2151 * Fused commands are currently not supported in the linux 2152 * implementation. 2153 * 2154 * As such, the implementation of the FC transport does not 2155 * look at the fused commands and order delivery to the upper 2156 * layer until we have both based on csn. 2157 */ 2158 2159 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done; 2160 2161 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) { 2162 fod->io_dir = NVMET_FCP_WRITE; 2163 if (!nvme_is_write(&cmdiu->sqe)) 2164 goto transport_error; 2165 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) { 2166 fod->io_dir = NVMET_FCP_READ; 2167 if (nvme_is_write(&cmdiu->sqe)) 2168 goto transport_error; 2169 } else { 2170 fod->io_dir = NVMET_FCP_NODATA; 2171 if (xfrlen) 2172 goto transport_error; 2173 } 2174 2175 fod->req.cmd = &fod->cmdiubuf.sqe; 2176 fod->req.rsp = &fod->rspiubuf.cqe; 2177 fod->req.port = fod->queue->port; 2178 2179 /* clear any response payload */ 2180 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf)); 2181 2182 fod->data_sg = NULL; 2183 fod->data_sg_cnt = 0; 2184 2185 ret = nvmet_req_init(&fod->req, 2186 &fod->queue->nvme_cq, 2187 &fod->queue->nvme_sq, 2188 &nvmet_fc_tgt_fcp_ops); 2189 if (!ret) { 2190 /* bad SQE content or invalid ctrl state */ 2191 /* nvmet layer has already called op done to send rsp. */ 2192 return; 2193 } 2194 2195 fod->req.transfer_len = xfrlen; 2196 2197 /* keep a running counter of tail position */ 2198 atomic_inc(&fod->queue->sqtail); 2199 2200 if (fod->req.transfer_len) { 2201 ret = nvmet_fc_alloc_tgt_pgs(fod); 2202 if (ret) { 2203 nvmet_req_complete(&fod->req, ret); 2204 return; 2205 } 2206 } 2207 fod->req.sg = fod->data_sg; 2208 fod->req.sg_cnt = fod->data_sg_cnt; 2209 fod->offset = 0; 2210 2211 if (fod->io_dir == NVMET_FCP_WRITE) { 2212 /* pull the data over before invoking nvmet layer */ 2213 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA); 2214 return; 2215 } 2216 2217 /* 2218 * Reads or no data: 2219 * 2220 * can invoke the nvmet_layer now. If read data, cmd completion will 2221 * push the data 2222 */ 2223 nvmet_req_execute(&fod->req); 2224 return; 2225 2226 transport_error: 2227 nvmet_fc_abort_op(tgtport, fod); 2228 } 2229 2230 /* 2231 * Actual processing routine for received FC-NVME LS Requests from the LLD 2232 */ 2233 static void 2234 nvmet_fc_handle_fcp_rqst_work(struct work_struct *work) 2235 { 2236 struct nvmet_fc_fcp_iod *fod = 2237 container_of(work, struct nvmet_fc_fcp_iod, work); 2238 struct nvmet_fc_tgtport *tgtport = fod->tgtport; 2239 2240 nvmet_fc_handle_fcp_rqst(tgtport, fod); 2241 } 2242 2243 /** 2244 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD 2245 * upon the reception of a NVME FCP CMD IU. 2246 * 2247 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc 2248 * layer for processing. 2249 * 2250 * The nvmet_fc layer allocates a local job structure (struct 2251 * nvmet_fc_fcp_iod) from the queue for the io and copies the 2252 * CMD IU buffer to the job structure. As such, on a successful 2253 * completion (returns 0), the LLDD may immediately free/reuse 2254 * the CMD IU buffer passed in the call. 2255 * 2256 * However, in some circumstances, due to the packetized nature of FC 2257 * and the api of the FC LLDD which may issue a hw command to send the 2258 * response, but the LLDD may not get the hw completion for that command 2259 * and upcall the nvmet_fc layer before a new command may be 2260 * asynchronously received - its possible for a command to be received 2261 * before the LLDD and nvmet_fc have recycled the job structure. It gives 2262 * the appearance of more commands received than fits in the sq. 2263 * To alleviate this scenario, a temporary queue is maintained in the 2264 * transport for pending LLDD requests waiting for a queue job structure. 2265 * In these "overrun" cases, a temporary queue element is allocated 2266 * the LLDD request and CMD iu buffer information remembered, and the 2267 * routine returns a -EOVERFLOW status. Subsequently, when a queue job 2268 * structure is freed, it is immediately reallocated for anything on the 2269 * pending request list. The LLDDs defer_rcv() callback is called, 2270 * informing the LLDD that it may reuse the CMD IU buffer, and the io 2271 * is then started normally with the transport. 2272 * 2273 * The LLDD, when receiving an -EOVERFLOW completion status, is to treat 2274 * the completion as successful but must not reuse the CMD IU buffer 2275 * until the LLDD's defer_rcv() callback has been called for the 2276 * corresponding struct nvmefc_tgt_fcp_req pointer. 2277 * 2278 * If there is any other condition in which an error occurs, the 2279 * transport will return a non-zero status indicating the error. 2280 * In all cases other than -EOVERFLOW, the transport has not accepted the 2281 * request and the LLDD should abort the exchange. 2282 * 2283 * @target_port: pointer to the (registered) target port the FCP CMD IU 2284 * was received on. 2285 * @fcpreq: pointer to a fcpreq request structure to be used to reference 2286 * the exchange corresponding to the FCP Exchange. 2287 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU 2288 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU 2289 */ 2290 int 2291 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port, 2292 struct nvmefc_tgt_fcp_req *fcpreq, 2293 void *cmdiubuf, u32 cmdiubuf_len) 2294 { 2295 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port); 2296 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf; 2297 struct nvmet_fc_tgt_queue *queue; 2298 struct nvmet_fc_fcp_iod *fod; 2299 struct nvmet_fc_defer_fcp_req *deferfcp; 2300 unsigned long flags; 2301 2302 /* validate iu, so the connection id can be used to find the queue */ 2303 if ((cmdiubuf_len != sizeof(*cmdiu)) || 2304 (cmdiu->scsi_id != NVME_CMD_SCSI_ID) || 2305 (cmdiu->fc_id != NVME_CMD_FC_ID) || 2306 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4))) 2307 return -EIO; 2308 2309 queue = nvmet_fc_find_target_queue(tgtport, 2310 be64_to_cpu(cmdiu->connection_id)); 2311 if (!queue) 2312 return -ENOTCONN; 2313 2314 /* 2315 * note: reference taken by find_target_queue 2316 * After successful fod allocation, the fod will inherit the 2317 * ownership of that reference and will remove the reference 2318 * when the fod is freed. 2319 */ 2320 2321 spin_lock_irqsave(&queue->qlock, flags); 2322 2323 fod = nvmet_fc_alloc_fcp_iod(queue); 2324 if (fod) { 2325 spin_unlock_irqrestore(&queue->qlock, flags); 2326 2327 fcpreq->nvmet_fc_private = fod; 2328 fod->fcpreq = fcpreq; 2329 2330 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len); 2331 2332 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq); 2333 2334 return 0; 2335 } 2336 2337 if (!tgtport->ops->defer_rcv) { 2338 spin_unlock_irqrestore(&queue->qlock, flags); 2339 /* release the queue lookup reference */ 2340 nvmet_fc_tgt_q_put(queue); 2341 return -ENOENT; 2342 } 2343 2344 deferfcp = list_first_entry_or_null(&queue->avail_defer_list, 2345 struct nvmet_fc_defer_fcp_req, req_list); 2346 if (deferfcp) { 2347 /* Just re-use one that was previously allocated */ 2348 list_del(&deferfcp->req_list); 2349 } else { 2350 spin_unlock_irqrestore(&queue->qlock, flags); 2351 2352 /* Now we need to dynamically allocate one */ 2353 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL); 2354 if (!deferfcp) { 2355 /* release the queue lookup reference */ 2356 nvmet_fc_tgt_q_put(queue); 2357 return -ENOMEM; 2358 } 2359 spin_lock_irqsave(&queue->qlock, flags); 2360 } 2361 2362 /* For now, use rspaddr / rsplen to save payload information */ 2363 fcpreq->rspaddr = cmdiubuf; 2364 fcpreq->rsplen = cmdiubuf_len; 2365 deferfcp->fcp_req = fcpreq; 2366 2367 /* defer processing till a fod becomes available */ 2368 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list); 2369 2370 /* NOTE: the queue lookup reference is still valid */ 2371 2372 spin_unlock_irqrestore(&queue->qlock, flags); 2373 2374 return -EOVERFLOW; 2375 } 2376 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req); 2377 2378 /** 2379 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD 2380 * upon the reception of an ABTS for a FCP command 2381 * 2382 * Notify the transport that an ABTS has been received for a FCP command 2383 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The 2384 * LLDD believes the command is still being worked on 2385 * (template_ops->fcp_req_release() has not been called). 2386 * 2387 * The transport will wait for any outstanding work (an op to the LLDD, 2388 * which the lldd should complete with error due to the ABTS; or the 2389 * completion from the nvmet layer of the nvme command), then will 2390 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to 2391 * return the i/o context to the LLDD. The LLDD may send the BA_ACC 2392 * to the ABTS either after return from this function (assuming any 2393 * outstanding op work has been terminated) or upon the callback being 2394 * called. 2395 * 2396 * @target_port: pointer to the (registered) target port the FCP CMD IU 2397 * was received on. 2398 * @fcpreq: pointer to the fcpreq request structure that corresponds 2399 * to the exchange that received the ABTS. 2400 */ 2401 void 2402 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port, 2403 struct nvmefc_tgt_fcp_req *fcpreq) 2404 { 2405 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private; 2406 struct nvmet_fc_tgt_queue *queue; 2407 unsigned long flags; 2408 2409 if (!fod || fod->fcpreq != fcpreq) 2410 /* job appears to have already completed, ignore abort */ 2411 return; 2412 2413 queue = fod->queue; 2414 2415 spin_lock_irqsave(&queue->qlock, flags); 2416 if (fod->active) { 2417 /* 2418 * mark as abort. The abort handler, invoked upon completion 2419 * of any work, will detect the aborted status and do the 2420 * callback. 2421 */ 2422 spin_lock(&fod->flock); 2423 fod->abort = true; 2424 fod->aborted = true; 2425 spin_unlock(&fod->flock); 2426 } 2427 spin_unlock_irqrestore(&queue->qlock, flags); 2428 } 2429 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort); 2430 2431 2432 struct nvmet_fc_traddr { 2433 u64 nn; 2434 u64 pn; 2435 }; 2436 2437 static int 2438 __nvme_fc_parse_u64(substring_t *sstr, u64 *val) 2439 { 2440 u64 token64; 2441 2442 if (match_u64(sstr, &token64)) 2443 return -EINVAL; 2444 *val = token64; 2445 2446 return 0; 2447 } 2448 2449 /* 2450 * This routine validates and extracts the WWN's from the TRADDR string. 2451 * As kernel parsers need the 0x to determine number base, universally 2452 * build string to parse with 0x prefix before parsing name strings. 2453 */ 2454 static int 2455 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen) 2456 { 2457 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1]; 2458 substring_t wwn = { name, &name[sizeof(name)-1] }; 2459 int nnoffset, pnoffset; 2460 2461 /* validate it string one of the 2 allowed formats */ 2462 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH && 2463 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) && 2464 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET], 2465 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) { 2466 nnoffset = NVME_FC_TRADDR_OXNNLEN; 2467 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET + 2468 NVME_FC_TRADDR_OXNNLEN; 2469 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH && 2470 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) && 2471 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET], 2472 "pn-", NVME_FC_TRADDR_NNLEN))) { 2473 nnoffset = NVME_FC_TRADDR_NNLEN; 2474 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN; 2475 } else 2476 goto out_einval; 2477 2478 name[0] = '0'; 2479 name[1] = 'x'; 2480 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0; 2481 2482 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN); 2483 if (__nvme_fc_parse_u64(&wwn, &traddr->nn)) 2484 goto out_einval; 2485 2486 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN); 2487 if (__nvme_fc_parse_u64(&wwn, &traddr->pn)) 2488 goto out_einval; 2489 2490 return 0; 2491 2492 out_einval: 2493 pr_warn("%s: bad traddr string\n", __func__); 2494 return -EINVAL; 2495 } 2496 2497 static int 2498 nvmet_fc_add_port(struct nvmet_port *port) 2499 { 2500 struct nvmet_fc_tgtport *tgtport; 2501 struct nvmet_fc_traddr traddr = { 0L, 0L }; 2502 unsigned long flags; 2503 int ret; 2504 2505 /* validate the address info */ 2506 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) || 2507 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC)) 2508 return -EINVAL; 2509 2510 /* map the traddr address info to a target port */ 2511 2512 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr, 2513 sizeof(port->disc_addr.traddr)); 2514 if (ret) 2515 return ret; 2516 2517 ret = -ENXIO; 2518 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 2519 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) { 2520 if ((tgtport->fc_target_port.node_name == traddr.nn) && 2521 (tgtport->fc_target_port.port_name == traddr.pn)) { 2522 /* a FC port can only be 1 nvmet port id */ 2523 if (!tgtport->port) { 2524 tgtport->port = port; 2525 port->priv = tgtport; 2526 nvmet_fc_tgtport_get(tgtport); 2527 ret = 0; 2528 } else 2529 ret = -EALREADY; 2530 break; 2531 } 2532 } 2533 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 2534 return ret; 2535 } 2536 2537 static void 2538 nvmet_fc_remove_port(struct nvmet_port *port) 2539 { 2540 struct nvmet_fc_tgtport *tgtport = port->priv; 2541 unsigned long flags; 2542 bool matched = false; 2543 2544 spin_lock_irqsave(&nvmet_fc_tgtlock, flags); 2545 if (tgtport->port == port) { 2546 matched = true; 2547 tgtport->port = NULL; 2548 } 2549 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags); 2550 2551 if (matched) 2552 nvmet_fc_tgtport_put(tgtport); 2553 } 2554 2555 static struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = { 2556 .owner = THIS_MODULE, 2557 .type = NVMF_TRTYPE_FC, 2558 .msdbd = 1, 2559 .add_port = nvmet_fc_add_port, 2560 .remove_port = nvmet_fc_remove_port, 2561 .queue_response = nvmet_fc_fcp_nvme_cmd_done, 2562 .delete_ctrl = nvmet_fc_delete_ctrl, 2563 }; 2564 2565 static int __init nvmet_fc_init_module(void) 2566 { 2567 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops); 2568 } 2569 2570 static void __exit nvmet_fc_exit_module(void) 2571 { 2572 /* sanity check - all lports should be removed */ 2573 if (!list_empty(&nvmet_fc_target_list)) 2574 pr_warn("%s: targetport list not empty\n", __func__); 2575 2576 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops); 2577 2578 ida_destroy(&nvmet_fc_tgtport_cnt); 2579 } 2580 2581 module_init(nvmet_fc_init_module); 2582 module_exit(nvmet_fc_exit_module); 2583 2584 MODULE_LICENSE("GPL v2"); 2585