1 /* 2 * Copyright (C) 2013 Shaohua Li <shli@kernel.org> 3 * Copyright (C) 2014 Red Hat, Inc. 4 * Copyright (C) 2015 Arrikto, Inc. 5 * Copyright (C) 2017 Chinamobile, Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 * more details. 15 * 16 * You should have received a copy of the GNU General Public License along with 17 * this program; if not, write to the Free Software Foundation, Inc., 18 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 19 */ 20 21 #include <linux/spinlock.h> 22 #include <linux/module.h> 23 #include <linux/idr.h> 24 #include <linux/kernel.h> 25 #include <linux/timer.h> 26 #include <linux/parser.h> 27 #include <linux/vmalloc.h> 28 #include <linux/uio_driver.h> 29 #include <linux/radix-tree.h> 30 #include <linux/stringify.h> 31 #include <linux/bitops.h> 32 #include <linux/highmem.h> 33 #include <linux/configfs.h> 34 #include <linux/mutex.h> 35 #include <linux/kthread.h> 36 #include <net/genetlink.h> 37 #include <scsi/scsi_common.h> 38 #include <scsi/scsi_proto.h> 39 #include <target/target_core_base.h> 40 #include <target/target_core_fabric.h> 41 #include <target/target_core_backend.h> 42 43 #include <linux/target_core_user.h> 44 45 /* 46 * Define a shared-memory interface for LIO to pass SCSI commands and 47 * data to userspace for processing. This is to allow backends that 48 * are too complex for in-kernel support to be possible. 49 * 50 * It uses the UIO framework to do a lot of the device-creation and 51 * introspection work for us. 52 * 53 * See the .h file for how the ring is laid out. Note that while the 54 * command ring is defined, the particulars of the data area are 55 * not. Offset values in the command entry point to other locations 56 * internal to the mmap()ed area. There is separate space outside the 57 * command ring for data buffers. This leaves maximum flexibility for 58 * moving buffer allocations, or even page flipping or other 59 * allocation techniques, without altering the command ring layout. 60 * 61 * SECURITY: 62 * The user process must be assumed to be malicious. There's no way to 63 * prevent it breaking the command ring protocol if it wants, but in 64 * order to prevent other issues we must only ever read *data* from 65 * the shared memory area, not offsets or sizes. This applies to 66 * command ring entries as well as the mailbox. Extra code needed for 67 * this may have a 'UAM' comment. 68 */ 69 70 #define TCMU_TIME_OUT (30 * MSEC_PER_SEC) 71 72 /* For cmd area, the size is fixed 8MB */ 73 #define CMDR_SIZE (8 * 1024 * 1024) 74 75 /* 76 * For data area, the block size is PAGE_SIZE and 77 * the total size is 256K * PAGE_SIZE. 78 */ 79 #define DATA_BLOCK_SIZE PAGE_SIZE 80 #define DATA_BLOCK_BITS (256 * 1024) 81 #define DATA_SIZE (DATA_BLOCK_BITS * DATA_BLOCK_SIZE) 82 #define DATA_BLOCK_INIT_BITS 128 83 84 /* The total size of the ring is 8M + 256K * PAGE_SIZE */ 85 #define TCMU_RING_SIZE (CMDR_SIZE + DATA_SIZE) 86 87 /* Default maximum of the global data blocks(512K * PAGE_SIZE) */ 88 #define TCMU_GLOBAL_MAX_BLOCKS (512 * 1024) 89 90 static u8 tcmu_kern_cmd_reply_supported; 91 92 static struct device *tcmu_root_device; 93 94 struct tcmu_hba { 95 u32 host_id; 96 }; 97 98 #define TCMU_CONFIG_LEN 256 99 100 struct tcmu_nl_cmd { 101 /* wake up thread waiting for reply */ 102 struct completion complete; 103 int cmd; 104 int status; 105 }; 106 107 struct tcmu_dev { 108 struct list_head node; 109 struct kref kref; 110 struct se_device se_dev; 111 112 char *name; 113 struct se_hba *hba; 114 115 #define TCMU_DEV_BIT_OPEN 0 116 #define TCMU_DEV_BIT_BROKEN 1 117 unsigned long flags; 118 119 struct uio_info uio_info; 120 121 struct inode *inode; 122 123 struct tcmu_mailbox *mb_addr; 124 size_t dev_size; 125 u32 cmdr_size; 126 u32 cmdr_last_cleaned; 127 /* Offset of data area from start of mb */ 128 /* Must add data_off and mb_addr to get the address */ 129 size_t data_off; 130 size_t data_size; 131 132 wait_queue_head_t wait_cmdr; 133 struct mutex cmdr_lock; 134 135 bool waiting_global; 136 uint32_t dbi_max; 137 uint32_t dbi_thresh; 138 DECLARE_BITMAP(data_bitmap, DATA_BLOCK_BITS); 139 struct radix_tree_root data_blocks; 140 141 struct idr commands; 142 spinlock_t commands_lock; 143 144 struct timer_list timeout; 145 unsigned int cmd_time_out; 146 147 spinlock_t nl_cmd_lock; 148 struct tcmu_nl_cmd curr_nl_cmd; 149 /* wake up threads waiting on curr_nl_cmd */ 150 wait_queue_head_t nl_cmd_wq; 151 152 char dev_config[TCMU_CONFIG_LEN]; 153 }; 154 155 #define TCMU_DEV(_se_dev) container_of(_se_dev, struct tcmu_dev, se_dev) 156 157 #define CMDR_OFF sizeof(struct tcmu_mailbox) 158 159 struct tcmu_cmd { 160 struct se_cmd *se_cmd; 161 struct tcmu_dev *tcmu_dev; 162 163 uint16_t cmd_id; 164 165 /* Can't use se_cmd when cleaning up expired cmds, because if 166 cmd has been completed then accessing se_cmd is off limits */ 167 uint32_t dbi_cnt; 168 uint32_t dbi_cur; 169 uint32_t *dbi; 170 171 unsigned long deadline; 172 173 #define TCMU_CMD_BIT_EXPIRED 0 174 unsigned long flags; 175 }; 176 177 static struct task_struct *unmap_thread; 178 static wait_queue_head_t unmap_wait; 179 static DEFINE_MUTEX(root_udev_mutex); 180 static LIST_HEAD(root_udev); 181 182 static atomic_t global_db_count = ATOMIC_INIT(0); 183 184 static struct kmem_cache *tcmu_cmd_cache; 185 186 /* multicast group */ 187 enum tcmu_multicast_groups { 188 TCMU_MCGRP_CONFIG, 189 }; 190 191 static const struct genl_multicast_group tcmu_mcgrps[] = { 192 [TCMU_MCGRP_CONFIG] = { .name = "config", }, 193 }; 194 195 static struct nla_policy tcmu_attr_policy[TCMU_ATTR_MAX+1] = { 196 [TCMU_ATTR_DEVICE] = { .type = NLA_STRING }, 197 [TCMU_ATTR_MINOR] = { .type = NLA_U32 }, 198 [TCMU_ATTR_CMD_STATUS] = { .type = NLA_S32 }, 199 [TCMU_ATTR_DEVICE_ID] = { .type = NLA_U32 }, 200 [TCMU_ATTR_SUPP_KERN_CMD_REPLY] = { .type = NLA_U8 }, 201 }; 202 203 static int tcmu_genl_cmd_done(struct genl_info *info, int completed_cmd) 204 { 205 struct se_device *dev; 206 struct tcmu_dev *udev; 207 struct tcmu_nl_cmd *nl_cmd; 208 int dev_id, rc, ret = 0; 209 bool is_removed = (completed_cmd == TCMU_CMD_REMOVED_DEVICE); 210 211 if (!info->attrs[TCMU_ATTR_CMD_STATUS] || 212 !info->attrs[TCMU_ATTR_DEVICE_ID]) { 213 printk(KERN_ERR "TCMU_ATTR_CMD_STATUS or TCMU_ATTR_DEVICE_ID not set, doing nothing\n"); 214 return -EINVAL; 215 } 216 217 dev_id = nla_get_u32(info->attrs[TCMU_ATTR_DEVICE_ID]); 218 rc = nla_get_s32(info->attrs[TCMU_ATTR_CMD_STATUS]); 219 220 dev = target_find_device(dev_id, !is_removed); 221 if (!dev) { 222 printk(KERN_ERR "tcmu nl cmd %u/%u completion could not find device with dev id %u.\n", 223 completed_cmd, rc, dev_id); 224 return -ENODEV; 225 } 226 udev = TCMU_DEV(dev); 227 228 spin_lock(&udev->nl_cmd_lock); 229 nl_cmd = &udev->curr_nl_cmd; 230 231 pr_debug("genl cmd done got id %d curr %d done %d rc %d\n", dev_id, 232 nl_cmd->cmd, completed_cmd, rc); 233 234 if (nl_cmd->cmd != completed_cmd) { 235 printk(KERN_ERR "Mismatched commands (Expecting reply for %d. Current %d).\n", 236 completed_cmd, nl_cmd->cmd); 237 ret = -EINVAL; 238 } else { 239 nl_cmd->status = rc; 240 } 241 242 spin_unlock(&udev->nl_cmd_lock); 243 if (!is_removed) 244 target_undepend_item(&dev->dev_group.cg_item); 245 if (!ret) 246 complete(&nl_cmd->complete); 247 return ret; 248 } 249 250 static int tcmu_genl_rm_dev_done(struct sk_buff *skb, struct genl_info *info) 251 { 252 return tcmu_genl_cmd_done(info, TCMU_CMD_REMOVED_DEVICE); 253 } 254 255 static int tcmu_genl_add_dev_done(struct sk_buff *skb, struct genl_info *info) 256 { 257 return tcmu_genl_cmd_done(info, TCMU_CMD_ADDED_DEVICE); 258 } 259 260 static int tcmu_genl_reconfig_dev_done(struct sk_buff *skb, 261 struct genl_info *info) 262 { 263 return tcmu_genl_cmd_done(info, TCMU_CMD_RECONFIG_DEVICE); 264 } 265 266 static int tcmu_genl_set_features(struct sk_buff *skb, struct genl_info *info) 267 { 268 if (info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]) { 269 tcmu_kern_cmd_reply_supported = 270 nla_get_u8(info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]); 271 printk(KERN_INFO "tcmu daemon: command reply support %u.\n", 272 tcmu_kern_cmd_reply_supported); 273 } 274 275 return 0; 276 } 277 278 static const struct genl_ops tcmu_genl_ops[] = { 279 { 280 .cmd = TCMU_CMD_SET_FEATURES, 281 .flags = GENL_ADMIN_PERM, 282 .policy = tcmu_attr_policy, 283 .doit = tcmu_genl_set_features, 284 }, 285 { 286 .cmd = TCMU_CMD_ADDED_DEVICE_DONE, 287 .flags = GENL_ADMIN_PERM, 288 .policy = tcmu_attr_policy, 289 .doit = tcmu_genl_add_dev_done, 290 }, 291 { 292 .cmd = TCMU_CMD_REMOVED_DEVICE_DONE, 293 .flags = GENL_ADMIN_PERM, 294 .policy = tcmu_attr_policy, 295 .doit = tcmu_genl_rm_dev_done, 296 }, 297 { 298 .cmd = TCMU_CMD_RECONFIG_DEVICE_DONE, 299 .flags = GENL_ADMIN_PERM, 300 .policy = tcmu_attr_policy, 301 .doit = tcmu_genl_reconfig_dev_done, 302 }, 303 }; 304 305 /* Our generic netlink family */ 306 static struct genl_family tcmu_genl_family __ro_after_init = { 307 .module = THIS_MODULE, 308 .hdrsize = 0, 309 .name = "TCM-USER", 310 .version = 2, 311 .maxattr = TCMU_ATTR_MAX, 312 .mcgrps = tcmu_mcgrps, 313 .n_mcgrps = ARRAY_SIZE(tcmu_mcgrps), 314 .netnsok = true, 315 .ops = tcmu_genl_ops, 316 .n_ops = ARRAY_SIZE(tcmu_genl_ops), 317 }; 318 319 #define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index)) 320 #define tcmu_cmd_reset_dbi_cur(cmd) tcmu_cmd_set_dbi_cur(cmd, 0) 321 #define tcmu_cmd_set_dbi(cmd, index) ((cmd)->dbi[(cmd)->dbi_cur++] = (index)) 322 #define tcmu_cmd_get_dbi(cmd) ((cmd)->dbi[(cmd)->dbi_cur++]) 323 324 static void tcmu_cmd_free_data(struct tcmu_cmd *tcmu_cmd, uint32_t len) 325 { 326 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev; 327 uint32_t i; 328 329 for (i = 0; i < len; i++) 330 clear_bit(tcmu_cmd->dbi[i], udev->data_bitmap); 331 } 332 333 static inline bool tcmu_get_empty_block(struct tcmu_dev *udev, 334 struct tcmu_cmd *tcmu_cmd) 335 { 336 struct page *page; 337 int ret, dbi; 338 339 dbi = find_first_zero_bit(udev->data_bitmap, udev->dbi_thresh); 340 if (dbi == udev->dbi_thresh) 341 return false; 342 343 page = radix_tree_lookup(&udev->data_blocks, dbi); 344 if (!page) { 345 if (atomic_add_return(1, &global_db_count) > 346 TCMU_GLOBAL_MAX_BLOCKS) { 347 atomic_dec(&global_db_count); 348 return false; 349 } 350 351 /* try to get new page from the mm */ 352 page = alloc_page(GFP_KERNEL); 353 if (!page) 354 goto err_alloc; 355 356 ret = radix_tree_insert(&udev->data_blocks, dbi, page); 357 if (ret) 358 goto err_insert; 359 } 360 361 if (dbi > udev->dbi_max) 362 udev->dbi_max = dbi; 363 364 set_bit(dbi, udev->data_bitmap); 365 tcmu_cmd_set_dbi(tcmu_cmd, dbi); 366 367 return true; 368 err_insert: 369 __free_page(page); 370 err_alloc: 371 atomic_dec(&global_db_count); 372 return false; 373 } 374 375 static bool tcmu_get_empty_blocks(struct tcmu_dev *udev, 376 struct tcmu_cmd *tcmu_cmd) 377 { 378 int i; 379 380 udev->waiting_global = false; 381 382 for (i = tcmu_cmd->dbi_cur; i < tcmu_cmd->dbi_cnt; i++) { 383 if (!tcmu_get_empty_block(udev, tcmu_cmd)) 384 goto err; 385 } 386 return true; 387 388 err: 389 udev->waiting_global = true; 390 /* Try to wake up the unmap thread */ 391 wake_up(&unmap_wait); 392 return false; 393 } 394 395 static inline struct page * 396 tcmu_get_block_page(struct tcmu_dev *udev, uint32_t dbi) 397 { 398 return radix_tree_lookup(&udev->data_blocks, dbi); 399 } 400 401 static inline void tcmu_free_cmd(struct tcmu_cmd *tcmu_cmd) 402 { 403 kfree(tcmu_cmd->dbi); 404 kmem_cache_free(tcmu_cmd_cache, tcmu_cmd); 405 } 406 407 static inline size_t tcmu_cmd_get_data_length(struct tcmu_cmd *tcmu_cmd) 408 { 409 struct se_cmd *se_cmd = tcmu_cmd->se_cmd; 410 size_t data_length = round_up(se_cmd->data_length, DATA_BLOCK_SIZE); 411 412 if (se_cmd->se_cmd_flags & SCF_BIDI) { 413 BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents)); 414 data_length += round_up(se_cmd->t_bidi_data_sg->length, 415 DATA_BLOCK_SIZE); 416 } 417 418 return data_length; 419 } 420 421 static inline uint32_t tcmu_cmd_get_block_cnt(struct tcmu_cmd *tcmu_cmd) 422 { 423 size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd); 424 425 return data_length / DATA_BLOCK_SIZE; 426 } 427 428 static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd) 429 { 430 struct se_device *se_dev = se_cmd->se_dev; 431 struct tcmu_dev *udev = TCMU_DEV(se_dev); 432 struct tcmu_cmd *tcmu_cmd; 433 int cmd_id; 434 435 tcmu_cmd = kmem_cache_zalloc(tcmu_cmd_cache, GFP_KERNEL); 436 if (!tcmu_cmd) 437 return NULL; 438 439 tcmu_cmd->se_cmd = se_cmd; 440 tcmu_cmd->tcmu_dev = udev; 441 if (udev->cmd_time_out) 442 tcmu_cmd->deadline = jiffies + 443 msecs_to_jiffies(udev->cmd_time_out); 444 445 tcmu_cmd_reset_dbi_cur(tcmu_cmd); 446 tcmu_cmd->dbi_cnt = tcmu_cmd_get_block_cnt(tcmu_cmd); 447 tcmu_cmd->dbi = kcalloc(tcmu_cmd->dbi_cnt, sizeof(uint32_t), 448 GFP_KERNEL); 449 if (!tcmu_cmd->dbi) { 450 kmem_cache_free(tcmu_cmd_cache, tcmu_cmd); 451 return NULL; 452 } 453 454 idr_preload(GFP_KERNEL); 455 spin_lock_irq(&udev->commands_lock); 456 cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 0, 457 USHRT_MAX, GFP_NOWAIT); 458 spin_unlock_irq(&udev->commands_lock); 459 idr_preload_end(); 460 461 if (cmd_id < 0) { 462 tcmu_free_cmd(tcmu_cmd); 463 return NULL; 464 } 465 tcmu_cmd->cmd_id = cmd_id; 466 467 return tcmu_cmd; 468 } 469 470 static inline void tcmu_flush_dcache_range(void *vaddr, size_t size) 471 { 472 unsigned long offset = offset_in_page(vaddr); 473 474 size = round_up(size+offset, PAGE_SIZE); 475 vaddr -= offset; 476 477 while (size) { 478 flush_dcache_page(virt_to_page(vaddr)); 479 size -= PAGE_SIZE; 480 } 481 } 482 483 /* 484 * Some ring helper functions. We don't assume size is a power of 2 so 485 * we can't use circ_buf.h. 486 */ 487 static inline size_t spc_used(size_t head, size_t tail, size_t size) 488 { 489 int diff = head - tail; 490 491 if (diff >= 0) 492 return diff; 493 else 494 return size + diff; 495 } 496 497 static inline size_t spc_free(size_t head, size_t tail, size_t size) 498 { 499 /* Keep 1 byte unused or we can't tell full from empty */ 500 return (size - spc_used(head, tail, size) - 1); 501 } 502 503 static inline size_t head_to_end(size_t head, size_t size) 504 { 505 return size - head; 506 } 507 508 static inline void new_iov(struct iovec **iov, int *iov_cnt, 509 struct tcmu_dev *udev) 510 { 511 struct iovec *iovec; 512 513 if (*iov_cnt != 0) 514 (*iov)++; 515 (*iov_cnt)++; 516 517 iovec = *iov; 518 memset(iovec, 0, sizeof(struct iovec)); 519 } 520 521 #define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size) 522 523 /* offset is relative to mb_addr */ 524 static inline size_t get_block_offset_user(struct tcmu_dev *dev, 525 int dbi, int remaining) 526 { 527 return dev->data_off + dbi * DATA_BLOCK_SIZE + 528 DATA_BLOCK_SIZE - remaining; 529 } 530 531 static inline size_t iov_tail(struct iovec *iov) 532 { 533 return (size_t)iov->iov_base + iov->iov_len; 534 } 535 536 static int scatter_data_area(struct tcmu_dev *udev, 537 struct tcmu_cmd *tcmu_cmd, struct scatterlist *data_sg, 538 unsigned int data_nents, struct iovec **iov, 539 int *iov_cnt, bool copy_data) 540 { 541 int i, dbi; 542 int block_remaining = 0; 543 void *from, *to = NULL; 544 size_t copy_bytes, to_offset, offset; 545 struct scatterlist *sg; 546 struct page *page; 547 548 for_each_sg(data_sg, sg, data_nents, i) { 549 int sg_remaining = sg->length; 550 from = kmap_atomic(sg_page(sg)) + sg->offset; 551 while (sg_remaining > 0) { 552 if (block_remaining == 0) { 553 if (to) 554 kunmap_atomic(to); 555 556 block_remaining = DATA_BLOCK_SIZE; 557 dbi = tcmu_cmd_get_dbi(tcmu_cmd); 558 page = tcmu_get_block_page(udev, dbi); 559 to = kmap_atomic(page); 560 } 561 562 copy_bytes = min_t(size_t, sg_remaining, 563 block_remaining); 564 to_offset = get_block_offset_user(udev, dbi, 565 block_remaining); 566 567 if (*iov_cnt != 0 && 568 to_offset == iov_tail(*iov)) { 569 (*iov)->iov_len += copy_bytes; 570 } else { 571 new_iov(iov, iov_cnt, udev); 572 (*iov)->iov_base = (void __user *)to_offset; 573 (*iov)->iov_len = copy_bytes; 574 } 575 if (copy_data) { 576 offset = DATA_BLOCK_SIZE - block_remaining; 577 memcpy(to + offset, 578 from + sg->length - sg_remaining, 579 copy_bytes); 580 tcmu_flush_dcache_range(to, copy_bytes); 581 } 582 sg_remaining -= copy_bytes; 583 block_remaining -= copy_bytes; 584 } 585 kunmap_atomic(from - sg->offset); 586 } 587 if (to) 588 kunmap_atomic(to); 589 590 return 0; 591 } 592 593 static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd, 594 bool bidi) 595 { 596 struct se_cmd *se_cmd = cmd->se_cmd; 597 int i, dbi; 598 int block_remaining = 0; 599 void *from = NULL, *to; 600 size_t copy_bytes, offset; 601 struct scatterlist *sg, *data_sg; 602 struct page *page; 603 unsigned int data_nents; 604 uint32_t count = 0; 605 606 if (!bidi) { 607 data_sg = se_cmd->t_data_sg; 608 data_nents = se_cmd->t_data_nents; 609 } else { 610 611 /* 612 * For bidi case, the first count blocks are for Data-Out 613 * buffer blocks, and before gathering the Data-In buffer 614 * the Data-Out buffer blocks should be discarded. 615 */ 616 count = DIV_ROUND_UP(se_cmd->data_length, DATA_BLOCK_SIZE); 617 618 data_sg = se_cmd->t_bidi_data_sg; 619 data_nents = se_cmd->t_bidi_data_nents; 620 } 621 622 tcmu_cmd_set_dbi_cur(cmd, count); 623 624 for_each_sg(data_sg, sg, data_nents, i) { 625 int sg_remaining = sg->length; 626 to = kmap_atomic(sg_page(sg)) + sg->offset; 627 while (sg_remaining > 0) { 628 if (block_remaining == 0) { 629 if (from) 630 kunmap_atomic(from); 631 632 block_remaining = DATA_BLOCK_SIZE; 633 dbi = tcmu_cmd_get_dbi(cmd); 634 page = tcmu_get_block_page(udev, dbi); 635 from = kmap_atomic(page); 636 } 637 copy_bytes = min_t(size_t, sg_remaining, 638 block_remaining); 639 offset = DATA_BLOCK_SIZE - block_remaining; 640 tcmu_flush_dcache_range(from, copy_bytes); 641 memcpy(to + sg->length - sg_remaining, from + offset, 642 copy_bytes); 643 644 sg_remaining -= copy_bytes; 645 block_remaining -= copy_bytes; 646 } 647 kunmap_atomic(to - sg->offset); 648 } 649 if (from) 650 kunmap_atomic(from); 651 } 652 653 static inline size_t spc_bitmap_free(unsigned long *bitmap, uint32_t thresh) 654 { 655 return DATA_BLOCK_SIZE * (thresh - bitmap_weight(bitmap, thresh)); 656 } 657 658 /* 659 * We can't queue a command until we have space available on the cmd ring *and* 660 * space available on the data area. 661 * 662 * Called with ring lock held. 663 */ 664 static bool is_ring_space_avail(struct tcmu_dev *udev, struct tcmu_cmd *cmd, 665 size_t cmd_size, size_t data_needed) 666 { 667 struct tcmu_mailbox *mb = udev->mb_addr; 668 uint32_t blocks_needed = (data_needed + DATA_BLOCK_SIZE - 1) 669 / DATA_BLOCK_SIZE; 670 size_t space, cmd_needed; 671 u32 cmd_head; 672 673 tcmu_flush_dcache_range(mb, sizeof(*mb)); 674 675 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ 676 677 /* 678 * If cmd end-of-ring space is too small then we need space for a NOP plus 679 * original cmd - cmds are internally contiguous. 680 */ 681 if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size) 682 cmd_needed = cmd_size; 683 else 684 cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size); 685 686 space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size); 687 if (space < cmd_needed) { 688 pr_debug("no cmd space: %u %u %u\n", cmd_head, 689 udev->cmdr_last_cleaned, udev->cmdr_size); 690 return false; 691 } 692 693 /* try to check and get the data blocks as needed */ 694 space = spc_bitmap_free(udev->data_bitmap, udev->dbi_thresh); 695 if (space < data_needed) { 696 unsigned long blocks_left = DATA_BLOCK_BITS - udev->dbi_thresh; 697 unsigned long grow; 698 699 if (blocks_left < blocks_needed) { 700 pr_debug("no data space: only %lu available, but ask for %zu\n", 701 blocks_left * DATA_BLOCK_SIZE, 702 data_needed); 703 return false; 704 } 705 706 /* Try to expand the thresh */ 707 if (!udev->dbi_thresh) { 708 /* From idle state */ 709 uint32_t init_thresh = DATA_BLOCK_INIT_BITS; 710 711 udev->dbi_thresh = max(blocks_needed, init_thresh); 712 } else { 713 /* 714 * Grow the data area by max(blocks needed, 715 * dbi_thresh / 2), but limited to the max 716 * DATA_BLOCK_BITS size. 717 */ 718 grow = max(blocks_needed, udev->dbi_thresh / 2); 719 udev->dbi_thresh += grow; 720 if (udev->dbi_thresh > DATA_BLOCK_BITS) 721 udev->dbi_thresh = DATA_BLOCK_BITS; 722 } 723 } 724 725 return tcmu_get_empty_blocks(udev, cmd); 726 } 727 728 static inline size_t tcmu_cmd_get_base_cmd_size(size_t iov_cnt) 729 { 730 return max(offsetof(struct tcmu_cmd_entry, req.iov[iov_cnt]), 731 sizeof(struct tcmu_cmd_entry)); 732 } 733 734 static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd *tcmu_cmd, 735 size_t base_command_size) 736 { 737 struct se_cmd *se_cmd = tcmu_cmd->se_cmd; 738 size_t command_size; 739 740 command_size = base_command_size + 741 round_up(scsi_command_size(se_cmd->t_task_cdb), 742 TCMU_OP_ALIGN_SIZE); 743 744 WARN_ON(command_size & (TCMU_OP_ALIGN_SIZE-1)); 745 746 return command_size; 747 } 748 749 static sense_reason_t 750 tcmu_queue_cmd_ring(struct tcmu_cmd *tcmu_cmd) 751 { 752 struct tcmu_dev *udev = tcmu_cmd->tcmu_dev; 753 struct se_cmd *se_cmd = tcmu_cmd->se_cmd; 754 size_t base_command_size, command_size; 755 struct tcmu_mailbox *mb; 756 struct tcmu_cmd_entry *entry; 757 struct iovec *iov; 758 int iov_cnt, ret; 759 uint32_t cmd_head; 760 uint64_t cdb_off; 761 bool copy_to_data_area; 762 size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd); 763 764 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) 765 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 766 767 /* 768 * Must be a certain minimum size for response sense info, but 769 * also may be larger if the iov array is large. 770 * 771 * We prepare as many iovs as possbile for potential uses here, 772 * because it's expensive to tell how many regions are freed in 773 * the bitmap & global data pool, as the size calculated here 774 * will only be used to do the checks. 775 * 776 * The size will be recalculated later as actually needed to save 777 * cmd area memories. 778 */ 779 base_command_size = tcmu_cmd_get_base_cmd_size(tcmu_cmd->dbi_cnt); 780 command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size); 781 782 mutex_lock(&udev->cmdr_lock); 783 784 mb = udev->mb_addr; 785 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ 786 if ((command_size > (udev->cmdr_size / 2)) || 787 data_length > udev->data_size) { 788 pr_warn("TCMU: Request of size %zu/%zu is too big for %u/%zu " 789 "cmd ring/data area\n", command_size, data_length, 790 udev->cmdr_size, udev->data_size); 791 mutex_unlock(&udev->cmdr_lock); 792 return TCM_INVALID_CDB_FIELD; 793 } 794 795 while (!is_ring_space_avail(udev, tcmu_cmd, command_size, data_length)) { 796 int ret; 797 DEFINE_WAIT(__wait); 798 799 prepare_to_wait(&udev->wait_cmdr, &__wait, TASK_INTERRUPTIBLE); 800 801 pr_debug("sleeping for ring space\n"); 802 mutex_unlock(&udev->cmdr_lock); 803 if (udev->cmd_time_out) 804 ret = schedule_timeout( 805 msecs_to_jiffies(udev->cmd_time_out)); 806 else 807 ret = schedule_timeout(msecs_to_jiffies(TCMU_TIME_OUT)); 808 finish_wait(&udev->wait_cmdr, &__wait); 809 if (!ret) { 810 pr_warn("tcmu: command timed out\n"); 811 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 812 } 813 814 mutex_lock(&udev->cmdr_lock); 815 816 /* We dropped cmdr_lock, cmd_head is stale */ 817 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ 818 } 819 820 /* Insert a PAD if end-of-ring space is too small */ 821 if (head_to_end(cmd_head, udev->cmdr_size) < command_size) { 822 size_t pad_size = head_to_end(cmd_head, udev->cmdr_size); 823 824 entry = (void *) mb + CMDR_OFF + cmd_head; 825 tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_PAD); 826 tcmu_hdr_set_len(&entry->hdr.len_op, pad_size); 827 entry->hdr.cmd_id = 0; /* not used for PAD */ 828 entry->hdr.kflags = 0; 829 entry->hdr.uflags = 0; 830 tcmu_flush_dcache_range(entry, sizeof(*entry)); 831 832 UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size); 833 tcmu_flush_dcache_range(mb, sizeof(*mb)); 834 835 cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */ 836 WARN_ON(cmd_head != 0); 837 } 838 839 entry = (void *) mb + CMDR_OFF + cmd_head; 840 memset(entry, 0, command_size); 841 tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_CMD); 842 entry->hdr.cmd_id = tcmu_cmd->cmd_id; 843 844 /* Handle allocating space from the data area */ 845 tcmu_cmd_reset_dbi_cur(tcmu_cmd); 846 iov = &entry->req.iov[0]; 847 iov_cnt = 0; 848 copy_to_data_area = (se_cmd->data_direction == DMA_TO_DEVICE 849 || se_cmd->se_cmd_flags & SCF_BIDI); 850 ret = scatter_data_area(udev, tcmu_cmd, se_cmd->t_data_sg, 851 se_cmd->t_data_nents, &iov, &iov_cnt, 852 copy_to_data_area); 853 if (ret) { 854 tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt); 855 mutex_unlock(&udev->cmdr_lock); 856 857 pr_err("tcmu: alloc and scatter data failed\n"); 858 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 859 } 860 entry->req.iov_cnt = iov_cnt; 861 862 /* Handle BIDI commands */ 863 iov_cnt = 0; 864 if (se_cmd->se_cmd_flags & SCF_BIDI) { 865 iov++; 866 ret = scatter_data_area(udev, tcmu_cmd, 867 se_cmd->t_bidi_data_sg, 868 se_cmd->t_bidi_data_nents, 869 &iov, &iov_cnt, false); 870 if (ret) { 871 tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt); 872 mutex_unlock(&udev->cmdr_lock); 873 874 pr_err("tcmu: alloc and scatter bidi data failed\n"); 875 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 876 } 877 } 878 entry->req.iov_bidi_cnt = iov_cnt; 879 880 /* 881 * Recalaulate the command's base size and size according 882 * to the actual needs 883 */ 884 base_command_size = tcmu_cmd_get_base_cmd_size(entry->req.iov_cnt + 885 entry->req.iov_bidi_cnt); 886 command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size); 887 888 tcmu_hdr_set_len(&entry->hdr.len_op, command_size); 889 890 /* All offsets relative to mb_addr, not start of entry! */ 891 cdb_off = CMDR_OFF + cmd_head + base_command_size; 892 memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb)); 893 entry->req.cdb_off = cdb_off; 894 tcmu_flush_dcache_range(entry, sizeof(*entry)); 895 896 UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size); 897 tcmu_flush_dcache_range(mb, sizeof(*mb)); 898 mutex_unlock(&udev->cmdr_lock); 899 900 /* TODO: only if FLUSH and FUA? */ 901 uio_event_notify(&udev->uio_info); 902 903 if (udev->cmd_time_out) 904 mod_timer(&udev->timeout, round_jiffies_up(jiffies + 905 msecs_to_jiffies(udev->cmd_time_out))); 906 907 return TCM_NO_SENSE; 908 } 909 910 static sense_reason_t 911 tcmu_queue_cmd(struct se_cmd *se_cmd) 912 { 913 struct se_device *se_dev = se_cmd->se_dev; 914 struct tcmu_dev *udev = TCMU_DEV(se_dev); 915 struct tcmu_cmd *tcmu_cmd; 916 sense_reason_t ret; 917 918 tcmu_cmd = tcmu_alloc_cmd(se_cmd); 919 if (!tcmu_cmd) 920 return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; 921 922 ret = tcmu_queue_cmd_ring(tcmu_cmd); 923 if (ret != TCM_NO_SENSE) { 924 pr_err("TCMU: Could not queue command\n"); 925 spin_lock_irq(&udev->commands_lock); 926 idr_remove(&udev->commands, tcmu_cmd->cmd_id); 927 spin_unlock_irq(&udev->commands_lock); 928 929 tcmu_free_cmd(tcmu_cmd); 930 } 931 932 return ret; 933 } 934 935 static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry *entry) 936 { 937 struct se_cmd *se_cmd = cmd->se_cmd; 938 struct tcmu_dev *udev = cmd->tcmu_dev; 939 940 /* 941 * cmd has been completed already from timeout, just reclaim 942 * data area space and free cmd 943 */ 944 if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) 945 goto out; 946 947 tcmu_cmd_reset_dbi_cur(cmd); 948 949 if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) { 950 pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n", 951 cmd->se_cmd); 952 entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION; 953 } else if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) { 954 transport_copy_sense_to_cmd(se_cmd, entry->rsp.sense_buffer); 955 } else if (se_cmd->se_cmd_flags & SCF_BIDI) { 956 /* Get Data-In buffer before clean up */ 957 gather_data_area(udev, cmd, true); 958 } else if (se_cmd->data_direction == DMA_FROM_DEVICE) { 959 gather_data_area(udev, cmd, false); 960 } else if (se_cmd->data_direction == DMA_TO_DEVICE) { 961 /* TODO: */ 962 } else if (se_cmd->data_direction != DMA_NONE) { 963 pr_warn("TCMU: data direction was %d!\n", 964 se_cmd->data_direction); 965 } 966 967 target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status); 968 969 out: 970 cmd->se_cmd = NULL; 971 tcmu_cmd_free_data(cmd, cmd->dbi_cnt); 972 tcmu_free_cmd(cmd); 973 } 974 975 static unsigned int tcmu_handle_completions(struct tcmu_dev *udev) 976 { 977 struct tcmu_mailbox *mb; 978 int handled = 0; 979 980 if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) { 981 pr_err("ring broken, not handling completions\n"); 982 return 0; 983 } 984 985 mb = udev->mb_addr; 986 tcmu_flush_dcache_range(mb, sizeof(*mb)); 987 988 while (udev->cmdr_last_cleaned != ACCESS_ONCE(mb->cmd_tail)) { 989 990 struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned; 991 struct tcmu_cmd *cmd; 992 993 tcmu_flush_dcache_range(entry, sizeof(*entry)); 994 995 if (tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_PAD) { 996 UPDATE_HEAD(udev->cmdr_last_cleaned, 997 tcmu_hdr_get_len(entry->hdr.len_op), 998 udev->cmdr_size); 999 continue; 1000 } 1001 WARN_ON(tcmu_hdr_get_op(entry->hdr.len_op) != TCMU_OP_CMD); 1002 1003 spin_lock(&udev->commands_lock); 1004 cmd = idr_remove(&udev->commands, entry->hdr.cmd_id); 1005 spin_unlock(&udev->commands_lock); 1006 1007 if (!cmd) { 1008 pr_err("cmd_id not found, ring is broken\n"); 1009 set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags); 1010 break; 1011 } 1012 1013 tcmu_handle_completion(cmd, entry); 1014 1015 UPDATE_HEAD(udev->cmdr_last_cleaned, 1016 tcmu_hdr_get_len(entry->hdr.len_op), 1017 udev->cmdr_size); 1018 1019 handled++; 1020 } 1021 1022 if (mb->cmd_tail == mb->cmd_head) 1023 del_timer(&udev->timeout); /* no more pending cmds */ 1024 1025 wake_up(&udev->wait_cmdr); 1026 1027 return handled; 1028 } 1029 1030 static int tcmu_check_expired_cmd(int id, void *p, void *data) 1031 { 1032 struct tcmu_cmd *cmd = p; 1033 1034 if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) 1035 return 0; 1036 1037 if (!time_after(jiffies, cmd->deadline)) 1038 return 0; 1039 1040 set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags); 1041 target_complete_cmd(cmd->se_cmd, SAM_STAT_CHECK_CONDITION); 1042 cmd->se_cmd = NULL; 1043 1044 return 0; 1045 } 1046 1047 static void tcmu_device_timedout(unsigned long data) 1048 { 1049 struct tcmu_dev *udev = (struct tcmu_dev *)data; 1050 unsigned long flags; 1051 1052 spin_lock_irqsave(&udev->commands_lock, flags); 1053 idr_for_each(&udev->commands, tcmu_check_expired_cmd, NULL); 1054 spin_unlock_irqrestore(&udev->commands_lock, flags); 1055 1056 /* Try to wake up the ummap thread */ 1057 wake_up(&unmap_wait); 1058 1059 /* 1060 * We don't need to wakeup threads on wait_cmdr since they have their 1061 * own timeout. 1062 */ 1063 } 1064 1065 static int tcmu_attach_hba(struct se_hba *hba, u32 host_id) 1066 { 1067 struct tcmu_hba *tcmu_hba; 1068 1069 tcmu_hba = kzalloc(sizeof(struct tcmu_hba), GFP_KERNEL); 1070 if (!tcmu_hba) 1071 return -ENOMEM; 1072 1073 tcmu_hba->host_id = host_id; 1074 hba->hba_ptr = tcmu_hba; 1075 1076 return 0; 1077 } 1078 1079 static void tcmu_detach_hba(struct se_hba *hba) 1080 { 1081 kfree(hba->hba_ptr); 1082 hba->hba_ptr = NULL; 1083 } 1084 1085 static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name) 1086 { 1087 struct tcmu_dev *udev; 1088 1089 udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL); 1090 if (!udev) 1091 return NULL; 1092 kref_init(&udev->kref); 1093 1094 udev->name = kstrdup(name, GFP_KERNEL); 1095 if (!udev->name) { 1096 kfree(udev); 1097 return NULL; 1098 } 1099 1100 udev->hba = hba; 1101 udev->cmd_time_out = TCMU_TIME_OUT; 1102 1103 init_waitqueue_head(&udev->wait_cmdr); 1104 mutex_init(&udev->cmdr_lock); 1105 1106 idr_init(&udev->commands); 1107 spin_lock_init(&udev->commands_lock); 1108 1109 setup_timer(&udev->timeout, tcmu_device_timedout, 1110 (unsigned long)udev); 1111 1112 init_waitqueue_head(&udev->nl_cmd_wq); 1113 spin_lock_init(&udev->nl_cmd_lock); 1114 1115 INIT_RADIX_TREE(&udev->data_blocks, GFP_KERNEL); 1116 1117 return &udev->se_dev; 1118 } 1119 1120 static int tcmu_irqcontrol(struct uio_info *info, s32 irq_on) 1121 { 1122 struct tcmu_dev *tcmu_dev = container_of(info, struct tcmu_dev, uio_info); 1123 1124 mutex_lock(&tcmu_dev->cmdr_lock); 1125 tcmu_handle_completions(tcmu_dev); 1126 mutex_unlock(&tcmu_dev->cmdr_lock); 1127 1128 return 0; 1129 } 1130 1131 /* 1132 * mmap code from uio.c. Copied here because we want to hook mmap() 1133 * and this stuff must come along. 1134 */ 1135 static int tcmu_find_mem_index(struct vm_area_struct *vma) 1136 { 1137 struct tcmu_dev *udev = vma->vm_private_data; 1138 struct uio_info *info = &udev->uio_info; 1139 1140 if (vma->vm_pgoff < MAX_UIO_MAPS) { 1141 if (info->mem[vma->vm_pgoff].size == 0) 1142 return -1; 1143 return (int)vma->vm_pgoff; 1144 } 1145 return -1; 1146 } 1147 1148 static struct page *tcmu_try_get_block_page(struct tcmu_dev *udev, uint32_t dbi) 1149 { 1150 struct page *page; 1151 int ret; 1152 1153 mutex_lock(&udev->cmdr_lock); 1154 page = tcmu_get_block_page(udev, dbi); 1155 if (likely(page)) { 1156 mutex_unlock(&udev->cmdr_lock); 1157 return page; 1158 } 1159 1160 /* 1161 * Normally it shouldn't be here: 1162 * Only when the userspace has touched the blocks which 1163 * are out of the tcmu_cmd's data iov[], and will return 1164 * one zeroed page. 1165 */ 1166 pr_warn("Block(%u) out of cmd's iov[] has been touched!\n", dbi); 1167 pr_warn("Mostly it will be a bug of userspace, please have a check!\n"); 1168 1169 if (dbi >= udev->dbi_thresh) { 1170 /* Extern the udev->dbi_thresh to dbi + 1 */ 1171 udev->dbi_thresh = dbi + 1; 1172 udev->dbi_max = dbi; 1173 } 1174 1175 page = radix_tree_lookup(&udev->data_blocks, dbi); 1176 if (!page) { 1177 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 1178 if (!page) { 1179 mutex_unlock(&udev->cmdr_lock); 1180 return NULL; 1181 } 1182 1183 ret = radix_tree_insert(&udev->data_blocks, dbi, page); 1184 if (ret) { 1185 mutex_unlock(&udev->cmdr_lock); 1186 __free_page(page); 1187 return NULL; 1188 } 1189 1190 /* 1191 * Since this case is rare in page fault routine, here we 1192 * will allow the global_db_count >= TCMU_GLOBAL_MAX_BLOCKS 1193 * to reduce possible page fault call trace. 1194 */ 1195 atomic_inc(&global_db_count); 1196 } 1197 mutex_unlock(&udev->cmdr_lock); 1198 1199 return page; 1200 } 1201 1202 static int tcmu_vma_fault(struct vm_fault *vmf) 1203 { 1204 struct tcmu_dev *udev = vmf->vma->vm_private_data; 1205 struct uio_info *info = &udev->uio_info; 1206 struct page *page; 1207 unsigned long offset; 1208 void *addr; 1209 1210 int mi = tcmu_find_mem_index(vmf->vma); 1211 if (mi < 0) 1212 return VM_FAULT_SIGBUS; 1213 1214 /* 1215 * We need to subtract mi because userspace uses offset = N*PAGE_SIZE 1216 * to use mem[N]. 1217 */ 1218 offset = (vmf->pgoff - mi) << PAGE_SHIFT; 1219 1220 if (offset < udev->data_off) { 1221 /* For the vmalloc()ed cmd area pages */ 1222 addr = (void *)(unsigned long)info->mem[mi].addr + offset; 1223 page = vmalloc_to_page(addr); 1224 } else { 1225 uint32_t dbi; 1226 1227 /* For the dynamically growing data area pages */ 1228 dbi = (offset - udev->data_off) / DATA_BLOCK_SIZE; 1229 page = tcmu_try_get_block_page(udev, dbi); 1230 if (!page) 1231 return VM_FAULT_NOPAGE; 1232 } 1233 1234 get_page(page); 1235 vmf->page = page; 1236 return 0; 1237 } 1238 1239 static const struct vm_operations_struct tcmu_vm_ops = { 1240 .fault = tcmu_vma_fault, 1241 }; 1242 1243 static int tcmu_mmap(struct uio_info *info, struct vm_area_struct *vma) 1244 { 1245 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info); 1246 1247 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; 1248 vma->vm_ops = &tcmu_vm_ops; 1249 1250 vma->vm_private_data = udev; 1251 1252 /* Ensure the mmap is exactly the right size */ 1253 if (vma_pages(vma) != (TCMU_RING_SIZE >> PAGE_SHIFT)) 1254 return -EINVAL; 1255 1256 return 0; 1257 } 1258 1259 static int tcmu_open(struct uio_info *info, struct inode *inode) 1260 { 1261 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info); 1262 1263 /* O_EXCL not supported for char devs, so fake it? */ 1264 if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags)) 1265 return -EBUSY; 1266 1267 udev->inode = inode; 1268 kref_get(&udev->kref); 1269 1270 pr_debug("open\n"); 1271 1272 return 0; 1273 } 1274 1275 static void tcmu_dev_call_rcu(struct rcu_head *p) 1276 { 1277 struct se_device *dev = container_of(p, struct se_device, rcu_head); 1278 struct tcmu_dev *udev = TCMU_DEV(dev); 1279 1280 kfree(udev->uio_info.name); 1281 kfree(udev->name); 1282 kfree(udev); 1283 } 1284 1285 static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd) 1286 { 1287 if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) { 1288 kmem_cache_free(tcmu_cmd_cache, cmd); 1289 return 0; 1290 } 1291 return -EINVAL; 1292 } 1293 1294 static void tcmu_blocks_release(struct tcmu_dev *udev) 1295 { 1296 int i; 1297 struct page *page; 1298 1299 /* Try to release all block pages */ 1300 mutex_lock(&udev->cmdr_lock); 1301 for (i = 0; i <= udev->dbi_max; i++) { 1302 page = radix_tree_delete(&udev->data_blocks, i); 1303 if (page) { 1304 __free_page(page); 1305 atomic_dec(&global_db_count); 1306 } 1307 } 1308 mutex_unlock(&udev->cmdr_lock); 1309 } 1310 1311 static void tcmu_dev_kref_release(struct kref *kref) 1312 { 1313 struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref); 1314 struct se_device *dev = &udev->se_dev; 1315 struct tcmu_cmd *cmd; 1316 bool all_expired = true; 1317 int i; 1318 1319 vfree(udev->mb_addr); 1320 udev->mb_addr = NULL; 1321 1322 /* Upper layer should drain all requests before calling this */ 1323 spin_lock_irq(&udev->commands_lock); 1324 idr_for_each_entry(&udev->commands, cmd, i) { 1325 if (tcmu_check_and_free_pending_cmd(cmd) != 0) 1326 all_expired = false; 1327 } 1328 idr_destroy(&udev->commands); 1329 spin_unlock_irq(&udev->commands_lock); 1330 WARN_ON(!all_expired); 1331 1332 tcmu_blocks_release(udev); 1333 1334 call_rcu(&dev->rcu_head, tcmu_dev_call_rcu); 1335 } 1336 1337 static int tcmu_release(struct uio_info *info, struct inode *inode) 1338 { 1339 struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info); 1340 1341 clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags); 1342 1343 pr_debug("close\n"); 1344 /* release ref from open */ 1345 kref_put(&udev->kref, tcmu_dev_kref_release); 1346 return 0; 1347 } 1348 1349 static void tcmu_init_genl_cmd_reply(struct tcmu_dev *udev, int cmd) 1350 { 1351 struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd; 1352 1353 if (!tcmu_kern_cmd_reply_supported) 1354 return; 1355 relock: 1356 spin_lock(&udev->nl_cmd_lock); 1357 1358 if (nl_cmd->cmd != TCMU_CMD_UNSPEC) { 1359 spin_unlock(&udev->nl_cmd_lock); 1360 pr_debug("sleeping for open nl cmd\n"); 1361 wait_event(udev->nl_cmd_wq, (nl_cmd->cmd == TCMU_CMD_UNSPEC)); 1362 goto relock; 1363 } 1364 1365 memset(nl_cmd, 0, sizeof(*nl_cmd)); 1366 nl_cmd->cmd = cmd; 1367 init_completion(&nl_cmd->complete); 1368 1369 spin_unlock(&udev->nl_cmd_lock); 1370 } 1371 1372 static int tcmu_wait_genl_cmd_reply(struct tcmu_dev *udev) 1373 { 1374 struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd; 1375 int ret; 1376 DEFINE_WAIT(__wait); 1377 1378 if (!tcmu_kern_cmd_reply_supported) 1379 return 0; 1380 1381 pr_debug("sleeping for nl reply\n"); 1382 wait_for_completion(&nl_cmd->complete); 1383 1384 spin_lock(&udev->nl_cmd_lock); 1385 nl_cmd->cmd = TCMU_CMD_UNSPEC; 1386 ret = nl_cmd->status; 1387 nl_cmd->status = 0; 1388 spin_unlock(&udev->nl_cmd_lock); 1389 1390 wake_up_all(&udev->nl_cmd_wq); 1391 1392 return ret;; 1393 } 1394 1395 static int tcmu_netlink_event(struct tcmu_dev *udev, enum tcmu_genl_cmd cmd, 1396 int reconfig_attr, const void *reconfig_data) 1397 { 1398 struct sk_buff *skb; 1399 void *msg_header; 1400 int ret = -ENOMEM; 1401 1402 skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL); 1403 if (!skb) 1404 return ret; 1405 1406 msg_header = genlmsg_put(skb, 0, 0, &tcmu_genl_family, 0, cmd); 1407 if (!msg_header) 1408 goto free_skb; 1409 1410 ret = nla_put_string(skb, TCMU_ATTR_DEVICE, udev->uio_info.name); 1411 if (ret < 0) 1412 goto free_skb; 1413 1414 ret = nla_put_u32(skb, TCMU_ATTR_MINOR, udev->uio_info.uio_dev->minor); 1415 if (ret < 0) 1416 goto free_skb; 1417 1418 ret = nla_put_u32(skb, TCMU_ATTR_DEVICE_ID, udev->se_dev.dev_index); 1419 if (ret < 0) 1420 goto free_skb; 1421 1422 if (cmd == TCMU_CMD_RECONFIG_DEVICE) { 1423 switch (reconfig_attr) { 1424 case TCMU_ATTR_DEV_CFG: 1425 ret = nla_put_string(skb, reconfig_attr, reconfig_data); 1426 break; 1427 case TCMU_ATTR_DEV_SIZE: 1428 ret = nla_put_u64_64bit(skb, reconfig_attr, 1429 *((u64 *)reconfig_data), 1430 TCMU_ATTR_PAD); 1431 break; 1432 case TCMU_ATTR_WRITECACHE: 1433 ret = nla_put_u8(skb, reconfig_attr, 1434 *((u8 *)reconfig_data)); 1435 break; 1436 default: 1437 BUG(); 1438 } 1439 1440 if (ret < 0) 1441 goto free_skb; 1442 } 1443 1444 genlmsg_end(skb, msg_header); 1445 1446 tcmu_init_genl_cmd_reply(udev, cmd); 1447 1448 ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0, 1449 TCMU_MCGRP_CONFIG, GFP_KERNEL); 1450 /* We don't care if no one is listening */ 1451 if (ret == -ESRCH) 1452 ret = 0; 1453 if (!ret) 1454 ret = tcmu_wait_genl_cmd_reply(udev); 1455 1456 return ret; 1457 free_skb: 1458 nlmsg_free(skb); 1459 return ret; 1460 } 1461 1462 static int tcmu_update_uio_info(struct tcmu_dev *udev) 1463 { 1464 struct tcmu_hba *hba = udev->hba->hba_ptr; 1465 struct uio_info *info; 1466 size_t size, used; 1467 char *str; 1468 1469 info = &udev->uio_info; 1470 size = snprintf(NULL, 0, "tcm-user/%u/%s/%s", hba->host_id, udev->name, 1471 udev->dev_config); 1472 size += 1; /* for \0 */ 1473 str = kmalloc(size, GFP_KERNEL); 1474 if (!str) 1475 return -ENOMEM; 1476 1477 used = snprintf(str, size, "tcm-user/%u/%s", hba->host_id, udev->name); 1478 if (udev->dev_config[0]) 1479 snprintf(str + used, size - used, "/%s", udev->dev_config); 1480 1481 /* If the old string exists, free it */ 1482 kfree(info->name); 1483 info->name = str; 1484 1485 return 0; 1486 } 1487 1488 static int tcmu_configure_device(struct se_device *dev) 1489 { 1490 struct tcmu_dev *udev = TCMU_DEV(dev); 1491 struct uio_info *info; 1492 struct tcmu_mailbox *mb; 1493 int ret = 0; 1494 1495 ret = tcmu_update_uio_info(udev); 1496 if (ret) 1497 return ret; 1498 1499 info = &udev->uio_info; 1500 1501 udev->mb_addr = vzalloc(CMDR_SIZE); 1502 if (!udev->mb_addr) { 1503 ret = -ENOMEM; 1504 goto err_vzalloc; 1505 } 1506 1507 /* mailbox fits in first part of CMDR space */ 1508 udev->cmdr_size = CMDR_SIZE - CMDR_OFF; 1509 udev->data_off = CMDR_SIZE; 1510 udev->data_size = DATA_SIZE; 1511 udev->dbi_thresh = 0; /* Default in Idle state */ 1512 udev->waiting_global = false; 1513 1514 /* Initialise the mailbox of the ring buffer */ 1515 mb = udev->mb_addr; 1516 mb->version = TCMU_MAILBOX_VERSION; 1517 mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC; 1518 mb->cmdr_off = CMDR_OFF; 1519 mb->cmdr_size = udev->cmdr_size; 1520 1521 WARN_ON(!PAGE_ALIGNED(udev->data_off)); 1522 WARN_ON(udev->data_size % PAGE_SIZE); 1523 WARN_ON(udev->data_size % DATA_BLOCK_SIZE); 1524 1525 info->version = __stringify(TCMU_MAILBOX_VERSION); 1526 1527 info->mem[0].name = "tcm-user command & data buffer"; 1528 info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr; 1529 info->mem[0].size = TCMU_RING_SIZE; 1530 info->mem[0].memtype = UIO_MEM_NONE; 1531 1532 info->irqcontrol = tcmu_irqcontrol; 1533 info->irq = UIO_IRQ_CUSTOM; 1534 1535 info->mmap = tcmu_mmap; 1536 info->open = tcmu_open; 1537 info->release = tcmu_release; 1538 1539 ret = uio_register_device(tcmu_root_device, info); 1540 if (ret) 1541 goto err_register; 1542 1543 /* User can set hw_block_size before enable the device */ 1544 if (dev->dev_attrib.hw_block_size == 0) 1545 dev->dev_attrib.hw_block_size = 512; 1546 /* Other attributes can be configured in userspace */ 1547 if (!dev->dev_attrib.hw_max_sectors) 1548 dev->dev_attrib.hw_max_sectors = 128; 1549 if (!dev->dev_attrib.emulate_write_cache) 1550 dev->dev_attrib.emulate_write_cache = 0; 1551 dev->dev_attrib.hw_queue_depth = 128; 1552 1553 /* 1554 * Get a ref incase userspace does a close on the uio device before 1555 * LIO has initiated tcmu_free_device. 1556 */ 1557 kref_get(&udev->kref); 1558 1559 ret = tcmu_netlink_event(udev, TCMU_CMD_ADDED_DEVICE, 0, NULL); 1560 if (ret) 1561 goto err_netlink; 1562 1563 mutex_lock(&root_udev_mutex); 1564 list_add(&udev->node, &root_udev); 1565 mutex_unlock(&root_udev_mutex); 1566 1567 return 0; 1568 1569 err_netlink: 1570 kref_put(&udev->kref, tcmu_dev_kref_release); 1571 uio_unregister_device(&udev->uio_info); 1572 err_register: 1573 vfree(udev->mb_addr); 1574 udev->mb_addr = NULL; 1575 err_vzalloc: 1576 kfree(info->name); 1577 info->name = NULL; 1578 1579 return ret; 1580 } 1581 1582 static bool tcmu_dev_configured(struct tcmu_dev *udev) 1583 { 1584 return udev->uio_info.uio_dev ? true : false; 1585 } 1586 1587 static void tcmu_free_device(struct se_device *dev) 1588 { 1589 struct tcmu_dev *udev = TCMU_DEV(dev); 1590 1591 /* release ref from init */ 1592 kref_put(&udev->kref, tcmu_dev_kref_release); 1593 } 1594 1595 static void tcmu_destroy_device(struct se_device *dev) 1596 { 1597 struct tcmu_dev *udev = TCMU_DEV(dev); 1598 1599 del_timer_sync(&udev->timeout); 1600 1601 mutex_lock(&root_udev_mutex); 1602 list_del(&udev->node); 1603 mutex_unlock(&root_udev_mutex); 1604 1605 tcmu_netlink_event(udev, TCMU_CMD_REMOVED_DEVICE, 0, NULL); 1606 1607 uio_unregister_device(&udev->uio_info); 1608 1609 /* release ref from configure */ 1610 kref_put(&udev->kref, tcmu_dev_kref_release); 1611 } 1612 1613 enum { 1614 Opt_dev_config, Opt_dev_size, Opt_hw_block_size, Opt_hw_max_sectors, 1615 Opt_err, 1616 }; 1617 1618 static match_table_t tokens = { 1619 {Opt_dev_config, "dev_config=%s"}, 1620 {Opt_dev_size, "dev_size=%u"}, 1621 {Opt_hw_block_size, "hw_block_size=%u"}, 1622 {Opt_hw_max_sectors, "hw_max_sectors=%u"}, 1623 {Opt_err, NULL} 1624 }; 1625 1626 static int tcmu_set_dev_attrib(substring_t *arg, u32 *dev_attrib) 1627 { 1628 unsigned long tmp_ul; 1629 char *arg_p; 1630 int ret; 1631 1632 arg_p = match_strdup(arg); 1633 if (!arg_p) 1634 return -ENOMEM; 1635 1636 ret = kstrtoul(arg_p, 0, &tmp_ul); 1637 kfree(arg_p); 1638 if (ret < 0) { 1639 pr_err("kstrtoul() failed for dev attrib\n"); 1640 return ret; 1641 } 1642 if (!tmp_ul) { 1643 pr_err("dev attrib must be nonzero\n"); 1644 return -EINVAL; 1645 } 1646 *dev_attrib = tmp_ul; 1647 return 0; 1648 } 1649 1650 static ssize_t tcmu_set_configfs_dev_params(struct se_device *dev, 1651 const char *page, ssize_t count) 1652 { 1653 struct tcmu_dev *udev = TCMU_DEV(dev); 1654 char *orig, *ptr, *opts, *arg_p; 1655 substring_t args[MAX_OPT_ARGS]; 1656 int ret = 0, token; 1657 1658 opts = kstrdup(page, GFP_KERNEL); 1659 if (!opts) 1660 return -ENOMEM; 1661 1662 orig = opts; 1663 1664 while ((ptr = strsep(&opts, ",\n")) != NULL) { 1665 if (!*ptr) 1666 continue; 1667 1668 token = match_token(ptr, tokens, args); 1669 switch (token) { 1670 case Opt_dev_config: 1671 if (match_strlcpy(udev->dev_config, &args[0], 1672 TCMU_CONFIG_LEN) == 0) { 1673 ret = -EINVAL; 1674 break; 1675 } 1676 pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config); 1677 break; 1678 case Opt_dev_size: 1679 arg_p = match_strdup(&args[0]); 1680 if (!arg_p) { 1681 ret = -ENOMEM; 1682 break; 1683 } 1684 ret = kstrtoul(arg_p, 0, (unsigned long *) &udev->dev_size); 1685 kfree(arg_p); 1686 if (ret < 0) 1687 pr_err("kstrtoul() failed for dev_size=\n"); 1688 break; 1689 case Opt_hw_block_size: 1690 ret = tcmu_set_dev_attrib(&args[0], 1691 &(dev->dev_attrib.hw_block_size)); 1692 break; 1693 case Opt_hw_max_sectors: 1694 ret = tcmu_set_dev_attrib(&args[0], 1695 &(dev->dev_attrib.hw_max_sectors)); 1696 break; 1697 default: 1698 break; 1699 } 1700 1701 if (ret) 1702 break; 1703 } 1704 1705 kfree(orig); 1706 return (!ret) ? count : ret; 1707 } 1708 1709 static ssize_t tcmu_show_configfs_dev_params(struct se_device *dev, char *b) 1710 { 1711 struct tcmu_dev *udev = TCMU_DEV(dev); 1712 ssize_t bl = 0; 1713 1714 bl = sprintf(b + bl, "Config: %s ", 1715 udev->dev_config[0] ? udev->dev_config : "NULL"); 1716 bl += sprintf(b + bl, "Size: %zu\n", udev->dev_size); 1717 1718 return bl; 1719 } 1720 1721 static sector_t tcmu_get_blocks(struct se_device *dev) 1722 { 1723 struct tcmu_dev *udev = TCMU_DEV(dev); 1724 1725 return div_u64(udev->dev_size - dev->dev_attrib.block_size, 1726 dev->dev_attrib.block_size); 1727 } 1728 1729 static sense_reason_t 1730 tcmu_parse_cdb(struct se_cmd *cmd) 1731 { 1732 return passthrough_parse_cdb(cmd, tcmu_queue_cmd); 1733 } 1734 1735 static ssize_t tcmu_cmd_time_out_show(struct config_item *item, char *page) 1736 { 1737 struct se_dev_attrib *da = container_of(to_config_group(item), 1738 struct se_dev_attrib, da_group); 1739 struct tcmu_dev *udev = container_of(da->da_dev, 1740 struct tcmu_dev, se_dev); 1741 1742 return snprintf(page, PAGE_SIZE, "%lu\n", udev->cmd_time_out / MSEC_PER_SEC); 1743 } 1744 1745 static ssize_t tcmu_cmd_time_out_store(struct config_item *item, const char *page, 1746 size_t count) 1747 { 1748 struct se_dev_attrib *da = container_of(to_config_group(item), 1749 struct se_dev_attrib, da_group); 1750 struct tcmu_dev *udev = container_of(da->da_dev, 1751 struct tcmu_dev, se_dev); 1752 u32 val; 1753 int ret; 1754 1755 if (da->da_dev->export_count) { 1756 pr_err("Unable to set tcmu cmd_time_out while exports exist\n"); 1757 return -EINVAL; 1758 } 1759 1760 ret = kstrtou32(page, 0, &val); 1761 if (ret < 0) 1762 return ret; 1763 1764 udev->cmd_time_out = val * MSEC_PER_SEC; 1765 return count; 1766 } 1767 CONFIGFS_ATTR(tcmu_, cmd_time_out); 1768 1769 static ssize_t tcmu_dev_config_show(struct config_item *item, char *page) 1770 { 1771 struct se_dev_attrib *da = container_of(to_config_group(item), 1772 struct se_dev_attrib, da_group); 1773 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); 1774 1775 return snprintf(page, PAGE_SIZE, "%s\n", udev->dev_config); 1776 } 1777 1778 static ssize_t tcmu_dev_config_store(struct config_item *item, const char *page, 1779 size_t count) 1780 { 1781 struct se_dev_attrib *da = container_of(to_config_group(item), 1782 struct se_dev_attrib, da_group); 1783 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); 1784 int ret, len; 1785 1786 len = strlen(page); 1787 if (!len || len > TCMU_CONFIG_LEN - 1) 1788 return -EINVAL; 1789 1790 /* Check if device has been configured before */ 1791 if (tcmu_dev_configured(udev)) { 1792 ret = tcmu_netlink_event(udev, TCMU_CMD_RECONFIG_DEVICE, 1793 TCMU_ATTR_DEV_CFG, page); 1794 if (ret) { 1795 pr_err("Unable to reconfigure device\n"); 1796 return ret; 1797 } 1798 strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN); 1799 1800 ret = tcmu_update_uio_info(udev); 1801 if (ret) 1802 return ret; 1803 return count; 1804 } 1805 strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN); 1806 1807 return count; 1808 } 1809 CONFIGFS_ATTR(tcmu_, dev_config); 1810 1811 static ssize_t tcmu_dev_size_show(struct config_item *item, char *page) 1812 { 1813 struct se_dev_attrib *da = container_of(to_config_group(item), 1814 struct se_dev_attrib, da_group); 1815 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); 1816 1817 return snprintf(page, PAGE_SIZE, "%zu\n", udev->dev_size); 1818 } 1819 1820 static ssize_t tcmu_dev_size_store(struct config_item *item, const char *page, 1821 size_t count) 1822 { 1823 struct se_dev_attrib *da = container_of(to_config_group(item), 1824 struct se_dev_attrib, da_group); 1825 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); 1826 u64 val; 1827 int ret; 1828 1829 ret = kstrtou64(page, 0, &val); 1830 if (ret < 0) 1831 return ret; 1832 1833 /* Check if device has been configured before */ 1834 if (tcmu_dev_configured(udev)) { 1835 ret = tcmu_netlink_event(udev, TCMU_CMD_RECONFIG_DEVICE, 1836 TCMU_ATTR_DEV_SIZE, &val); 1837 if (ret) { 1838 pr_err("Unable to reconfigure device\n"); 1839 return ret; 1840 } 1841 } 1842 udev->dev_size = val; 1843 return count; 1844 } 1845 CONFIGFS_ATTR(tcmu_, dev_size); 1846 1847 static ssize_t tcmu_emulate_write_cache_show(struct config_item *item, 1848 char *page) 1849 { 1850 struct se_dev_attrib *da = container_of(to_config_group(item), 1851 struct se_dev_attrib, da_group); 1852 1853 return snprintf(page, PAGE_SIZE, "%i\n", da->emulate_write_cache); 1854 } 1855 1856 static ssize_t tcmu_emulate_write_cache_store(struct config_item *item, 1857 const char *page, size_t count) 1858 { 1859 struct se_dev_attrib *da = container_of(to_config_group(item), 1860 struct se_dev_attrib, da_group); 1861 struct tcmu_dev *udev = TCMU_DEV(da->da_dev); 1862 u8 val; 1863 int ret; 1864 1865 ret = kstrtou8(page, 0, &val); 1866 if (ret < 0) 1867 return ret; 1868 1869 /* Check if device has been configured before */ 1870 if (tcmu_dev_configured(udev)) { 1871 ret = tcmu_netlink_event(udev, TCMU_CMD_RECONFIG_DEVICE, 1872 TCMU_ATTR_WRITECACHE, &val); 1873 if (ret) { 1874 pr_err("Unable to reconfigure device\n"); 1875 return ret; 1876 } 1877 } 1878 1879 da->emulate_write_cache = val; 1880 return count; 1881 } 1882 CONFIGFS_ATTR(tcmu_, emulate_write_cache); 1883 1884 static struct configfs_attribute *tcmu_attrib_attrs[] = { 1885 &tcmu_attr_cmd_time_out, 1886 &tcmu_attr_dev_config, 1887 &tcmu_attr_dev_size, 1888 &tcmu_attr_emulate_write_cache, 1889 NULL, 1890 }; 1891 1892 static struct configfs_attribute **tcmu_attrs; 1893 1894 static struct target_backend_ops tcmu_ops = { 1895 .name = "user", 1896 .owner = THIS_MODULE, 1897 .transport_flags = TRANSPORT_FLAG_PASSTHROUGH, 1898 .attach_hba = tcmu_attach_hba, 1899 .detach_hba = tcmu_detach_hba, 1900 .alloc_device = tcmu_alloc_device, 1901 .configure_device = tcmu_configure_device, 1902 .destroy_device = tcmu_destroy_device, 1903 .free_device = tcmu_free_device, 1904 .parse_cdb = tcmu_parse_cdb, 1905 .set_configfs_dev_params = tcmu_set_configfs_dev_params, 1906 .show_configfs_dev_params = tcmu_show_configfs_dev_params, 1907 .get_device_type = sbc_get_device_type, 1908 .get_blocks = tcmu_get_blocks, 1909 .tb_dev_attrib_attrs = NULL, 1910 }; 1911 1912 static int unmap_thread_fn(void *data) 1913 { 1914 struct tcmu_dev *udev; 1915 loff_t off; 1916 uint32_t start, end, block; 1917 struct page *page; 1918 int i; 1919 1920 while (!kthread_should_stop()) { 1921 DEFINE_WAIT(__wait); 1922 1923 prepare_to_wait(&unmap_wait, &__wait, TASK_INTERRUPTIBLE); 1924 schedule(); 1925 finish_wait(&unmap_wait, &__wait); 1926 1927 if (kthread_should_stop()) 1928 break; 1929 1930 mutex_lock(&root_udev_mutex); 1931 list_for_each_entry(udev, &root_udev, node) { 1932 mutex_lock(&udev->cmdr_lock); 1933 1934 /* Try to complete the finished commands first */ 1935 tcmu_handle_completions(udev); 1936 1937 /* Skip the udevs waiting the global pool or in idle */ 1938 if (udev->waiting_global || !udev->dbi_thresh) { 1939 mutex_unlock(&udev->cmdr_lock); 1940 continue; 1941 } 1942 1943 end = udev->dbi_max + 1; 1944 block = find_last_bit(udev->data_bitmap, end); 1945 if (block == udev->dbi_max) { 1946 /* 1947 * The last bit is dbi_max, so there is 1948 * no need to shrink any blocks. 1949 */ 1950 mutex_unlock(&udev->cmdr_lock); 1951 continue; 1952 } else if (block == end) { 1953 /* The current udev will goto idle state */ 1954 udev->dbi_thresh = start = 0; 1955 udev->dbi_max = 0; 1956 } else { 1957 udev->dbi_thresh = start = block + 1; 1958 udev->dbi_max = block; 1959 } 1960 1961 /* Here will truncate the data area from off */ 1962 off = udev->data_off + start * DATA_BLOCK_SIZE; 1963 unmap_mapping_range(udev->inode->i_mapping, off, 0, 1); 1964 1965 /* Release the block pages */ 1966 for (i = start; i < end; i++) { 1967 page = radix_tree_delete(&udev->data_blocks, i); 1968 if (page) { 1969 __free_page(page); 1970 atomic_dec(&global_db_count); 1971 } 1972 } 1973 mutex_unlock(&udev->cmdr_lock); 1974 } 1975 1976 /* 1977 * Try to wake up the udevs who are waiting 1978 * for the global data pool. 1979 */ 1980 list_for_each_entry(udev, &root_udev, node) { 1981 if (udev->waiting_global) 1982 wake_up(&udev->wait_cmdr); 1983 } 1984 mutex_unlock(&root_udev_mutex); 1985 } 1986 1987 return 0; 1988 } 1989 1990 static int __init tcmu_module_init(void) 1991 { 1992 int ret, i, k, len = 0; 1993 1994 BUILD_BUG_ON((sizeof(struct tcmu_cmd_entry) % TCMU_OP_ALIGN_SIZE) != 0); 1995 1996 tcmu_cmd_cache = kmem_cache_create("tcmu_cmd_cache", 1997 sizeof(struct tcmu_cmd), 1998 __alignof__(struct tcmu_cmd), 1999 0, NULL); 2000 if (!tcmu_cmd_cache) 2001 return -ENOMEM; 2002 2003 tcmu_root_device = root_device_register("tcm_user"); 2004 if (IS_ERR(tcmu_root_device)) { 2005 ret = PTR_ERR(tcmu_root_device); 2006 goto out_free_cache; 2007 } 2008 2009 ret = genl_register_family(&tcmu_genl_family); 2010 if (ret < 0) { 2011 goto out_unreg_device; 2012 } 2013 2014 for (i = 0; passthrough_attrib_attrs[i] != NULL; i++) { 2015 len += sizeof(struct configfs_attribute *); 2016 } 2017 for (i = 0; tcmu_attrib_attrs[i] != NULL; i++) { 2018 len += sizeof(struct configfs_attribute *); 2019 } 2020 len += sizeof(struct configfs_attribute *); 2021 2022 tcmu_attrs = kzalloc(len, GFP_KERNEL); 2023 if (!tcmu_attrs) { 2024 ret = -ENOMEM; 2025 goto out_unreg_genl; 2026 } 2027 2028 for (i = 0; passthrough_attrib_attrs[i] != NULL; i++) { 2029 tcmu_attrs[i] = passthrough_attrib_attrs[i]; 2030 } 2031 for (k = 0; tcmu_attrib_attrs[k] != NULL; k++) { 2032 tcmu_attrs[i] = tcmu_attrib_attrs[k]; 2033 i++; 2034 } 2035 tcmu_ops.tb_dev_attrib_attrs = tcmu_attrs; 2036 2037 ret = transport_backend_register(&tcmu_ops); 2038 if (ret) 2039 goto out_attrs; 2040 2041 init_waitqueue_head(&unmap_wait); 2042 unmap_thread = kthread_run(unmap_thread_fn, NULL, "tcmu_unmap"); 2043 if (IS_ERR(unmap_thread)) { 2044 ret = PTR_ERR(unmap_thread); 2045 goto out_unreg_transport; 2046 } 2047 2048 return 0; 2049 2050 out_unreg_transport: 2051 target_backend_unregister(&tcmu_ops); 2052 out_attrs: 2053 kfree(tcmu_attrs); 2054 out_unreg_genl: 2055 genl_unregister_family(&tcmu_genl_family); 2056 out_unreg_device: 2057 root_device_unregister(tcmu_root_device); 2058 out_free_cache: 2059 kmem_cache_destroy(tcmu_cmd_cache); 2060 2061 return ret; 2062 } 2063 2064 static void __exit tcmu_module_exit(void) 2065 { 2066 kthread_stop(unmap_thread); 2067 target_backend_unregister(&tcmu_ops); 2068 kfree(tcmu_attrs); 2069 genl_unregister_family(&tcmu_genl_family); 2070 root_device_unregister(tcmu_root_device); 2071 kmem_cache_destroy(tcmu_cmd_cache); 2072 } 2073 2074 MODULE_DESCRIPTION("TCM USER subsystem plugin"); 2075 MODULE_AUTHOR("Shaohua Li <shli@kernel.org>"); 2076 MODULE_AUTHOR("Andy Grover <agrover@redhat.com>"); 2077 MODULE_LICENSE("GPL"); 2078 2079 module_init(tcmu_module_init); 2080 module_exit(tcmu_module_exit); 2081