1 /*
2  * Copyright (C) 2013 Shaohua Li <shli@kernel.org>
3  * Copyright (C) 2014 Red Hat, Inc.
4  * Copyright (C) 2015 Arrikto, Inc.
5  * Copyright (C) 2017 Chinamobile, Inc.
6  *
7  * This program is free software; you can redistribute it and/or modify it
8  * under the terms and conditions of the GNU General Public License,
9  * version 2, as published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14  * more details.
15  *
16  * You should have received a copy of the GNU General Public License along with
17  * this program; if not, write to the Free Software Foundation, Inc.,
18  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19  */
20 
21 #include <linux/spinlock.h>
22 #include <linux/module.h>
23 #include <linux/idr.h>
24 #include <linux/kernel.h>
25 #include <linux/timer.h>
26 #include <linux/parser.h>
27 #include <linux/vmalloc.h>
28 #include <linux/uio_driver.h>
29 #include <linux/radix-tree.h>
30 #include <linux/stringify.h>
31 #include <linux/bitops.h>
32 #include <linux/highmem.h>
33 #include <linux/configfs.h>
34 #include <linux/mutex.h>
35 #include <linux/workqueue.h>
36 #include <net/genetlink.h>
37 #include <scsi/scsi_common.h>
38 #include <scsi/scsi_proto.h>
39 #include <target/target_core_base.h>
40 #include <target/target_core_fabric.h>
41 #include <target/target_core_backend.h>
42 
43 #include <linux/target_core_user.h>
44 
45 /**
46  * DOC: Userspace I/O
47  * Userspace I/O
48  * -------------
49  *
50  * Define a shared-memory interface for LIO to pass SCSI commands and
51  * data to userspace for processing. This is to allow backends that
52  * are too complex for in-kernel support to be possible.
53  *
54  * It uses the UIO framework to do a lot of the device-creation and
55  * introspection work for us.
56  *
57  * See the .h file for how the ring is laid out. Note that while the
58  * command ring is defined, the particulars of the data area are
59  * not. Offset values in the command entry point to other locations
60  * internal to the mmap-ed area. There is separate space outside the
61  * command ring for data buffers. This leaves maximum flexibility for
62  * moving buffer allocations, or even page flipping or other
63  * allocation techniques, without altering the command ring layout.
64  *
65  * SECURITY:
66  * The user process must be assumed to be malicious. There's no way to
67  * prevent it breaking the command ring protocol if it wants, but in
68  * order to prevent other issues we must only ever read *data* from
69  * the shared memory area, not offsets or sizes. This applies to
70  * command ring entries as well as the mailbox. Extra code needed for
71  * this may have a 'UAM' comment.
72  */
73 
74 #define TCMU_TIME_OUT (30 * MSEC_PER_SEC)
75 
76 /* For cmd area, the size is fixed 8MB */
77 #define CMDR_SIZE (8 * 1024 * 1024)
78 
79 /*
80  * For data area, the block size is PAGE_SIZE and
81  * the total size is 256K * PAGE_SIZE.
82  */
83 #define DATA_BLOCK_SIZE PAGE_SIZE
84 #define DATA_BLOCK_SHIFT PAGE_SHIFT
85 #define DATA_BLOCK_BITS_DEF (256 * 1024)
86 
87 #define TCMU_MBS_TO_BLOCKS(_mbs) (_mbs << (20 - DATA_BLOCK_SHIFT))
88 #define TCMU_BLOCKS_TO_MBS(_blocks) (_blocks >> (20 - DATA_BLOCK_SHIFT))
89 
90 /*
91  * Default number of global data blocks(512K * PAGE_SIZE)
92  * when the unmap thread will be started.
93  */
94 #define TCMU_GLOBAL_MAX_BLOCKS_DEF (512 * 1024)
95 
96 static u8 tcmu_kern_cmd_reply_supported;
97 static u8 tcmu_netlink_blocked;
98 
99 static struct device *tcmu_root_device;
100 
101 struct tcmu_hba {
102 	u32 host_id;
103 };
104 
105 #define TCMU_CONFIG_LEN 256
106 
107 static DEFINE_MUTEX(tcmu_nl_cmd_mutex);
108 static LIST_HEAD(tcmu_nl_cmd_list);
109 
110 struct tcmu_dev;
111 
112 struct tcmu_nl_cmd {
113 	/* wake up thread waiting for reply */
114 	struct completion complete;
115 	struct list_head nl_list;
116 	struct tcmu_dev *udev;
117 	int cmd;
118 	int status;
119 };
120 
121 struct tcmu_dev {
122 	struct list_head node;
123 	struct kref kref;
124 
125 	struct se_device se_dev;
126 
127 	char *name;
128 	struct se_hba *hba;
129 
130 #define TCMU_DEV_BIT_OPEN 0
131 #define TCMU_DEV_BIT_BROKEN 1
132 #define TCMU_DEV_BIT_BLOCKED 2
133 	unsigned long flags;
134 
135 	struct uio_info uio_info;
136 
137 	struct inode *inode;
138 
139 	struct tcmu_mailbox *mb_addr;
140 	uint64_t dev_size;
141 	u32 cmdr_size;
142 	u32 cmdr_last_cleaned;
143 	/* Offset of data area from start of mb */
144 	/* Must add data_off and mb_addr to get the address */
145 	size_t data_off;
146 	size_t data_size;
147 	uint32_t max_blocks;
148 	size_t ring_size;
149 
150 	struct mutex cmdr_lock;
151 	struct list_head qfull_queue;
152 
153 	uint32_t dbi_max;
154 	uint32_t dbi_thresh;
155 	unsigned long *data_bitmap;
156 	struct radix_tree_root data_blocks;
157 
158 	struct idr commands;
159 
160 	struct timer_list cmd_timer;
161 	unsigned int cmd_time_out;
162 	struct list_head inflight_queue;
163 
164 	struct timer_list qfull_timer;
165 	int qfull_time_out;
166 
167 	struct list_head timedout_entry;
168 
169 	struct tcmu_nl_cmd curr_nl_cmd;
170 
171 	char dev_config[TCMU_CONFIG_LEN];
172 
173 	int nl_reply_supported;
174 };
175 
176 #define TCMU_DEV(_se_dev) container_of(_se_dev, struct tcmu_dev, se_dev)
177 
178 #define CMDR_OFF sizeof(struct tcmu_mailbox)
179 
180 struct tcmu_cmd {
181 	struct se_cmd *se_cmd;
182 	struct tcmu_dev *tcmu_dev;
183 	struct list_head queue_entry;
184 
185 	uint16_t cmd_id;
186 
187 	/* Can't use se_cmd when cleaning up expired cmds, because if
188 	   cmd has been completed then accessing se_cmd is off limits */
189 	uint32_t dbi_cnt;
190 	uint32_t dbi_cur;
191 	uint32_t *dbi;
192 
193 	unsigned long deadline;
194 
195 #define TCMU_CMD_BIT_EXPIRED 0
196 #define TCMU_CMD_BIT_INFLIGHT 1
197 	unsigned long flags;
198 };
199 /*
200  * To avoid dead lock the mutex lock order should always be:
201  *
202  * mutex_lock(&root_udev_mutex);
203  * ...
204  * mutex_lock(&tcmu_dev->cmdr_lock);
205  * mutex_unlock(&tcmu_dev->cmdr_lock);
206  * ...
207  * mutex_unlock(&root_udev_mutex);
208  */
209 static DEFINE_MUTEX(root_udev_mutex);
210 static LIST_HEAD(root_udev);
211 
212 static DEFINE_SPINLOCK(timed_out_udevs_lock);
213 static LIST_HEAD(timed_out_udevs);
214 
215 static struct kmem_cache *tcmu_cmd_cache;
216 
217 static atomic_t global_db_count = ATOMIC_INIT(0);
218 static struct delayed_work tcmu_unmap_work;
219 static int tcmu_global_max_blocks = TCMU_GLOBAL_MAX_BLOCKS_DEF;
220 
221 static int tcmu_set_global_max_data_area(const char *str,
222 					 const struct kernel_param *kp)
223 {
224 	int ret, max_area_mb;
225 
226 	ret = kstrtoint(str, 10, &max_area_mb);
227 	if (ret)
228 		return -EINVAL;
229 
230 	if (max_area_mb <= 0) {
231 		pr_err("global_max_data_area must be larger than 0.\n");
232 		return -EINVAL;
233 	}
234 
235 	tcmu_global_max_blocks = TCMU_MBS_TO_BLOCKS(max_area_mb);
236 	if (atomic_read(&global_db_count) > tcmu_global_max_blocks)
237 		schedule_delayed_work(&tcmu_unmap_work, 0);
238 	else
239 		cancel_delayed_work_sync(&tcmu_unmap_work);
240 
241 	return 0;
242 }
243 
244 static int tcmu_get_global_max_data_area(char *buffer,
245 					 const struct kernel_param *kp)
246 {
247 	return sprintf(buffer, "%d", TCMU_BLOCKS_TO_MBS(tcmu_global_max_blocks));
248 }
249 
250 static const struct kernel_param_ops tcmu_global_max_data_area_op = {
251 	.set = tcmu_set_global_max_data_area,
252 	.get = tcmu_get_global_max_data_area,
253 };
254 
255 module_param_cb(global_max_data_area_mb, &tcmu_global_max_data_area_op, NULL,
256 		S_IWUSR | S_IRUGO);
257 MODULE_PARM_DESC(global_max_data_area_mb,
258 		 "Max MBs allowed to be allocated to all the tcmu device's "
259 		 "data areas.");
260 
261 static int tcmu_get_block_netlink(char *buffer,
262 				  const struct kernel_param *kp)
263 {
264 	return sprintf(buffer, "%s\n", tcmu_netlink_blocked ?
265 		       "blocked" : "unblocked");
266 }
267 
268 static int tcmu_set_block_netlink(const char *str,
269 				  const struct kernel_param *kp)
270 {
271 	int ret;
272 	u8 val;
273 
274 	ret = kstrtou8(str, 0, &val);
275 	if (ret < 0)
276 		return ret;
277 
278 	if (val > 1) {
279 		pr_err("Invalid block netlink value %u\n", val);
280 		return -EINVAL;
281 	}
282 
283 	tcmu_netlink_blocked = val;
284 	return 0;
285 }
286 
287 static const struct kernel_param_ops tcmu_block_netlink_op = {
288 	.set = tcmu_set_block_netlink,
289 	.get = tcmu_get_block_netlink,
290 };
291 
292 module_param_cb(block_netlink, &tcmu_block_netlink_op, NULL, S_IWUSR | S_IRUGO);
293 MODULE_PARM_DESC(block_netlink, "Block new netlink commands.");
294 
295 static int tcmu_fail_netlink_cmd(struct tcmu_nl_cmd *nl_cmd)
296 {
297 	struct tcmu_dev *udev = nl_cmd->udev;
298 
299 	if (!tcmu_netlink_blocked) {
300 		pr_err("Could not reset device's netlink interface. Netlink is not blocked.\n");
301 		return -EBUSY;
302 	}
303 
304 	if (nl_cmd->cmd != TCMU_CMD_UNSPEC) {
305 		pr_debug("Aborting nl cmd %d on %s\n", nl_cmd->cmd, udev->name);
306 		nl_cmd->status = -EINTR;
307 		list_del(&nl_cmd->nl_list);
308 		complete(&nl_cmd->complete);
309 	}
310 	return 0;
311 }
312 
313 static int tcmu_set_reset_netlink(const char *str,
314 				  const struct kernel_param *kp)
315 {
316 	struct tcmu_nl_cmd *nl_cmd, *tmp_cmd;
317 	int ret;
318 	u8 val;
319 
320 	ret = kstrtou8(str, 0, &val);
321 	if (ret < 0)
322 		return ret;
323 
324 	if (val != 1) {
325 		pr_err("Invalid reset netlink value %u\n", val);
326 		return -EINVAL;
327 	}
328 
329 	mutex_lock(&tcmu_nl_cmd_mutex);
330 	list_for_each_entry_safe(nl_cmd, tmp_cmd, &tcmu_nl_cmd_list, nl_list) {
331 		ret = tcmu_fail_netlink_cmd(nl_cmd);
332 		if (ret)
333 			break;
334 	}
335 	mutex_unlock(&tcmu_nl_cmd_mutex);
336 
337 	return ret;
338 }
339 
340 static const struct kernel_param_ops tcmu_reset_netlink_op = {
341 	.set = tcmu_set_reset_netlink,
342 };
343 
344 module_param_cb(reset_netlink, &tcmu_reset_netlink_op, NULL, S_IWUSR);
345 MODULE_PARM_DESC(reset_netlink, "Reset netlink commands.");
346 
347 /* multicast group */
348 enum tcmu_multicast_groups {
349 	TCMU_MCGRP_CONFIG,
350 };
351 
352 static const struct genl_multicast_group tcmu_mcgrps[] = {
353 	[TCMU_MCGRP_CONFIG] = { .name = "config", },
354 };
355 
356 static struct nla_policy tcmu_attr_policy[TCMU_ATTR_MAX+1] = {
357 	[TCMU_ATTR_DEVICE]	= { .type = NLA_STRING },
358 	[TCMU_ATTR_MINOR]	= { .type = NLA_U32 },
359 	[TCMU_ATTR_CMD_STATUS]	= { .type = NLA_S32 },
360 	[TCMU_ATTR_DEVICE_ID]	= { .type = NLA_U32 },
361 	[TCMU_ATTR_SUPP_KERN_CMD_REPLY] = { .type = NLA_U8 },
362 };
363 
364 static int tcmu_genl_cmd_done(struct genl_info *info, int completed_cmd)
365 {
366 	struct tcmu_dev *udev = NULL;
367 	struct tcmu_nl_cmd *nl_cmd;
368 	int dev_id, rc, ret = 0;
369 
370 	if (!info->attrs[TCMU_ATTR_CMD_STATUS] ||
371 	    !info->attrs[TCMU_ATTR_DEVICE_ID]) {
372 		printk(KERN_ERR "TCMU_ATTR_CMD_STATUS or TCMU_ATTR_DEVICE_ID not set, doing nothing\n");
373 		return -EINVAL;
374         }
375 
376 	dev_id = nla_get_u32(info->attrs[TCMU_ATTR_DEVICE_ID]);
377 	rc = nla_get_s32(info->attrs[TCMU_ATTR_CMD_STATUS]);
378 
379 	mutex_lock(&tcmu_nl_cmd_mutex);
380 	list_for_each_entry(nl_cmd, &tcmu_nl_cmd_list, nl_list) {
381 		if (nl_cmd->udev->se_dev.dev_index == dev_id) {
382 			udev = nl_cmd->udev;
383 			break;
384 		}
385 	}
386 
387 	if (!udev) {
388 		pr_err("tcmu nl cmd %u/%d completion could not find device with dev id %u.\n",
389 		       completed_cmd, rc, dev_id);
390 		ret = -ENODEV;
391 		goto unlock;
392 	}
393 	list_del(&nl_cmd->nl_list);
394 
395 	pr_debug("%s genl cmd done got id %d curr %d done %d rc %d stat %d\n",
396 		 udev->name, dev_id, nl_cmd->cmd, completed_cmd, rc,
397 		 nl_cmd->status);
398 
399 	if (nl_cmd->cmd != completed_cmd) {
400 		pr_err("Mismatched commands on %s (Expecting reply for %d. Current %d).\n",
401 		       udev->name, completed_cmd, nl_cmd->cmd);
402 		ret = -EINVAL;
403 		goto unlock;
404 	}
405 
406 	nl_cmd->status = rc;
407 	complete(&nl_cmd->complete);
408 unlock:
409 	mutex_unlock(&tcmu_nl_cmd_mutex);
410 	return ret;
411 }
412 
413 static int tcmu_genl_rm_dev_done(struct sk_buff *skb, struct genl_info *info)
414 {
415 	return tcmu_genl_cmd_done(info, TCMU_CMD_REMOVED_DEVICE);
416 }
417 
418 static int tcmu_genl_add_dev_done(struct sk_buff *skb, struct genl_info *info)
419 {
420 	return tcmu_genl_cmd_done(info, TCMU_CMD_ADDED_DEVICE);
421 }
422 
423 static int tcmu_genl_reconfig_dev_done(struct sk_buff *skb,
424 				       struct genl_info *info)
425 {
426 	return tcmu_genl_cmd_done(info, TCMU_CMD_RECONFIG_DEVICE);
427 }
428 
429 static int tcmu_genl_set_features(struct sk_buff *skb, struct genl_info *info)
430 {
431 	if (info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]) {
432 		tcmu_kern_cmd_reply_supported  =
433 			nla_get_u8(info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]);
434 		printk(KERN_INFO "tcmu daemon: command reply support %u.\n",
435 		       tcmu_kern_cmd_reply_supported);
436 	}
437 
438 	return 0;
439 }
440 
441 static const struct genl_ops tcmu_genl_ops[] = {
442 	{
443 		.cmd	= TCMU_CMD_SET_FEATURES,
444 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
445 		.flags	= GENL_ADMIN_PERM,
446 		.doit	= tcmu_genl_set_features,
447 	},
448 	{
449 		.cmd	= TCMU_CMD_ADDED_DEVICE_DONE,
450 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
451 		.flags	= GENL_ADMIN_PERM,
452 		.doit	= tcmu_genl_add_dev_done,
453 	},
454 	{
455 		.cmd	= TCMU_CMD_REMOVED_DEVICE_DONE,
456 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
457 		.flags	= GENL_ADMIN_PERM,
458 		.doit	= tcmu_genl_rm_dev_done,
459 	},
460 	{
461 		.cmd	= TCMU_CMD_RECONFIG_DEVICE_DONE,
462 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
463 		.flags	= GENL_ADMIN_PERM,
464 		.doit	= tcmu_genl_reconfig_dev_done,
465 	},
466 };
467 
468 /* Our generic netlink family */
469 static struct genl_family tcmu_genl_family __ro_after_init = {
470 	.module = THIS_MODULE,
471 	.hdrsize = 0,
472 	.name = "TCM-USER",
473 	.version = 2,
474 	.maxattr = TCMU_ATTR_MAX,
475 	.policy = tcmu_attr_policy,
476 	.mcgrps = tcmu_mcgrps,
477 	.n_mcgrps = ARRAY_SIZE(tcmu_mcgrps),
478 	.netnsok = true,
479 	.ops = tcmu_genl_ops,
480 	.n_ops = ARRAY_SIZE(tcmu_genl_ops),
481 };
482 
483 #define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index))
484 #define tcmu_cmd_reset_dbi_cur(cmd) tcmu_cmd_set_dbi_cur(cmd, 0)
485 #define tcmu_cmd_set_dbi(cmd, index) ((cmd)->dbi[(cmd)->dbi_cur++] = (index))
486 #define tcmu_cmd_get_dbi(cmd) ((cmd)->dbi[(cmd)->dbi_cur++])
487 
488 static void tcmu_cmd_free_data(struct tcmu_cmd *tcmu_cmd, uint32_t len)
489 {
490 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
491 	uint32_t i;
492 
493 	for (i = 0; i < len; i++)
494 		clear_bit(tcmu_cmd->dbi[i], udev->data_bitmap);
495 }
496 
497 static inline bool tcmu_get_empty_block(struct tcmu_dev *udev,
498 					struct tcmu_cmd *tcmu_cmd)
499 {
500 	struct page *page;
501 	int ret, dbi;
502 
503 	dbi = find_first_zero_bit(udev->data_bitmap, udev->dbi_thresh);
504 	if (dbi == udev->dbi_thresh)
505 		return false;
506 
507 	page = radix_tree_lookup(&udev->data_blocks, dbi);
508 	if (!page) {
509 		if (atomic_add_return(1, &global_db_count) >
510 				      tcmu_global_max_blocks)
511 			schedule_delayed_work(&tcmu_unmap_work, 0);
512 
513 		/* try to get new page from the mm */
514 		page = alloc_page(GFP_KERNEL);
515 		if (!page)
516 			goto err_alloc;
517 
518 		ret = radix_tree_insert(&udev->data_blocks, dbi, page);
519 		if (ret)
520 			goto err_insert;
521 	}
522 
523 	if (dbi > udev->dbi_max)
524 		udev->dbi_max = dbi;
525 
526 	set_bit(dbi, udev->data_bitmap);
527 	tcmu_cmd_set_dbi(tcmu_cmd, dbi);
528 
529 	return true;
530 err_insert:
531 	__free_page(page);
532 err_alloc:
533 	atomic_dec(&global_db_count);
534 	return false;
535 }
536 
537 static bool tcmu_get_empty_blocks(struct tcmu_dev *udev,
538 				  struct tcmu_cmd *tcmu_cmd)
539 {
540 	int i;
541 
542 	for (i = tcmu_cmd->dbi_cur; i < tcmu_cmd->dbi_cnt; i++) {
543 		if (!tcmu_get_empty_block(udev, tcmu_cmd))
544 			return false;
545 	}
546 	return true;
547 }
548 
549 static inline struct page *
550 tcmu_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
551 {
552 	return radix_tree_lookup(&udev->data_blocks, dbi);
553 }
554 
555 static inline void tcmu_free_cmd(struct tcmu_cmd *tcmu_cmd)
556 {
557 	kfree(tcmu_cmd->dbi);
558 	kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
559 }
560 
561 static inline size_t tcmu_cmd_get_data_length(struct tcmu_cmd *tcmu_cmd)
562 {
563 	struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
564 	size_t data_length = round_up(se_cmd->data_length, DATA_BLOCK_SIZE);
565 
566 	if (se_cmd->se_cmd_flags & SCF_BIDI) {
567 		BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
568 		data_length += round_up(se_cmd->t_bidi_data_sg->length,
569 				DATA_BLOCK_SIZE);
570 	}
571 
572 	return data_length;
573 }
574 
575 static inline uint32_t tcmu_cmd_get_block_cnt(struct tcmu_cmd *tcmu_cmd)
576 {
577 	size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd);
578 
579 	return data_length / DATA_BLOCK_SIZE;
580 }
581 
582 static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
583 {
584 	struct se_device *se_dev = se_cmd->se_dev;
585 	struct tcmu_dev *udev = TCMU_DEV(se_dev);
586 	struct tcmu_cmd *tcmu_cmd;
587 
588 	tcmu_cmd = kmem_cache_zalloc(tcmu_cmd_cache, GFP_KERNEL);
589 	if (!tcmu_cmd)
590 		return NULL;
591 
592 	INIT_LIST_HEAD(&tcmu_cmd->queue_entry);
593 	tcmu_cmd->se_cmd = se_cmd;
594 	tcmu_cmd->tcmu_dev = udev;
595 
596 	tcmu_cmd_reset_dbi_cur(tcmu_cmd);
597 	tcmu_cmd->dbi_cnt = tcmu_cmd_get_block_cnt(tcmu_cmd);
598 	tcmu_cmd->dbi = kcalloc(tcmu_cmd->dbi_cnt, sizeof(uint32_t),
599 				GFP_KERNEL);
600 	if (!tcmu_cmd->dbi) {
601 		kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
602 		return NULL;
603 	}
604 
605 	return tcmu_cmd;
606 }
607 
608 static inline void tcmu_flush_dcache_range(void *vaddr, size_t size)
609 {
610 	unsigned long offset = offset_in_page(vaddr);
611 	void *start = vaddr - offset;
612 
613 	size = round_up(size+offset, PAGE_SIZE);
614 
615 	while (size) {
616 		flush_dcache_page(virt_to_page(start));
617 		start += PAGE_SIZE;
618 		size -= PAGE_SIZE;
619 	}
620 }
621 
622 /*
623  * Some ring helper functions. We don't assume size is a power of 2 so
624  * we can't use circ_buf.h.
625  */
626 static inline size_t spc_used(size_t head, size_t tail, size_t size)
627 {
628 	int diff = head - tail;
629 
630 	if (diff >= 0)
631 		return diff;
632 	else
633 		return size + diff;
634 }
635 
636 static inline size_t spc_free(size_t head, size_t tail, size_t size)
637 {
638 	/* Keep 1 byte unused or we can't tell full from empty */
639 	return (size - spc_used(head, tail, size) - 1);
640 }
641 
642 static inline size_t head_to_end(size_t head, size_t size)
643 {
644 	return size - head;
645 }
646 
647 static inline void new_iov(struct iovec **iov, int *iov_cnt)
648 {
649 	struct iovec *iovec;
650 
651 	if (*iov_cnt != 0)
652 		(*iov)++;
653 	(*iov_cnt)++;
654 
655 	iovec = *iov;
656 	memset(iovec, 0, sizeof(struct iovec));
657 }
658 
659 #define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size)
660 
661 /* offset is relative to mb_addr */
662 static inline size_t get_block_offset_user(struct tcmu_dev *dev,
663 		int dbi, int remaining)
664 {
665 	return dev->data_off + dbi * DATA_BLOCK_SIZE +
666 		DATA_BLOCK_SIZE - remaining;
667 }
668 
669 static inline size_t iov_tail(struct iovec *iov)
670 {
671 	return (size_t)iov->iov_base + iov->iov_len;
672 }
673 
674 static void scatter_data_area(struct tcmu_dev *udev,
675 	struct tcmu_cmd *tcmu_cmd, struct scatterlist *data_sg,
676 	unsigned int data_nents, struct iovec **iov,
677 	int *iov_cnt, bool copy_data)
678 {
679 	int i, dbi;
680 	int block_remaining = 0;
681 	void *from, *to = NULL;
682 	size_t copy_bytes, to_offset, offset;
683 	struct scatterlist *sg;
684 	struct page *page;
685 
686 	for_each_sg(data_sg, sg, data_nents, i) {
687 		int sg_remaining = sg->length;
688 		from = kmap_atomic(sg_page(sg)) + sg->offset;
689 		while (sg_remaining > 0) {
690 			if (block_remaining == 0) {
691 				if (to)
692 					kunmap_atomic(to);
693 
694 				block_remaining = DATA_BLOCK_SIZE;
695 				dbi = tcmu_cmd_get_dbi(tcmu_cmd);
696 				page = tcmu_get_block_page(udev, dbi);
697 				to = kmap_atomic(page);
698 			}
699 
700 			/*
701 			 * Covert to virtual offset of the ring data area.
702 			 */
703 			to_offset = get_block_offset_user(udev, dbi,
704 					block_remaining);
705 
706 			/*
707 			 * The following code will gather and map the blocks
708 			 * to the same iovec when the blocks are all next to
709 			 * each other.
710 			 */
711 			copy_bytes = min_t(size_t, sg_remaining,
712 					block_remaining);
713 			if (*iov_cnt != 0 &&
714 			    to_offset == iov_tail(*iov)) {
715 				/*
716 				 * Will append to the current iovec, because
717 				 * the current block page is next to the
718 				 * previous one.
719 				 */
720 				(*iov)->iov_len += copy_bytes;
721 			} else {
722 				/*
723 				 * Will allocate a new iovec because we are
724 				 * first time here or the current block page
725 				 * is not next to the previous one.
726 				 */
727 				new_iov(iov, iov_cnt);
728 				(*iov)->iov_base = (void __user *)to_offset;
729 				(*iov)->iov_len = copy_bytes;
730 			}
731 
732 			if (copy_data) {
733 				offset = DATA_BLOCK_SIZE - block_remaining;
734 				memcpy(to + offset,
735 				       from + sg->length - sg_remaining,
736 				       copy_bytes);
737 				tcmu_flush_dcache_range(to, copy_bytes);
738 			}
739 
740 			sg_remaining -= copy_bytes;
741 			block_remaining -= copy_bytes;
742 		}
743 		kunmap_atomic(from - sg->offset);
744 	}
745 
746 	if (to)
747 		kunmap_atomic(to);
748 }
749 
750 static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
751 			     bool bidi, uint32_t read_len)
752 {
753 	struct se_cmd *se_cmd = cmd->se_cmd;
754 	int i, dbi;
755 	int block_remaining = 0;
756 	void *from = NULL, *to;
757 	size_t copy_bytes, offset;
758 	struct scatterlist *sg, *data_sg;
759 	struct page *page;
760 	unsigned int data_nents;
761 	uint32_t count = 0;
762 
763 	if (!bidi) {
764 		data_sg = se_cmd->t_data_sg;
765 		data_nents = se_cmd->t_data_nents;
766 	} else {
767 
768 		/*
769 		 * For bidi case, the first count blocks are for Data-Out
770 		 * buffer blocks, and before gathering the Data-In buffer
771 		 * the Data-Out buffer blocks should be discarded.
772 		 */
773 		count = DIV_ROUND_UP(se_cmd->data_length, DATA_BLOCK_SIZE);
774 
775 		data_sg = se_cmd->t_bidi_data_sg;
776 		data_nents = se_cmd->t_bidi_data_nents;
777 	}
778 
779 	tcmu_cmd_set_dbi_cur(cmd, count);
780 
781 	for_each_sg(data_sg, sg, data_nents, i) {
782 		int sg_remaining = sg->length;
783 		to = kmap_atomic(sg_page(sg)) + sg->offset;
784 		while (sg_remaining > 0 && read_len > 0) {
785 			if (block_remaining == 0) {
786 				if (from)
787 					kunmap_atomic(from);
788 
789 				block_remaining = DATA_BLOCK_SIZE;
790 				dbi = tcmu_cmd_get_dbi(cmd);
791 				page = tcmu_get_block_page(udev, dbi);
792 				from = kmap_atomic(page);
793 			}
794 			copy_bytes = min_t(size_t, sg_remaining,
795 					block_remaining);
796 			if (read_len < copy_bytes)
797 				copy_bytes = read_len;
798 			offset = DATA_BLOCK_SIZE - block_remaining;
799 			tcmu_flush_dcache_range(from, copy_bytes);
800 			memcpy(to + sg->length - sg_remaining, from + offset,
801 					copy_bytes);
802 
803 			sg_remaining -= copy_bytes;
804 			block_remaining -= copy_bytes;
805 			read_len -= copy_bytes;
806 		}
807 		kunmap_atomic(to - sg->offset);
808 		if (read_len == 0)
809 			break;
810 	}
811 	if (from)
812 		kunmap_atomic(from);
813 }
814 
815 static inline size_t spc_bitmap_free(unsigned long *bitmap, uint32_t thresh)
816 {
817 	return thresh - bitmap_weight(bitmap, thresh);
818 }
819 
820 /*
821  * We can't queue a command until we have space available on the cmd ring *and*
822  * space available on the data area.
823  *
824  * Called with ring lock held.
825  */
826 static bool is_ring_space_avail(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
827 		size_t cmd_size, size_t data_needed)
828 {
829 	struct tcmu_mailbox *mb = udev->mb_addr;
830 	uint32_t blocks_needed = (data_needed + DATA_BLOCK_SIZE - 1)
831 				/ DATA_BLOCK_SIZE;
832 	size_t space, cmd_needed;
833 	u32 cmd_head;
834 
835 	tcmu_flush_dcache_range(mb, sizeof(*mb));
836 
837 	cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
838 
839 	/*
840 	 * If cmd end-of-ring space is too small then we need space for a NOP plus
841 	 * original cmd - cmds are internally contiguous.
842 	 */
843 	if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size)
844 		cmd_needed = cmd_size;
845 	else
846 		cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size);
847 
848 	space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size);
849 	if (space < cmd_needed) {
850 		pr_debug("no cmd space: %u %u %u\n", cmd_head,
851 		       udev->cmdr_last_cleaned, udev->cmdr_size);
852 		return false;
853 	}
854 
855 	/* try to check and get the data blocks as needed */
856 	space = spc_bitmap_free(udev->data_bitmap, udev->dbi_thresh);
857 	if ((space * DATA_BLOCK_SIZE) < data_needed) {
858 		unsigned long blocks_left =
859 				(udev->max_blocks - udev->dbi_thresh) + space;
860 
861 		if (blocks_left < blocks_needed) {
862 			pr_debug("no data space: only %lu available, but ask for %zu\n",
863 					blocks_left * DATA_BLOCK_SIZE,
864 					data_needed);
865 			return false;
866 		}
867 
868 		udev->dbi_thresh += blocks_needed;
869 		if (udev->dbi_thresh > udev->max_blocks)
870 			udev->dbi_thresh = udev->max_blocks;
871 	}
872 
873 	return tcmu_get_empty_blocks(udev, cmd);
874 }
875 
876 static inline size_t tcmu_cmd_get_base_cmd_size(size_t iov_cnt)
877 {
878 	return max(offsetof(struct tcmu_cmd_entry, req.iov[iov_cnt]),
879 			sizeof(struct tcmu_cmd_entry));
880 }
881 
882 static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd *tcmu_cmd,
883 					   size_t base_command_size)
884 {
885 	struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
886 	size_t command_size;
887 
888 	command_size = base_command_size +
889 		round_up(scsi_command_size(se_cmd->t_task_cdb),
890 				TCMU_OP_ALIGN_SIZE);
891 
892 	WARN_ON(command_size & (TCMU_OP_ALIGN_SIZE-1));
893 
894 	return command_size;
895 }
896 
897 static int tcmu_setup_cmd_timer(struct tcmu_cmd *tcmu_cmd, unsigned int tmo,
898 				struct timer_list *timer)
899 {
900 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
901 	int cmd_id;
902 
903 	if (tcmu_cmd->cmd_id)
904 		goto setup_timer;
905 
906 	cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 1, USHRT_MAX, GFP_NOWAIT);
907 	if (cmd_id < 0) {
908 		pr_err("tcmu: Could not allocate cmd id.\n");
909 		return cmd_id;
910 	}
911 	tcmu_cmd->cmd_id = cmd_id;
912 
913 	pr_debug("allocated cmd %u for dev %s tmo %lu\n", tcmu_cmd->cmd_id,
914 		 udev->name, tmo / MSEC_PER_SEC);
915 
916 setup_timer:
917 	if (!tmo)
918 		return 0;
919 
920 	tcmu_cmd->deadline = round_jiffies_up(jiffies + msecs_to_jiffies(tmo));
921 	if (!timer_pending(timer))
922 		mod_timer(timer, tcmu_cmd->deadline);
923 
924 	return 0;
925 }
926 
927 static int add_to_qfull_queue(struct tcmu_cmd *tcmu_cmd)
928 {
929 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
930 	unsigned int tmo;
931 	int ret;
932 
933 	/*
934 	 * For backwards compat if qfull_time_out is not set use
935 	 * cmd_time_out and if that's not set use the default time out.
936 	 */
937 	if (!udev->qfull_time_out)
938 		return -ETIMEDOUT;
939 	else if (udev->qfull_time_out > 0)
940 		tmo = udev->qfull_time_out;
941 	else if (udev->cmd_time_out)
942 		tmo = udev->cmd_time_out;
943 	else
944 		tmo = TCMU_TIME_OUT;
945 
946 	ret = tcmu_setup_cmd_timer(tcmu_cmd, tmo, &udev->qfull_timer);
947 	if (ret)
948 		return ret;
949 
950 	list_add_tail(&tcmu_cmd->queue_entry, &udev->qfull_queue);
951 	pr_debug("adding cmd %u on dev %s to ring space wait queue\n",
952 		 tcmu_cmd->cmd_id, udev->name);
953 	return 0;
954 }
955 
956 /**
957  * queue_cmd_ring - queue cmd to ring or internally
958  * @tcmu_cmd: cmd to queue
959  * @scsi_err: TCM error code if failure (-1) returned.
960  *
961  * Returns:
962  * -1 we cannot queue internally or to the ring.
963  *  0 success
964  *  1 internally queued to wait for ring memory to free.
965  */
966 static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)
967 {
968 	struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
969 	struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
970 	size_t base_command_size, command_size;
971 	struct tcmu_mailbox *mb;
972 	struct tcmu_cmd_entry *entry;
973 	struct iovec *iov;
974 	int iov_cnt, ret;
975 	uint32_t cmd_head;
976 	uint64_t cdb_off;
977 	bool copy_to_data_area;
978 	size_t data_length = tcmu_cmd_get_data_length(tcmu_cmd);
979 
980 	*scsi_err = TCM_NO_SENSE;
981 
982 	if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags)) {
983 		*scsi_err = TCM_LUN_BUSY;
984 		return -1;
985 	}
986 
987 	if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
988 		*scsi_err = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
989 		return -1;
990 	}
991 
992 	/*
993 	 * Must be a certain minimum size for response sense info, but
994 	 * also may be larger if the iov array is large.
995 	 *
996 	 * We prepare as many iovs as possbile for potential uses here,
997 	 * because it's expensive to tell how many regions are freed in
998 	 * the bitmap & global data pool, as the size calculated here
999 	 * will only be used to do the checks.
1000 	 *
1001 	 * The size will be recalculated later as actually needed to save
1002 	 * cmd area memories.
1003 	 */
1004 	base_command_size = tcmu_cmd_get_base_cmd_size(tcmu_cmd->dbi_cnt);
1005 	command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size);
1006 
1007 	if (!list_empty(&udev->qfull_queue))
1008 		goto queue;
1009 
1010 	mb = udev->mb_addr;
1011 	cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
1012 	if ((command_size > (udev->cmdr_size / 2)) ||
1013 	    data_length > udev->data_size) {
1014 		pr_warn("TCMU: Request of size %zu/%zu is too big for %u/%zu "
1015 			"cmd ring/data area\n", command_size, data_length,
1016 			udev->cmdr_size, udev->data_size);
1017 		*scsi_err = TCM_INVALID_CDB_FIELD;
1018 		return -1;
1019 	}
1020 
1021 	if (!is_ring_space_avail(udev, tcmu_cmd, command_size, data_length)) {
1022 		/*
1023 		 * Don't leave commands partially setup because the unmap
1024 		 * thread might need the blocks to make forward progress.
1025 		 */
1026 		tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);
1027 		tcmu_cmd_reset_dbi_cur(tcmu_cmd);
1028 		goto queue;
1029 	}
1030 
1031 	/* Insert a PAD if end-of-ring space is too small */
1032 	if (head_to_end(cmd_head, udev->cmdr_size) < command_size) {
1033 		size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);
1034 
1035 		entry = (void *) mb + CMDR_OFF + cmd_head;
1036 		tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_PAD);
1037 		tcmu_hdr_set_len(&entry->hdr.len_op, pad_size);
1038 		entry->hdr.cmd_id = 0; /* not used for PAD */
1039 		entry->hdr.kflags = 0;
1040 		entry->hdr.uflags = 0;
1041 		tcmu_flush_dcache_range(entry, sizeof(*entry));
1042 
1043 		UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);
1044 		tcmu_flush_dcache_range(mb, sizeof(*mb));
1045 
1046 		cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
1047 		WARN_ON(cmd_head != 0);
1048 	}
1049 
1050 	entry = (void *) mb + CMDR_OFF + cmd_head;
1051 	memset(entry, 0, command_size);
1052 	tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_CMD);
1053 
1054 	/* Handle allocating space from the data area */
1055 	tcmu_cmd_reset_dbi_cur(tcmu_cmd);
1056 	iov = &entry->req.iov[0];
1057 	iov_cnt = 0;
1058 	copy_to_data_area = (se_cmd->data_direction == DMA_TO_DEVICE
1059 		|| se_cmd->se_cmd_flags & SCF_BIDI);
1060 	scatter_data_area(udev, tcmu_cmd, se_cmd->t_data_sg,
1061 			  se_cmd->t_data_nents, &iov, &iov_cnt,
1062 			  copy_to_data_area);
1063 	entry->req.iov_cnt = iov_cnt;
1064 
1065 	/* Handle BIDI commands */
1066 	iov_cnt = 0;
1067 	if (se_cmd->se_cmd_flags & SCF_BIDI) {
1068 		iov++;
1069 		scatter_data_area(udev, tcmu_cmd, se_cmd->t_bidi_data_sg,
1070 				  se_cmd->t_bidi_data_nents, &iov, &iov_cnt,
1071 				  false);
1072 	}
1073 	entry->req.iov_bidi_cnt = iov_cnt;
1074 
1075 	ret = tcmu_setup_cmd_timer(tcmu_cmd, udev->cmd_time_out,
1076 				   &udev->cmd_timer);
1077 	if (ret) {
1078 		tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);
1079 
1080 		*scsi_err = TCM_OUT_OF_RESOURCES;
1081 		return -1;
1082 	}
1083 	entry->hdr.cmd_id = tcmu_cmd->cmd_id;
1084 
1085 	/*
1086 	 * Recalaulate the command's base size and size according
1087 	 * to the actual needs
1088 	 */
1089 	base_command_size = tcmu_cmd_get_base_cmd_size(entry->req.iov_cnt +
1090 						       entry->req.iov_bidi_cnt);
1091 	command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size);
1092 
1093 	tcmu_hdr_set_len(&entry->hdr.len_op, command_size);
1094 
1095 	/* All offsets relative to mb_addr, not start of entry! */
1096 	cdb_off = CMDR_OFF + cmd_head + base_command_size;
1097 	memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb));
1098 	entry->req.cdb_off = cdb_off;
1099 	tcmu_flush_dcache_range(entry, sizeof(*entry));
1100 
1101 	UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);
1102 	tcmu_flush_dcache_range(mb, sizeof(*mb));
1103 
1104 	list_add_tail(&tcmu_cmd->queue_entry, &udev->inflight_queue);
1105 	set_bit(TCMU_CMD_BIT_INFLIGHT, &tcmu_cmd->flags);
1106 
1107 	/* TODO: only if FLUSH and FUA? */
1108 	uio_event_notify(&udev->uio_info);
1109 
1110 	return 0;
1111 
1112 queue:
1113 	if (add_to_qfull_queue(tcmu_cmd)) {
1114 		*scsi_err = TCM_OUT_OF_RESOURCES;
1115 		return -1;
1116 	}
1117 
1118 	return 1;
1119 }
1120 
1121 static sense_reason_t
1122 tcmu_queue_cmd(struct se_cmd *se_cmd)
1123 {
1124 	struct se_device *se_dev = se_cmd->se_dev;
1125 	struct tcmu_dev *udev = TCMU_DEV(se_dev);
1126 	struct tcmu_cmd *tcmu_cmd;
1127 	sense_reason_t scsi_ret;
1128 	int ret;
1129 
1130 	tcmu_cmd = tcmu_alloc_cmd(se_cmd);
1131 	if (!tcmu_cmd)
1132 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
1133 
1134 	mutex_lock(&udev->cmdr_lock);
1135 	ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
1136 	mutex_unlock(&udev->cmdr_lock);
1137 	if (ret < 0)
1138 		tcmu_free_cmd(tcmu_cmd);
1139 	return scsi_ret;
1140 }
1141 
1142 static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry *entry)
1143 {
1144 	struct se_cmd *se_cmd = cmd->se_cmd;
1145 	struct tcmu_dev *udev = cmd->tcmu_dev;
1146 	bool read_len_valid = false;
1147 	uint32_t read_len = se_cmd->data_length;
1148 
1149 	/*
1150 	 * cmd has been completed already from timeout, just reclaim
1151 	 * data area space and free cmd
1152 	 */
1153 	if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags))
1154 		goto out;
1155 
1156 	list_del_init(&cmd->queue_entry);
1157 
1158 	tcmu_cmd_reset_dbi_cur(cmd);
1159 
1160 	if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) {
1161 		pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n",
1162 			cmd->se_cmd);
1163 		entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION;
1164 		goto done;
1165 	}
1166 
1167 	if (se_cmd->data_direction == DMA_FROM_DEVICE &&
1168 	    (entry->hdr.uflags & TCMU_UFLAG_READ_LEN) && entry->rsp.read_len) {
1169 		read_len_valid = true;
1170 		if (entry->rsp.read_len < read_len)
1171 			read_len = entry->rsp.read_len;
1172 	}
1173 
1174 	if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) {
1175 		transport_copy_sense_to_cmd(se_cmd, entry->rsp.sense_buffer);
1176 		if (!read_len_valid )
1177 			goto done;
1178 		else
1179 			se_cmd->se_cmd_flags |= SCF_TREAT_READ_AS_NORMAL;
1180 	}
1181 	if (se_cmd->se_cmd_flags & SCF_BIDI) {
1182 		/* Get Data-In buffer before clean up */
1183 		gather_data_area(udev, cmd, true, read_len);
1184 	} else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
1185 		gather_data_area(udev, cmd, false, read_len);
1186 	} else if (se_cmd->data_direction == DMA_TO_DEVICE) {
1187 		/* TODO: */
1188 	} else if (se_cmd->data_direction != DMA_NONE) {
1189 		pr_warn("TCMU: data direction was %d!\n",
1190 			se_cmd->data_direction);
1191 	}
1192 
1193 done:
1194 	if (read_len_valid) {
1195 		pr_debug("read_len = %d\n", read_len);
1196 		target_complete_cmd_with_length(cmd->se_cmd,
1197 					entry->rsp.scsi_status, read_len);
1198 	} else
1199 		target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status);
1200 
1201 out:
1202 	cmd->se_cmd = NULL;
1203 	tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
1204 	tcmu_free_cmd(cmd);
1205 }
1206 
1207 static void tcmu_set_next_deadline(struct list_head *queue,
1208 				   struct timer_list *timer)
1209 {
1210 	struct tcmu_cmd *tcmu_cmd, *tmp_cmd;
1211 	unsigned long deadline = 0;
1212 
1213 	list_for_each_entry_safe(tcmu_cmd, tmp_cmd, queue, queue_entry) {
1214 		if (!time_after(jiffies, tcmu_cmd->deadline)) {
1215 			deadline = tcmu_cmd->deadline;
1216 			break;
1217 		}
1218 	}
1219 
1220 	if (deadline)
1221 		mod_timer(timer, deadline);
1222 	else
1223 		del_timer(timer);
1224 }
1225 
1226 static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
1227 {
1228 	struct tcmu_mailbox *mb;
1229 	struct tcmu_cmd *cmd;
1230 	int handled = 0;
1231 
1232 	if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
1233 		pr_err("ring broken, not handling completions\n");
1234 		return 0;
1235 	}
1236 
1237 	mb = udev->mb_addr;
1238 	tcmu_flush_dcache_range(mb, sizeof(*mb));
1239 
1240 	while (udev->cmdr_last_cleaned != READ_ONCE(mb->cmd_tail)) {
1241 
1242 		struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned;
1243 
1244 		tcmu_flush_dcache_range(entry, sizeof(*entry));
1245 
1246 		if (tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_PAD) {
1247 			UPDATE_HEAD(udev->cmdr_last_cleaned,
1248 				    tcmu_hdr_get_len(entry->hdr.len_op),
1249 				    udev->cmdr_size);
1250 			continue;
1251 		}
1252 		WARN_ON(tcmu_hdr_get_op(entry->hdr.len_op) != TCMU_OP_CMD);
1253 
1254 		cmd = idr_remove(&udev->commands, entry->hdr.cmd_id);
1255 		if (!cmd) {
1256 			pr_err("cmd_id %u not found, ring is broken\n",
1257 			       entry->hdr.cmd_id);
1258 			set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);
1259 			break;
1260 		}
1261 
1262 		tcmu_handle_completion(cmd, entry);
1263 
1264 		UPDATE_HEAD(udev->cmdr_last_cleaned,
1265 			    tcmu_hdr_get_len(entry->hdr.len_op),
1266 			    udev->cmdr_size);
1267 
1268 		handled++;
1269 	}
1270 
1271 	if (mb->cmd_tail == mb->cmd_head) {
1272 		/* no more pending commands */
1273 		del_timer(&udev->cmd_timer);
1274 
1275 		if (list_empty(&udev->qfull_queue)) {
1276 			/*
1277 			 * no more pending or waiting commands so try to
1278 			 * reclaim blocks if needed.
1279 			 */
1280 			if (atomic_read(&global_db_count) >
1281 			    tcmu_global_max_blocks)
1282 				schedule_delayed_work(&tcmu_unmap_work, 0);
1283 		}
1284 	} else if (udev->cmd_time_out) {
1285 		tcmu_set_next_deadline(&udev->inflight_queue, &udev->cmd_timer);
1286 	}
1287 
1288 	return handled;
1289 }
1290 
1291 static int tcmu_check_expired_cmd(int id, void *p, void *data)
1292 {
1293 	struct tcmu_cmd *cmd = p;
1294 	struct tcmu_dev *udev = cmd->tcmu_dev;
1295 	u8 scsi_status;
1296 	struct se_cmd *se_cmd;
1297 	bool is_running;
1298 
1299 	if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags))
1300 		return 0;
1301 
1302 	if (!time_after(jiffies, cmd->deadline))
1303 		return 0;
1304 
1305 	is_running = test_bit(TCMU_CMD_BIT_INFLIGHT, &cmd->flags);
1306 	se_cmd = cmd->se_cmd;
1307 
1308 	if (is_running) {
1309 		/*
1310 		 * If cmd_time_out is disabled but qfull is set deadline
1311 		 * will only reflect the qfull timeout. Ignore it.
1312 		 */
1313 		if (!udev->cmd_time_out)
1314 			return 0;
1315 
1316 		set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags);
1317 		/*
1318 		 * target_complete_cmd will translate this to LUN COMM FAILURE
1319 		 */
1320 		scsi_status = SAM_STAT_CHECK_CONDITION;
1321 		list_del_init(&cmd->queue_entry);
1322 	} else {
1323 		list_del_init(&cmd->queue_entry);
1324 		idr_remove(&udev->commands, id);
1325 		tcmu_free_cmd(cmd);
1326 		scsi_status = SAM_STAT_TASK_SET_FULL;
1327 	}
1328 
1329 	pr_debug("Timing out cmd %u on dev %s that is %s.\n",
1330 		 id, udev->name, is_running ? "inflight" : "queued");
1331 
1332 	target_complete_cmd(se_cmd, scsi_status);
1333 	return 0;
1334 }
1335 
1336 static void tcmu_device_timedout(struct tcmu_dev *udev)
1337 {
1338 	spin_lock(&timed_out_udevs_lock);
1339 	if (list_empty(&udev->timedout_entry))
1340 		list_add_tail(&udev->timedout_entry, &timed_out_udevs);
1341 	spin_unlock(&timed_out_udevs_lock);
1342 
1343 	schedule_delayed_work(&tcmu_unmap_work, 0);
1344 }
1345 
1346 static void tcmu_cmd_timedout(struct timer_list *t)
1347 {
1348 	struct tcmu_dev *udev = from_timer(udev, t, cmd_timer);
1349 
1350 	pr_debug("%s cmd timeout has expired\n", udev->name);
1351 	tcmu_device_timedout(udev);
1352 }
1353 
1354 static void tcmu_qfull_timedout(struct timer_list *t)
1355 {
1356 	struct tcmu_dev *udev = from_timer(udev, t, qfull_timer);
1357 
1358 	pr_debug("%s qfull timeout has expired\n", udev->name);
1359 	tcmu_device_timedout(udev);
1360 }
1361 
1362 static int tcmu_attach_hba(struct se_hba *hba, u32 host_id)
1363 {
1364 	struct tcmu_hba *tcmu_hba;
1365 
1366 	tcmu_hba = kzalloc(sizeof(struct tcmu_hba), GFP_KERNEL);
1367 	if (!tcmu_hba)
1368 		return -ENOMEM;
1369 
1370 	tcmu_hba->host_id = host_id;
1371 	hba->hba_ptr = tcmu_hba;
1372 
1373 	return 0;
1374 }
1375 
1376 static void tcmu_detach_hba(struct se_hba *hba)
1377 {
1378 	kfree(hba->hba_ptr);
1379 	hba->hba_ptr = NULL;
1380 }
1381 
1382 static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name)
1383 {
1384 	struct tcmu_dev *udev;
1385 
1386 	udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL);
1387 	if (!udev)
1388 		return NULL;
1389 	kref_init(&udev->kref);
1390 
1391 	udev->name = kstrdup(name, GFP_KERNEL);
1392 	if (!udev->name) {
1393 		kfree(udev);
1394 		return NULL;
1395 	}
1396 
1397 	udev->hba = hba;
1398 	udev->cmd_time_out = TCMU_TIME_OUT;
1399 	udev->qfull_time_out = -1;
1400 
1401 	udev->max_blocks = DATA_BLOCK_BITS_DEF;
1402 	mutex_init(&udev->cmdr_lock);
1403 
1404 	INIT_LIST_HEAD(&udev->node);
1405 	INIT_LIST_HEAD(&udev->timedout_entry);
1406 	INIT_LIST_HEAD(&udev->qfull_queue);
1407 	INIT_LIST_HEAD(&udev->inflight_queue);
1408 	idr_init(&udev->commands);
1409 
1410 	timer_setup(&udev->qfull_timer, tcmu_qfull_timedout, 0);
1411 	timer_setup(&udev->cmd_timer, tcmu_cmd_timedout, 0);
1412 
1413 	INIT_RADIX_TREE(&udev->data_blocks, GFP_KERNEL);
1414 
1415 	return &udev->se_dev;
1416 }
1417 
1418 static bool run_qfull_queue(struct tcmu_dev *udev, bool fail)
1419 {
1420 	struct tcmu_cmd *tcmu_cmd, *tmp_cmd;
1421 	LIST_HEAD(cmds);
1422 	bool drained = true;
1423 	sense_reason_t scsi_ret;
1424 	int ret;
1425 
1426 	if (list_empty(&udev->qfull_queue))
1427 		return true;
1428 
1429 	pr_debug("running %s's cmdr queue forcefail %d\n", udev->name, fail);
1430 
1431 	list_splice_init(&udev->qfull_queue, &cmds);
1432 
1433 	list_for_each_entry_safe(tcmu_cmd, tmp_cmd, &cmds, queue_entry) {
1434 		list_del_init(&tcmu_cmd->queue_entry);
1435 
1436 	        pr_debug("removing cmd %u on dev %s from queue\n",
1437 		         tcmu_cmd->cmd_id, udev->name);
1438 
1439 		if (fail) {
1440 			idr_remove(&udev->commands, tcmu_cmd->cmd_id);
1441 			/*
1442 			 * We were not able to even start the command, so
1443 			 * fail with busy to allow a retry in case runner
1444 			 * was only temporarily down. If the device is being
1445 			 * removed then LIO core will do the right thing and
1446 			 * fail the retry.
1447 			 */
1448 			target_complete_cmd(tcmu_cmd->se_cmd, SAM_STAT_BUSY);
1449 			tcmu_free_cmd(tcmu_cmd);
1450 			continue;
1451 		}
1452 
1453 		ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
1454 		if (ret < 0) {
1455 		        pr_debug("cmd %u on dev %s failed with %u\n",
1456 			         tcmu_cmd->cmd_id, udev->name, scsi_ret);
1457 
1458 			idr_remove(&udev->commands, tcmu_cmd->cmd_id);
1459 			/*
1460 			 * Ignore scsi_ret for now. target_complete_cmd
1461 			 * drops it.
1462 			 */
1463 			target_complete_cmd(tcmu_cmd->se_cmd,
1464 					    SAM_STAT_CHECK_CONDITION);
1465 			tcmu_free_cmd(tcmu_cmd);
1466 		} else if (ret > 0) {
1467 			pr_debug("ran out of space during cmdr queue run\n");
1468 			/*
1469 			 * cmd was requeued, so just put all cmds back in
1470 			 * the queue
1471 			 */
1472 			list_splice_tail(&cmds, &udev->qfull_queue);
1473 			drained = false;
1474 			break;
1475 		}
1476 	}
1477 
1478 	tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
1479 	return drained;
1480 }
1481 
1482 static int tcmu_irqcontrol(struct uio_info *info, s32 irq_on)
1483 {
1484 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
1485 
1486 	mutex_lock(&udev->cmdr_lock);
1487 	tcmu_handle_completions(udev);
1488 	run_qfull_queue(udev, false);
1489 	mutex_unlock(&udev->cmdr_lock);
1490 
1491 	return 0;
1492 }
1493 
1494 /*
1495  * mmap code from uio.c. Copied here because we want to hook mmap()
1496  * and this stuff must come along.
1497  */
1498 static int tcmu_find_mem_index(struct vm_area_struct *vma)
1499 {
1500 	struct tcmu_dev *udev = vma->vm_private_data;
1501 	struct uio_info *info = &udev->uio_info;
1502 
1503 	if (vma->vm_pgoff < MAX_UIO_MAPS) {
1504 		if (info->mem[vma->vm_pgoff].size == 0)
1505 			return -1;
1506 		return (int)vma->vm_pgoff;
1507 	}
1508 	return -1;
1509 }
1510 
1511 static struct page *tcmu_try_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
1512 {
1513 	struct page *page;
1514 
1515 	mutex_lock(&udev->cmdr_lock);
1516 	page = tcmu_get_block_page(udev, dbi);
1517 	if (likely(page)) {
1518 		mutex_unlock(&udev->cmdr_lock);
1519 		return page;
1520 	}
1521 
1522 	/*
1523 	 * Userspace messed up and passed in a address not in the
1524 	 * data iov passed to it.
1525 	 */
1526 	pr_err("Invalid addr to data block mapping  (dbi %u) on device %s\n",
1527 	       dbi, udev->name);
1528 	page = NULL;
1529 	mutex_unlock(&udev->cmdr_lock);
1530 
1531 	return page;
1532 }
1533 
1534 static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf)
1535 {
1536 	struct tcmu_dev *udev = vmf->vma->vm_private_data;
1537 	struct uio_info *info = &udev->uio_info;
1538 	struct page *page;
1539 	unsigned long offset;
1540 	void *addr;
1541 
1542 	int mi = tcmu_find_mem_index(vmf->vma);
1543 	if (mi < 0)
1544 		return VM_FAULT_SIGBUS;
1545 
1546 	/*
1547 	 * We need to subtract mi because userspace uses offset = N*PAGE_SIZE
1548 	 * to use mem[N].
1549 	 */
1550 	offset = (vmf->pgoff - mi) << PAGE_SHIFT;
1551 
1552 	if (offset < udev->data_off) {
1553 		/* For the vmalloc()ed cmd area pages */
1554 		addr = (void *)(unsigned long)info->mem[mi].addr + offset;
1555 		page = vmalloc_to_page(addr);
1556 	} else {
1557 		uint32_t dbi;
1558 
1559 		/* For the dynamically growing data area pages */
1560 		dbi = (offset - udev->data_off) / DATA_BLOCK_SIZE;
1561 		page = tcmu_try_get_block_page(udev, dbi);
1562 		if (!page)
1563 			return VM_FAULT_SIGBUS;
1564 	}
1565 
1566 	get_page(page);
1567 	vmf->page = page;
1568 	return 0;
1569 }
1570 
1571 static const struct vm_operations_struct tcmu_vm_ops = {
1572 	.fault = tcmu_vma_fault,
1573 };
1574 
1575 static int tcmu_mmap(struct uio_info *info, struct vm_area_struct *vma)
1576 {
1577 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
1578 
1579 	vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
1580 	vma->vm_ops = &tcmu_vm_ops;
1581 
1582 	vma->vm_private_data = udev;
1583 
1584 	/* Ensure the mmap is exactly the right size */
1585 	if (vma_pages(vma) != (udev->ring_size >> PAGE_SHIFT))
1586 		return -EINVAL;
1587 
1588 	return 0;
1589 }
1590 
1591 static int tcmu_open(struct uio_info *info, struct inode *inode)
1592 {
1593 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
1594 
1595 	/* O_EXCL not supported for char devs, so fake it? */
1596 	if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags))
1597 		return -EBUSY;
1598 
1599 	udev->inode = inode;
1600 	kref_get(&udev->kref);
1601 
1602 	pr_debug("open\n");
1603 
1604 	return 0;
1605 }
1606 
1607 static void tcmu_dev_call_rcu(struct rcu_head *p)
1608 {
1609 	struct se_device *dev = container_of(p, struct se_device, rcu_head);
1610 	struct tcmu_dev *udev = TCMU_DEV(dev);
1611 
1612 	kfree(udev->uio_info.name);
1613 	kfree(udev->name);
1614 	kfree(udev);
1615 }
1616 
1617 static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd)
1618 {
1619 	if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
1620 		kmem_cache_free(tcmu_cmd_cache, cmd);
1621 		return 0;
1622 	}
1623 	return -EINVAL;
1624 }
1625 
1626 static void tcmu_blocks_release(struct radix_tree_root *blocks,
1627 				int start, int end)
1628 {
1629 	int i;
1630 	struct page *page;
1631 
1632 	for (i = start; i < end; i++) {
1633 		page = radix_tree_delete(blocks, i);
1634 		if (page) {
1635 			__free_page(page);
1636 			atomic_dec(&global_db_count);
1637 		}
1638 	}
1639 }
1640 
1641 static void tcmu_dev_kref_release(struct kref *kref)
1642 {
1643 	struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref);
1644 	struct se_device *dev = &udev->se_dev;
1645 	struct tcmu_cmd *cmd;
1646 	bool all_expired = true;
1647 	int i;
1648 
1649 	vfree(udev->mb_addr);
1650 	udev->mb_addr = NULL;
1651 
1652 	spin_lock_bh(&timed_out_udevs_lock);
1653 	if (!list_empty(&udev->timedout_entry))
1654 		list_del(&udev->timedout_entry);
1655 	spin_unlock_bh(&timed_out_udevs_lock);
1656 
1657 	/* Upper layer should drain all requests before calling this */
1658 	mutex_lock(&udev->cmdr_lock);
1659 	idr_for_each_entry(&udev->commands, cmd, i) {
1660 		if (tcmu_check_and_free_pending_cmd(cmd) != 0)
1661 			all_expired = false;
1662 	}
1663 	idr_destroy(&udev->commands);
1664 	WARN_ON(!all_expired);
1665 
1666 	tcmu_blocks_release(&udev->data_blocks, 0, udev->dbi_max + 1);
1667 	bitmap_free(udev->data_bitmap);
1668 	mutex_unlock(&udev->cmdr_lock);
1669 
1670 	call_rcu(&dev->rcu_head, tcmu_dev_call_rcu);
1671 }
1672 
1673 static int tcmu_release(struct uio_info *info, struct inode *inode)
1674 {
1675 	struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
1676 
1677 	clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags);
1678 
1679 	pr_debug("close\n");
1680 	/* release ref from open */
1681 	kref_put(&udev->kref, tcmu_dev_kref_release);
1682 	return 0;
1683 }
1684 
1685 static int tcmu_init_genl_cmd_reply(struct tcmu_dev *udev, int cmd)
1686 {
1687 	struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
1688 
1689 	if (!tcmu_kern_cmd_reply_supported)
1690 		return 0;
1691 
1692 	if (udev->nl_reply_supported <= 0)
1693 		return 0;
1694 
1695 	mutex_lock(&tcmu_nl_cmd_mutex);
1696 
1697 	if (tcmu_netlink_blocked) {
1698 		mutex_unlock(&tcmu_nl_cmd_mutex);
1699 		pr_warn("Failing nl cmd %d on %s. Interface is blocked.\n", cmd,
1700 			udev->name);
1701 		return -EAGAIN;
1702 	}
1703 
1704 	if (nl_cmd->cmd != TCMU_CMD_UNSPEC) {
1705 		mutex_unlock(&tcmu_nl_cmd_mutex);
1706 		pr_warn("netlink cmd %d already executing on %s\n",
1707 			 nl_cmd->cmd, udev->name);
1708 		return -EBUSY;
1709 	}
1710 
1711 	memset(nl_cmd, 0, sizeof(*nl_cmd));
1712 	nl_cmd->cmd = cmd;
1713 	nl_cmd->udev = udev;
1714 	init_completion(&nl_cmd->complete);
1715 	INIT_LIST_HEAD(&nl_cmd->nl_list);
1716 
1717 	list_add_tail(&nl_cmd->nl_list, &tcmu_nl_cmd_list);
1718 
1719 	mutex_unlock(&tcmu_nl_cmd_mutex);
1720 	return 0;
1721 }
1722 
1723 static int tcmu_wait_genl_cmd_reply(struct tcmu_dev *udev)
1724 {
1725 	struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
1726 	int ret;
1727 
1728 	if (!tcmu_kern_cmd_reply_supported)
1729 		return 0;
1730 
1731 	if (udev->nl_reply_supported <= 0)
1732 		return 0;
1733 
1734 	pr_debug("sleeping for nl reply\n");
1735 	wait_for_completion(&nl_cmd->complete);
1736 
1737 	mutex_lock(&tcmu_nl_cmd_mutex);
1738 	nl_cmd->cmd = TCMU_CMD_UNSPEC;
1739 	ret = nl_cmd->status;
1740 	mutex_unlock(&tcmu_nl_cmd_mutex);
1741 
1742 	return ret;
1743 }
1744 
1745 static int tcmu_netlink_event_init(struct tcmu_dev *udev,
1746 				   enum tcmu_genl_cmd cmd,
1747 				   struct sk_buff **buf, void **hdr)
1748 {
1749 	struct sk_buff *skb;
1750 	void *msg_header;
1751 	int ret = -ENOMEM;
1752 
1753 	skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1754 	if (!skb)
1755 		return ret;
1756 
1757 	msg_header = genlmsg_put(skb, 0, 0, &tcmu_genl_family, 0, cmd);
1758 	if (!msg_header)
1759 		goto free_skb;
1760 
1761 	ret = nla_put_string(skb, TCMU_ATTR_DEVICE, udev->uio_info.name);
1762 	if (ret < 0)
1763 		goto free_skb;
1764 
1765 	ret = nla_put_u32(skb, TCMU_ATTR_MINOR, udev->uio_info.uio_dev->minor);
1766 	if (ret < 0)
1767 		goto free_skb;
1768 
1769 	ret = nla_put_u32(skb, TCMU_ATTR_DEVICE_ID, udev->se_dev.dev_index);
1770 	if (ret < 0)
1771 		goto free_skb;
1772 
1773 	*buf = skb;
1774 	*hdr = msg_header;
1775 	return ret;
1776 
1777 free_skb:
1778 	nlmsg_free(skb);
1779 	return ret;
1780 }
1781 
1782 static int tcmu_netlink_event_send(struct tcmu_dev *udev,
1783 				   enum tcmu_genl_cmd cmd,
1784 				   struct sk_buff *skb, void *msg_header)
1785 {
1786 	int ret;
1787 
1788 	genlmsg_end(skb, msg_header);
1789 
1790 	ret = tcmu_init_genl_cmd_reply(udev, cmd);
1791 	if (ret) {
1792 		nlmsg_free(skb);
1793 		return ret;
1794 	}
1795 
1796 	ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0,
1797 				      TCMU_MCGRP_CONFIG, GFP_KERNEL);
1798 
1799 	/* Wait during an add as the listener may not be up yet */
1800 	if (ret == 0 ||
1801 	   (ret == -ESRCH && cmd == TCMU_CMD_ADDED_DEVICE))
1802 		return tcmu_wait_genl_cmd_reply(udev);
1803 
1804 	return ret;
1805 }
1806 
1807 static int tcmu_send_dev_add_event(struct tcmu_dev *udev)
1808 {
1809 	struct sk_buff *skb = NULL;
1810 	void *msg_header = NULL;
1811 	int ret = 0;
1812 
1813 	ret = tcmu_netlink_event_init(udev, TCMU_CMD_ADDED_DEVICE, &skb,
1814 				      &msg_header);
1815 	if (ret < 0)
1816 		return ret;
1817 	return tcmu_netlink_event_send(udev, TCMU_CMD_ADDED_DEVICE, skb,
1818 				       msg_header);
1819 }
1820 
1821 static int tcmu_send_dev_remove_event(struct tcmu_dev *udev)
1822 {
1823 	struct sk_buff *skb = NULL;
1824 	void *msg_header = NULL;
1825 	int ret = 0;
1826 
1827 	ret = tcmu_netlink_event_init(udev, TCMU_CMD_REMOVED_DEVICE,
1828 				      &skb, &msg_header);
1829 	if (ret < 0)
1830 		return ret;
1831 	return tcmu_netlink_event_send(udev, TCMU_CMD_REMOVED_DEVICE,
1832 				       skb, msg_header);
1833 }
1834 
1835 static int tcmu_update_uio_info(struct tcmu_dev *udev)
1836 {
1837 	struct tcmu_hba *hba = udev->hba->hba_ptr;
1838 	struct uio_info *info;
1839 	size_t size, used;
1840 	char *str;
1841 
1842 	info = &udev->uio_info;
1843 	size = snprintf(NULL, 0, "tcm-user/%u/%s/%s", hba->host_id, udev->name,
1844 			udev->dev_config);
1845 	size += 1; /* for \0 */
1846 	str = kmalloc(size, GFP_KERNEL);
1847 	if (!str)
1848 		return -ENOMEM;
1849 
1850 	used = snprintf(str, size, "tcm-user/%u/%s", hba->host_id, udev->name);
1851 	if (udev->dev_config[0])
1852 		snprintf(str + used, size - used, "/%s", udev->dev_config);
1853 
1854 	/* If the old string exists, free it */
1855 	kfree(info->name);
1856 	info->name = str;
1857 
1858 	return 0;
1859 }
1860 
1861 static int tcmu_configure_device(struct se_device *dev)
1862 {
1863 	struct tcmu_dev *udev = TCMU_DEV(dev);
1864 	struct uio_info *info;
1865 	struct tcmu_mailbox *mb;
1866 	int ret = 0;
1867 
1868 	ret = tcmu_update_uio_info(udev);
1869 	if (ret)
1870 		return ret;
1871 
1872 	info = &udev->uio_info;
1873 
1874 	mutex_lock(&udev->cmdr_lock);
1875 	udev->data_bitmap = bitmap_zalloc(udev->max_blocks, GFP_KERNEL);
1876 	mutex_unlock(&udev->cmdr_lock);
1877 	if (!udev->data_bitmap) {
1878 		ret = -ENOMEM;
1879 		goto err_bitmap_alloc;
1880 	}
1881 
1882 	udev->mb_addr = vzalloc(CMDR_SIZE);
1883 	if (!udev->mb_addr) {
1884 		ret = -ENOMEM;
1885 		goto err_vzalloc;
1886 	}
1887 
1888 	/* mailbox fits in first part of CMDR space */
1889 	udev->cmdr_size = CMDR_SIZE - CMDR_OFF;
1890 	udev->data_off = CMDR_SIZE;
1891 	udev->data_size = udev->max_blocks * DATA_BLOCK_SIZE;
1892 	udev->dbi_thresh = 0; /* Default in Idle state */
1893 
1894 	/* Initialise the mailbox of the ring buffer */
1895 	mb = udev->mb_addr;
1896 	mb->version = TCMU_MAILBOX_VERSION;
1897 	mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC | TCMU_MAILBOX_FLAG_CAP_READ_LEN;
1898 	mb->cmdr_off = CMDR_OFF;
1899 	mb->cmdr_size = udev->cmdr_size;
1900 
1901 	WARN_ON(!PAGE_ALIGNED(udev->data_off));
1902 	WARN_ON(udev->data_size % PAGE_SIZE);
1903 	WARN_ON(udev->data_size % DATA_BLOCK_SIZE);
1904 
1905 	info->version = __stringify(TCMU_MAILBOX_VERSION);
1906 
1907 	info->mem[0].name = "tcm-user command & data buffer";
1908 	info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr;
1909 	info->mem[0].size = udev->ring_size = udev->data_size + CMDR_SIZE;
1910 	info->mem[0].memtype = UIO_MEM_NONE;
1911 
1912 	info->irqcontrol = tcmu_irqcontrol;
1913 	info->irq = UIO_IRQ_CUSTOM;
1914 
1915 	info->mmap = tcmu_mmap;
1916 	info->open = tcmu_open;
1917 	info->release = tcmu_release;
1918 
1919 	ret = uio_register_device(tcmu_root_device, info);
1920 	if (ret)
1921 		goto err_register;
1922 
1923 	/* User can set hw_block_size before enable the device */
1924 	if (dev->dev_attrib.hw_block_size == 0)
1925 		dev->dev_attrib.hw_block_size = 512;
1926 	/* Other attributes can be configured in userspace */
1927 	if (!dev->dev_attrib.hw_max_sectors)
1928 		dev->dev_attrib.hw_max_sectors = 128;
1929 	if (!dev->dev_attrib.emulate_write_cache)
1930 		dev->dev_attrib.emulate_write_cache = 0;
1931 	dev->dev_attrib.hw_queue_depth = 128;
1932 
1933 	/* If user didn't explicitly disable netlink reply support, use
1934 	 * module scope setting.
1935 	 */
1936 	if (udev->nl_reply_supported >= 0)
1937 		udev->nl_reply_supported = tcmu_kern_cmd_reply_supported;
1938 
1939 	/*
1940 	 * Get a ref incase userspace does a close on the uio device before
1941 	 * LIO has initiated tcmu_free_device.
1942 	 */
1943 	kref_get(&udev->kref);
1944 
1945 	ret = tcmu_send_dev_add_event(udev);
1946 	if (ret)
1947 		goto err_netlink;
1948 
1949 	mutex_lock(&root_udev_mutex);
1950 	list_add(&udev->node, &root_udev);
1951 	mutex_unlock(&root_udev_mutex);
1952 
1953 	return 0;
1954 
1955 err_netlink:
1956 	kref_put(&udev->kref, tcmu_dev_kref_release);
1957 	uio_unregister_device(&udev->uio_info);
1958 err_register:
1959 	vfree(udev->mb_addr);
1960 	udev->mb_addr = NULL;
1961 err_vzalloc:
1962 	bitmap_free(udev->data_bitmap);
1963 	udev->data_bitmap = NULL;
1964 err_bitmap_alloc:
1965 	kfree(info->name);
1966 	info->name = NULL;
1967 
1968 	return ret;
1969 }
1970 
1971 static void tcmu_free_device(struct se_device *dev)
1972 {
1973 	struct tcmu_dev *udev = TCMU_DEV(dev);
1974 
1975 	/* release ref from init */
1976 	kref_put(&udev->kref, tcmu_dev_kref_release);
1977 }
1978 
1979 static void tcmu_destroy_device(struct se_device *dev)
1980 {
1981 	struct tcmu_dev *udev = TCMU_DEV(dev);
1982 
1983 	del_timer_sync(&udev->cmd_timer);
1984 	del_timer_sync(&udev->qfull_timer);
1985 
1986 	mutex_lock(&root_udev_mutex);
1987 	list_del(&udev->node);
1988 	mutex_unlock(&root_udev_mutex);
1989 
1990 	tcmu_send_dev_remove_event(udev);
1991 
1992 	uio_unregister_device(&udev->uio_info);
1993 
1994 	/* release ref from configure */
1995 	kref_put(&udev->kref, tcmu_dev_kref_release);
1996 }
1997 
1998 static void tcmu_unblock_dev(struct tcmu_dev *udev)
1999 {
2000 	mutex_lock(&udev->cmdr_lock);
2001 	clear_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags);
2002 	mutex_unlock(&udev->cmdr_lock);
2003 }
2004 
2005 static void tcmu_block_dev(struct tcmu_dev *udev)
2006 {
2007 	mutex_lock(&udev->cmdr_lock);
2008 
2009 	if (test_and_set_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags))
2010 		goto unlock;
2011 
2012 	/* complete IO that has executed successfully */
2013 	tcmu_handle_completions(udev);
2014 	/* fail IO waiting to be queued */
2015 	run_qfull_queue(udev, true);
2016 
2017 unlock:
2018 	mutex_unlock(&udev->cmdr_lock);
2019 }
2020 
2021 static void tcmu_reset_ring(struct tcmu_dev *udev, u8 err_level)
2022 {
2023 	struct tcmu_mailbox *mb;
2024 	struct tcmu_cmd *cmd;
2025 	int i;
2026 
2027 	mutex_lock(&udev->cmdr_lock);
2028 
2029 	idr_for_each_entry(&udev->commands, cmd, i) {
2030 		if (!test_bit(TCMU_CMD_BIT_INFLIGHT, &cmd->flags))
2031 			continue;
2032 
2033 		pr_debug("removing cmd %u on dev %s from ring (is expired %d)\n",
2034 			  cmd->cmd_id, udev->name,
2035 			  test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags));
2036 
2037 		idr_remove(&udev->commands, i);
2038 		if (!test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
2039 			list_del_init(&cmd->queue_entry);
2040 			if (err_level == 1) {
2041 				/*
2042 				 * Userspace was not able to start the
2043 				 * command or it is retryable.
2044 				 */
2045 				target_complete_cmd(cmd->se_cmd, SAM_STAT_BUSY);
2046 			} else {
2047 				/* hard failure */
2048 				target_complete_cmd(cmd->se_cmd,
2049 						    SAM_STAT_CHECK_CONDITION);
2050 			}
2051 		}
2052 		tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
2053 		tcmu_free_cmd(cmd);
2054 	}
2055 
2056 	mb = udev->mb_addr;
2057 	tcmu_flush_dcache_range(mb, sizeof(*mb));
2058 	pr_debug("mb last %u head %u tail %u\n", udev->cmdr_last_cleaned,
2059 		 mb->cmd_tail, mb->cmd_head);
2060 
2061 	udev->cmdr_last_cleaned = 0;
2062 	mb->cmd_tail = 0;
2063 	mb->cmd_head = 0;
2064 	tcmu_flush_dcache_range(mb, sizeof(*mb));
2065 
2066 	del_timer(&udev->cmd_timer);
2067 
2068 	mutex_unlock(&udev->cmdr_lock);
2069 }
2070 
2071 enum {
2072 	Opt_dev_config, Opt_dev_size, Opt_hw_block_size, Opt_hw_max_sectors,
2073 	Opt_nl_reply_supported, Opt_max_data_area_mb, Opt_err,
2074 };
2075 
2076 static match_table_t tokens = {
2077 	{Opt_dev_config, "dev_config=%s"},
2078 	{Opt_dev_size, "dev_size=%s"},
2079 	{Opt_hw_block_size, "hw_block_size=%d"},
2080 	{Opt_hw_max_sectors, "hw_max_sectors=%d"},
2081 	{Opt_nl_reply_supported, "nl_reply_supported=%d"},
2082 	{Opt_max_data_area_mb, "max_data_area_mb=%d"},
2083 	{Opt_err, NULL}
2084 };
2085 
2086 static int tcmu_set_dev_attrib(substring_t *arg, u32 *dev_attrib)
2087 {
2088 	int val, ret;
2089 
2090 	ret = match_int(arg, &val);
2091 	if (ret < 0) {
2092 		pr_err("match_int() failed for dev attrib. Error %d.\n",
2093 		       ret);
2094 		return ret;
2095 	}
2096 
2097 	if (val <= 0) {
2098 		pr_err("Invalid dev attrib value %d. Must be greater than zero.\n",
2099 		       val);
2100 		return -EINVAL;
2101 	}
2102 	*dev_attrib = val;
2103 	return 0;
2104 }
2105 
2106 static int tcmu_set_max_blocks_param(struct tcmu_dev *udev, substring_t *arg)
2107 {
2108 	int val, ret;
2109 
2110 	ret = match_int(arg, &val);
2111 	if (ret < 0) {
2112 		pr_err("match_int() failed for max_data_area_mb=. Error %d.\n",
2113 		       ret);
2114 		return ret;
2115 	}
2116 
2117 	if (val <= 0) {
2118 		pr_err("Invalid max_data_area %d.\n", val);
2119 		return -EINVAL;
2120 	}
2121 
2122 	mutex_lock(&udev->cmdr_lock);
2123 	if (udev->data_bitmap) {
2124 		pr_err("Cannot set max_data_area_mb after it has been enabled.\n");
2125 		ret = -EINVAL;
2126 		goto unlock;
2127 	}
2128 
2129 	udev->max_blocks = TCMU_MBS_TO_BLOCKS(val);
2130 	if (udev->max_blocks > tcmu_global_max_blocks) {
2131 		pr_err("%d is too large. Adjusting max_data_area_mb to global limit of %u\n",
2132 		       val, TCMU_BLOCKS_TO_MBS(tcmu_global_max_blocks));
2133 		udev->max_blocks = tcmu_global_max_blocks;
2134 	}
2135 
2136 unlock:
2137 	mutex_unlock(&udev->cmdr_lock);
2138 	return ret;
2139 }
2140 
2141 static ssize_t tcmu_set_configfs_dev_params(struct se_device *dev,
2142 		const char *page, ssize_t count)
2143 {
2144 	struct tcmu_dev *udev = TCMU_DEV(dev);
2145 	char *orig, *ptr, *opts;
2146 	substring_t args[MAX_OPT_ARGS];
2147 	int ret = 0, token;
2148 
2149 	opts = kstrdup(page, GFP_KERNEL);
2150 	if (!opts)
2151 		return -ENOMEM;
2152 
2153 	orig = opts;
2154 
2155 	while ((ptr = strsep(&opts, ",\n")) != NULL) {
2156 		if (!*ptr)
2157 			continue;
2158 
2159 		token = match_token(ptr, tokens, args);
2160 		switch (token) {
2161 		case Opt_dev_config:
2162 			if (match_strlcpy(udev->dev_config, &args[0],
2163 					  TCMU_CONFIG_LEN) == 0) {
2164 				ret = -EINVAL;
2165 				break;
2166 			}
2167 			pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config);
2168 			break;
2169 		case Opt_dev_size:
2170 			ret = match_u64(&args[0], &udev->dev_size);
2171 			if (ret < 0)
2172 				pr_err("match_u64() failed for dev_size=. Error %d.\n",
2173 				       ret);
2174 			break;
2175 		case Opt_hw_block_size:
2176 			ret = tcmu_set_dev_attrib(&args[0],
2177 					&(dev->dev_attrib.hw_block_size));
2178 			break;
2179 		case Opt_hw_max_sectors:
2180 			ret = tcmu_set_dev_attrib(&args[0],
2181 					&(dev->dev_attrib.hw_max_sectors));
2182 			break;
2183 		case Opt_nl_reply_supported:
2184 			ret = match_int(&args[0], &udev->nl_reply_supported);
2185 			if (ret < 0)
2186 				pr_err("match_int() failed for nl_reply_supported=. Error %d.\n",
2187 				       ret);
2188 			break;
2189 		case Opt_max_data_area_mb:
2190 			ret = tcmu_set_max_blocks_param(udev, &args[0]);
2191 			break;
2192 		default:
2193 			break;
2194 		}
2195 
2196 		if (ret)
2197 			break;
2198 	}
2199 
2200 	kfree(orig);
2201 	return (!ret) ? count : ret;
2202 }
2203 
2204 static ssize_t tcmu_show_configfs_dev_params(struct se_device *dev, char *b)
2205 {
2206 	struct tcmu_dev *udev = TCMU_DEV(dev);
2207 	ssize_t bl = 0;
2208 
2209 	bl = sprintf(b + bl, "Config: %s ",
2210 		     udev->dev_config[0] ? udev->dev_config : "NULL");
2211 	bl += sprintf(b + bl, "Size: %llu ", udev->dev_size);
2212 	bl += sprintf(b + bl, "MaxDataAreaMB: %u\n",
2213 		      TCMU_BLOCKS_TO_MBS(udev->max_blocks));
2214 
2215 	return bl;
2216 }
2217 
2218 static sector_t tcmu_get_blocks(struct se_device *dev)
2219 {
2220 	struct tcmu_dev *udev = TCMU_DEV(dev);
2221 
2222 	return div_u64(udev->dev_size - dev->dev_attrib.block_size,
2223 		       dev->dev_attrib.block_size);
2224 }
2225 
2226 static sense_reason_t
2227 tcmu_parse_cdb(struct se_cmd *cmd)
2228 {
2229 	return passthrough_parse_cdb(cmd, tcmu_queue_cmd);
2230 }
2231 
2232 static ssize_t tcmu_cmd_time_out_show(struct config_item *item, char *page)
2233 {
2234 	struct se_dev_attrib *da = container_of(to_config_group(item),
2235 					struct se_dev_attrib, da_group);
2236 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2237 
2238 	return snprintf(page, PAGE_SIZE, "%lu\n", udev->cmd_time_out / MSEC_PER_SEC);
2239 }
2240 
2241 static ssize_t tcmu_cmd_time_out_store(struct config_item *item, const char *page,
2242 				       size_t count)
2243 {
2244 	struct se_dev_attrib *da = container_of(to_config_group(item),
2245 					struct se_dev_attrib, da_group);
2246 	struct tcmu_dev *udev = container_of(da->da_dev,
2247 					struct tcmu_dev, se_dev);
2248 	u32 val;
2249 	int ret;
2250 
2251 	if (da->da_dev->export_count) {
2252 		pr_err("Unable to set tcmu cmd_time_out while exports exist\n");
2253 		return -EINVAL;
2254 	}
2255 
2256 	ret = kstrtou32(page, 0, &val);
2257 	if (ret < 0)
2258 		return ret;
2259 
2260 	udev->cmd_time_out = val * MSEC_PER_SEC;
2261 	return count;
2262 }
2263 CONFIGFS_ATTR(tcmu_, cmd_time_out);
2264 
2265 static ssize_t tcmu_qfull_time_out_show(struct config_item *item, char *page)
2266 {
2267 	struct se_dev_attrib *da = container_of(to_config_group(item),
2268 						struct se_dev_attrib, da_group);
2269 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2270 
2271 	return snprintf(page, PAGE_SIZE, "%ld\n", udev->qfull_time_out <= 0 ?
2272 			udev->qfull_time_out :
2273 			udev->qfull_time_out / MSEC_PER_SEC);
2274 }
2275 
2276 static ssize_t tcmu_qfull_time_out_store(struct config_item *item,
2277 					 const char *page, size_t count)
2278 {
2279 	struct se_dev_attrib *da = container_of(to_config_group(item),
2280 					struct se_dev_attrib, da_group);
2281 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2282 	s32 val;
2283 	int ret;
2284 
2285 	ret = kstrtos32(page, 0, &val);
2286 	if (ret < 0)
2287 		return ret;
2288 
2289 	if (val >= 0) {
2290 		udev->qfull_time_out = val * MSEC_PER_SEC;
2291 	} else if (val == -1) {
2292 		udev->qfull_time_out = val;
2293 	} else {
2294 		printk(KERN_ERR "Invalid qfull timeout value %d\n", val);
2295 		return -EINVAL;
2296 	}
2297 	return count;
2298 }
2299 CONFIGFS_ATTR(tcmu_, qfull_time_out);
2300 
2301 static ssize_t tcmu_max_data_area_mb_show(struct config_item *item, char *page)
2302 {
2303 	struct se_dev_attrib *da = container_of(to_config_group(item),
2304 						struct se_dev_attrib, da_group);
2305 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2306 
2307 	return snprintf(page, PAGE_SIZE, "%u\n",
2308 			TCMU_BLOCKS_TO_MBS(udev->max_blocks));
2309 }
2310 CONFIGFS_ATTR_RO(tcmu_, max_data_area_mb);
2311 
2312 static ssize_t tcmu_dev_config_show(struct config_item *item, char *page)
2313 {
2314 	struct se_dev_attrib *da = container_of(to_config_group(item),
2315 						struct se_dev_attrib, da_group);
2316 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2317 
2318 	return snprintf(page, PAGE_SIZE, "%s\n", udev->dev_config);
2319 }
2320 
2321 static int tcmu_send_dev_config_event(struct tcmu_dev *udev,
2322 				      const char *reconfig_data)
2323 {
2324 	struct sk_buff *skb = NULL;
2325 	void *msg_header = NULL;
2326 	int ret = 0;
2327 
2328 	ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
2329 				      &skb, &msg_header);
2330 	if (ret < 0)
2331 		return ret;
2332 	ret = nla_put_string(skb, TCMU_ATTR_DEV_CFG, reconfig_data);
2333 	if (ret < 0) {
2334 		nlmsg_free(skb);
2335 		return ret;
2336 	}
2337 	return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
2338 				       skb, msg_header);
2339 }
2340 
2341 
2342 static ssize_t tcmu_dev_config_store(struct config_item *item, const char *page,
2343 				     size_t count)
2344 {
2345 	struct se_dev_attrib *da = container_of(to_config_group(item),
2346 						struct se_dev_attrib, da_group);
2347 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2348 	int ret, len;
2349 
2350 	len = strlen(page);
2351 	if (!len || len > TCMU_CONFIG_LEN - 1)
2352 		return -EINVAL;
2353 
2354 	/* Check if device has been configured before */
2355 	if (target_dev_configured(&udev->se_dev)) {
2356 		ret = tcmu_send_dev_config_event(udev, page);
2357 		if (ret) {
2358 			pr_err("Unable to reconfigure device\n");
2359 			return ret;
2360 		}
2361 		strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN);
2362 
2363 		ret = tcmu_update_uio_info(udev);
2364 		if (ret)
2365 			return ret;
2366 		return count;
2367 	}
2368 	strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN);
2369 
2370 	return count;
2371 }
2372 CONFIGFS_ATTR(tcmu_, dev_config);
2373 
2374 static ssize_t tcmu_dev_size_show(struct config_item *item, char *page)
2375 {
2376 	struct se_dev_attrib *da = container_of(to_config_group(item),
2377 						struct se_dev_attrib, da_group);
2378 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2379 
2380 	return snprintf(page, PAGE_SIZE, "%llu\n", udev->dev_size);
2381 }
2382 
2383 static int tcmu_send_dev_size_event(struct tcmu_dev *udev, u64 size)
2384 {
2385 	struct sk_buff *skb = NULL;
2386 	void *msg_header = NULL;
2387 	int ret = 0;
2388 
2389 	ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
2390 				      &skb, &msg_header);
2391 	if (ret < 0)
2392 		return ret;
2393 	ret = nla_put_u64_64bit(skb, TCMU_ATTR_DEV_SIZE,
2394 				size, TCMU_ATTR_PAD);
2395 	if (ret < 0) {
2396 		nlmsg_free(skb);
2397 		return ret;
2398 	}
2399 	return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
2400 				       skb, msg_header);
2401 }
2402 
2403 static ssize_t tcmu_dev_size_store(struct config_item *item, const char *page,
2404 				   size_t count)
2405 {
2406 	struct se_dev_attrib *da = container_of(to_config_group(item),
2407 						struct se_dev_attrib, da_group);
2408 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2409 	u64 val;
2410 	int ret;
2411 
2412 	ret = kstrtou64(page, 0, &val);
2413 	if (ret < 0)
2414 		return ret;
2415 
2416 	/* Check if device has been configured before */
2417 	if (target_dev_configured(&udev->se_dev)) {
2418 		ret = tcmu_send_dev_size_event(udev, val);
2419 		if (ret) {
2420 			pr_err("Unable to reconfigure device\n");
2421 			return ret;
2422 		}
2423 	}
2424 	udev->dev_size = val;
2425 	return count;
2426 }
2427 CONFIGFS_ATTR(tcmu_, dev_size);
2428 
2429 static ssize_t tcmu_nl_reply_supported_show(struct config_item *item,
2430 		char *page)
2431 {
2432 	struct se_dev_attrib *da = container_of(to_config_group(item),
2433 						struct se_dev_attrib, da_group);
2434 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2435 
2436 	return snprintf(page, PAGE_SIZE, "%d\n", udev->nl_reply_supported);
2437 }
2438 
2439 static ssize_t tcmu_nl_reply_supported_store(struct config_item *item,
2440 		const char *page, size_t count)
2441 {
2442 	struct se_dev_attrib *da = container_of(to_config_group(item),
2443 						struct se_dev_attrib, da_group);
2444 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2445 	s8 val;
2446 	int ret;
2447 
2448 	ret = kstrtos8(page, 0, &val);
2449 	if (ret < 0)
2450 		return ret;
2451 
2452 	udev->nl_reply_supported = val;
2453 	return count;
2454 }
2455 CONFIGFS_ATTR(tcmu_, nl_reply_supported);
2456 
2457 static ssize_t tcmu_emulate_write_cache_show(struct config_item *item,
2458 					     char *page)
2459 {
2460 	struct se_dev_attrib *da = container_of(to_config_group(item),
2461 					struct se_dev_attrib, da_group);
2462 
2463 	return snprintf(page, PAGE_SIZE, "%i\n", da->emulate_write_cache);
2464 }
2465 
2466 static int tcmu_send_emulate_write_cache(struct tcmu_dev *udev, u8 val)
2467 {
2468 	struct sk_buff *skb = NULL;
2469 	void *msg_header = NULL;
2470 	int ret = 0;
2471 
2472 	ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
2473 				      &skb, &msg_header);
2474 	if (ret < 0)
2475 		return ret;
2476 	ret = nla_put_u8(skb, TCMU_ATTR_WRITECACHE, val);
2477 	if (ret < 0) {
2478 		nlmsg_free(skb);
2479 		return ret;
2480 	}
2481 	return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
2482 				       skb, msg_header);
2483 }
2484 
2485 static ssize_t tcmu_emulate_write_cache_store(struct config_item *item,
2486 					      const char *page, size_t count)
2487 {
2488 	struct se_dev_attrib *da = container_of(to_config_group(item),
2489 					struct se_dev_attrib, da_group);
2490 	struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
2491 	u8 val;
2492 	int ret;
2493 
2494 	ret = kstrtou8(page, 0, &val);
2495 	if (ret < 0)
2496 		return ret;
2497 
2498 	/* Check if device has been configured before */
2499 	if (target_dev_configured(&udev->se_dev)) {
2500 		ret = tcmu_send_emulate_write_cache(udev, val);
2501 		if (ret) {
2502 			pr_err("Unable to reconfigure device\n");
2503 			return ret;
2504 		}
2505 	}
2506 
2507 	da->emulate_write_cache = val;
2508 	return count;
2509 }
2510 CONFIGFS_ATTR(tcmu_, emulate_write_cache);
2511 
2512 static ssize_t tcmu_block_dev_show(struct config_item *item, char *page)
2513 {
2514 	struct se_device *se_dev = container_of(to_config_group(item),
2515 						struct se_device,
2516 						dev_action_group);
2517 	struct tcmu_dev *udev = TCMU_DEV(se_dev);
2518 
2519 	if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags))
2520 		return snprintf(page, PAGE_SIZE, "%s\n", "blocked");
2521 	else
2522 		return snprintf(page, PAGE_SIZE, "%s\n", "unblocked");
2523 }
2524 
2525 static ssize_t tcmu_block_dev_store(struct config_item *item, const char *page,
2526 				    size_t count)
2527 {
2528 	struct se_device *se_dev = container_of(to_config_group(item),
2529 						struct se_device,
2530 						dev_action_group);
2531 	struct tcmu_dev *udev = TCMU_DEV(se_dev);
2532 	u8 val;
2533 	int ret;
2534 
2535 	if (!target_dev_configured(&udev->se_dev)) {
2536 		pr_err("Device is not configured.\n");
2537 		return -EINVAL;
2538 	}
2539 
2540 	ret = kstrtou8(page, 0, &val);
2541 	if (ret < 0)
2542 		return ret;
2543 
2544 	if (val > 1) {
2545 		pr_err("Invalid block value %d\n", val);
2546 		return -EINVAL;
2547 	}
2548 
2549 	if (!val)
2550 		tcmu_unblock_dev(udev);
2551 	else
2552 		tcmu_block_dev(udev);
2553 	return count;
2554 }
2555 CONFIGFS_ATTR(tcmu_, block_dev);
2556 
2557 static ssize_t tcmu_reset_ring_store(struct config_item *item, const char *page,
2558 				     size_t count)
2559 {
2560 	struct se_device *se_dev = container_of(to_config_group(item),
2561 						struct se_device,
2562 						dev_action_group);
2563 	struct tcmu_dev *udev = TCMU_DEV(se_dev);
2564 	u8 val;
2565 	int ret;
2566 
2567 	if (!target_dev_configured(&udev->se_dev)) {
2568 		pr_err("Device is not configured.\n");
2569 		return -EINVAL;
2570 	}
2571 
2572 	ret = kstrtou8(page, 0, &val);
2573 	if (ret < 0)
2574 		return ret;
2575 
2576 	if (val != 1 && val != 2) {
2577 		pr_err("Invalid reset ring value %d\n", val);
2578 		return -EINVAL;
2579 	}
2580 
2581 	tcmu_reset_ring(udev, val);
2582 	return count;
2583 }
2584 CONFIGFS_ATTR_WO(tcmu_, reset_ring);
2585 
2586 static struct configfs_attribute *tcmu_attrib_attrs[] = {
2587 	&tcmu_attr_cmd_time_out,
2588 	&tcmu_attr_qfull_time_out,
2589 	&tcmu_attr_max_data_area_mb,
2590 	&tcmu_attr_dev_config,
2591 	&tcmu_attr_dev_size,
2592 	&tcmu_attr_emulate_write_cache,
2593 	&tcmu_attr_nl_reply_supported,
2594 	NULL,
2595 };
2596 
2597 static struct configfs_attribute **tcmu_attrs;
2598 
2599 static struct configfs_attribute *tcmu_action_attrs[] = {
2600 	&tcmu_attr_block_dev,
2601 	&tcmu_attr_reset_ring,
2602 	NULL,
2603 };
2604 
2605 static struct target_backend_ops tcmu_ops = {
2606 	.name			= "user",
2607 	.owner			= THIS_MODULE,
2608 	.transport_flags	= TRANSPORT_FLAG_PASSTHROUGH,
2609 	.attach_hba		= tcmu_attach_hba,
2610 	.detach_hba		= tcmu_detach_hba,
2611 	.alloc_device		= tcmu_alloc_device,
2612 	.configure_device	= tcmu_configure_device,
2613 	.destroy_device		= tcmu_destroy_device,
2614 	.free_device		= tcmu_free_device,
2615 	.parse_cdb		= tcmu_parse_cdb,
2616 	.set_configfs_dev_params = tcmu_set_configfs_dev_params,
2617 	.show_configfs_dev_params = tcmu_show_configfs_dev_params,
2618 	.get_device_type	= sbc_get_device_type,
2619 	.get_blocks		= tcmu_get_blocks,
2620 	.tb_dev_action_attrs	= tcmu_action_attrs,
2621 };
2622 
2623 static void find_free_blocks(void)
2624 {
2625 	struct tcmu_dev *udev;
2626 	loff_t off;
2627 	u32 start, end, block, total_freed = 0;
2628 
2629 	if (atomic_read(&global_db_count) <= tcmu_global_max_blocks)
2630 		return;
2631 
2632 	mutex_lock(&root_udev_mutex);
2633 	list_for_each_entry(udev, &root_udev, node) {
2634 		mutex_lock(&udev->cmdr_lock);
2635 
2636 		if (!target_dev_configured(&udev->se_dev)) {
2637 			mutex_unlock(&udev->cmdr_lock);
2638 			continue;
2639 		}
2640 
2641 		/* Try to complete the finished commands first */
2642 		tcmu_handle_completions(udev);
2643 
2644 		/* Skip the udevs in idle */
2645 		if (!udev->dbi_thresh) {
2646 			mutex_unlock(&udev->cmdr_lock);
2647 			continue;
2648 		}
2649 
2650 		end = udev->dbi_max + 1;
2651 		block = find_last_bit(udev->data_bitmap, end);
2652 		if (block == udev->dbi_max) {
2653 			/*
2654 			 * The last bit is dbi_max, so it is not possible
2655 			 * reclaim any blocks.
2656 			 */
2657 			mutex_unlock(&udev->cmdr_lock);
2658 			continue;
2659 		} else if (block == end) {
2660 			/* The current udev will goto idle state */
2661 			udev->dbi_thresh = start = 0;
2662 			udev->dbi_max = 0;
2663 		} else {
2664 			udev->dbi_thresh = start = block + 1;
2665 			udev->dbi_max = block;
2666 		}
2667 
2668 		/* Here will truncate the data area from off */
2669 		off = udev->data_off + start * DATA_BLOCK_SIZE;
2670 		unmap_mapping_range(udev->inode->i_mapping, off, 0, 1);
2671 
2672 		/* Release the block pages */
2673 		tcmu_blocks_release(&udev->data_blocks, start, end);
2674 		mutex_unlock(&udev->cmdr_lock);
2675 
2676 		total_freed += end - start;
2677 		pr_debug("Freed %u blocks (total %u) from %s.\n", end - start,
2678 			 total_freed, udev->name);
2679 	}
2680 	mutex_unlock(&root_udev_mutex);
2681 
2682 	if (atomic_read(&global_db_count) > tcmu_global_max_blocks)
2683 		schedule_delayed_work(&tcmu_unmap_work, msecs_to_jiffies(5000));
2684 }
2685 
2686 static void check_timedout_devices(void)
2687 {
2688 	struct tcmu_dev *udev, *tmp_dev;
2689 	LIST_HEAD(devs);
2690 
2691 	spin_lock_bh(&timed_out_udevs_lock);
2692 	list_splice_init(&timed_out_udevs, &devs);
2693 
2694 	list_for_each_entry_safe(udev, tmp_dev, &devs, timedout_entry) {
2695 		list_del_init(&udev->timedout_entry);
2696 		spin_unlock_bh(&timed_out_udevs_lock);
2697 
2698 		mutex_lock(&udev->cmdr_lock);
2699 		idr_for_each(&udev->commands, tcmu_check_expired_cmd, NULL);
2700 
2701 		tcmu_set_next_deadline(&udev->inflight_queue, &udev->cmd_timer);
2702 		tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
2703 
2704 		mutex_unlock(&udev->cmdr_lock);
2705 
2706 		spin_lock_bh(&timed_out_udevs_lock);
2707 	}
2708 
2709 	spin_unlock_bh(&timed_out_udevs_lock);
2710 }
2711 
2712 static void tcmu_unmap_work_fn(struct work_struct *work)
2713 {
2714 	check_timedout_devices();
2715 	find_free_blocks();
2716 }
2717 
2718 static int __init tcmu_module_init(void)
2719 {
2720 	int ret, i, k, len = 0;
2721 
2722 	BUILD_BUG_ON((sizeof(struct tcmu_cmd_entry) % TCMU_OP_ALIGN_SIZE) != 0);
2723 
2724 	INIT_DELAYED_WORK(&tcmu_unmap_work, tcmu_unmap_work_fn);
2725 
2726 	tcmu_cmd_cache = kmem_cache_create("tcmu_cmd_cache",
2727 				sizeof(struct tcmu_cmd),
2728 				__alignof__(struct tcmu_cmd),
2729 				0, NULL);
2730 	if (!tcmu_cmd_cache)
2731 		return -ENOMEM;
2732 
2733 	tcmu_root_device = root_device_register("tcm_user");
2734 	if (IS_ERR(tcmu_root_device)) {
2735 		ret = PTR_ERR(tcmu_root_device);
2736 		goto out_free_cache;
2737 	}
2738 
2739 	ret = genl_register_family(&tcmu_genl_family);
2740 	if (ret < 0) {
2741 		goto out_unreg_device;
2742 	}
2743 
2744 	for (i = 0; passthrough_attrib_attrs[i] != NULL; i++) {
2745 		len += sizeof(struct configfs_attribute *);
2746 	}
2747 	for (i = 0; tcmu_attrib_attrs[i] != NULL; i++) {
2748 		len += sizeof(struct configfs_attribute *);
2749 	}
2750 	len += sizeof(struct configfs_attribute *);
2751 
2752 	tcmu_attrs = kzalloc(len, GFP_KERNEL);
2753 	if (!tcmu_attrs) {
2754 		ret = -ENOMEM;
2755 		goto out_unreg_genl;
2756 	}
2757 
2758 	for (i = 0; passthrough_attrib_attrs[i] != NULL; i++) {
2759 		tcmu_attrs[i] = passthrough_attrib_attrs[i];
2760 	}
2761 	for (k = 0; tcmu_attrib_attrs[k] != NULL; k++) {
2762 		tcmu_attrs[i] = tcmu_attrib_attrs[k];
2763 		i++;
2764 	}
2765 	tcmu_ops.tb_dev_attrib_attrs = tcmu_attrs;
2766 
2767 	ret = transport_backend_register(&tcmu_ops);
2768 	if (ret)
2769 		goto out_attrs;
2770 
2771 	return 0;
2772 
2773 out_attrs:
2774 	kfree(tcmu_attrs);
2775 out_unreg_genl:
2776 	genl_unregister_family(&tcmu_genl_family);
2777 out_unreg_device:
2778 	root_device_unregister(tcmu_root_device);
2779 out_free_cache:
2780 	kmem_cache_destroy(tcmu_cmd_cache);
2781 
2782 	return ret;
2783 }
2784 
2785 static void __exit tcmu_module_exit(void)
2786 {
2787 	cancel_delayed_work_sync(&tcmu_unmap_work);
2788 	target_backend_unregister(&tcmu_ops);
2789 	kfree(tcmu_attrs);
2790 	genl_unregister_family(&tcmu_genl_family);
2791 	root_device_unregister(tcmu_root_device);
2792 	kmem_cache_destroy(tcmu_cmd_cache);
2793 }
2794 
2795 MODULE_DESCRIPTION("TCM USER subsystem plugin");
2796 MODULE_AUTHOR("Shaohua Li <shli@kernel.org>");
2797 MODULE_AUTHOR("Andy Grover <agrover@redhat.com>");
2798 MODULE_LICENSE("GPL");
2799 
2800 module_init(tcmu_module_init);
2801 module_exit(tcmu_module_exit);
2802