1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*******************************************************************************
3  * Filename:  target_core_iblock.c
4  *
5  * This file contains the Storage Engine  <-> Linux BlockIO transport
6  * specific functions.
7  *
8  * (c) Copyright 2003-2013 Datera, Inc.
9  *
10  * Nicholas A. Bellinger <nab@kernel.org>
11  *
12  ******************************************************************************/
13 
14 #include <linux/string.h>
15 #include <linux/parser.h>
16 #include <linux/timer.h>
17 #include <linux/fs.h>
18 #include <linux/blkdev.h>
19 #include <linux/blk-integrity.h>
20 #include <linux/slab.h>
21 #include <linux/spinlock.h>
22 #include <linux/bio.h>
23 #include <linux/file.h>
24 #include <linux/module.h>
25 #include <linux/scatterlist.h>
26 #include <scsi/scsi_proto.h>
27 #include <asm/unaligned.h>
28 
29 #include <target/target_core_base.h>
30 #include <target/target_core_backend.h>
31 
32 #include "target_core_iblock.h"
33 
34 #define IBLOCK_MAX_BIO_PER_TASK	 32	/* max # of bios to submit at a time */
35 #define IBLOCK_BIO_POOL_SIZE	128
36 
37 static inline struct iblock_dev *IBLOCK_DEV(struct se_device *dev)
38 {
39 	return container_of(dev, struct iblock_dev, dev);
40 }
41 
42 
43 static int iblock_attach_hba(struct se_hba *hba, u32 host_id)
44 {
45 	pr_debug("CORE_HBA[%d] - TCM iBlock HBA Driver %s on"
46 		" Generic Target Core Stack %s\n", hba->hba_id,
47 		IBLOCK_VERSION, TARGET_CORE_VERSION);
48 	return 0;
49 }
50 
51 static void iblock_detach_hba(struct se_hba *hba)
52 {
53 }
54 
55 static struct se_device *iblock_alloc_device(struct se_hba *hba, const char *name)
56 {
57 	struct iblock_dev *ib_dev = NULL;
58 
59 	ib_dev = kzalloc(sizeof(struct iblock_dev), GFP_KERNEL);
60 	if (!ib_dev) {
61 		pr_err("Unable to allocate struct iblock_dev\n");
62 		return NULL;
63 	}
64 
65 	ib_dev->ibd_plug = kcalloc(nr_cpu_ids, sizeof(*ib_dev->ibd_plug),
66 				   GFP_KERNEL);
67 	if (!ib_dev->ibd_plug)
68 		goto free_dev;
69 
70 	pr_debug( "IBLOCK: Allocated ib_dev for %s\n", name);
71 
72 	return &ib_dev->dev;
73 
74 free_dev:
75 	kfree(ib_dev);
76 	return NULL;
77 }
78 
79 static bool iblock_configure_unmap(struct se_device *dev)
80 {
81 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
82 
83 	return target_configure_unmap_from_queue(&dev->dev_attrib,
84 						 ib_dev->ibd_bd);
85 }
86 
87 static int iblock_configure_device(struct se_device *dev)
88 {
89 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
90 	struct request_queue *q;
91 	struct block_device *bd = NULL;
92 	struct blk_integrity *bi;
93 	fmode_t mode;
94 	unsigned int max_write_zeroes_sectors;
95 	int ret;
96 
97 	if (!(ib_dev->ibd_flags & IBDF_HAS_UDEV_PATH)) {
98 		pr_err("Missing udev_path= parameters for IBLOCK\n");
99 		return -EINVAL;
100 	}
101 
102 	ret = bioset_init(&ib_dev->ibd_bio_set, IBLOCK_BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
103 	if (ret) {
104 		pr_err("IBLOCK: Unable to create bioset\n");
105 		goto out;
106 	}
107 
108 	pr_debug( "IBLOCK: Claiming struct block_device: %s\n",
109 			ib_dev->ibd_udev_path);
110 
111 	mode = FMODE_READ|FMODE_EXCL;
112 	if (!ib_dev->ibd_readonly)
113 		mode |= FMODE_WRITE;
114 	else
115 		dev->dev_flags |= DF_READ_ONLY;
116 
117 	bd = blkdev_get_by_path(ib_dev->ibd_udev_path, mode, ib_dev);
118 	if (IS_ERR(bd)) {
119 		ret = PTR_ERR(bd);
120 		goto out_free_bioset;
121 	}
122 	ib_dev->ibd_bd = bd;
123 
124 	q = bdev_get_queue(bd);
125 
126 	dev->dev_attrib.hw_block_size = bdev_logical_block_size(bd);
127 	dev->dev_attrib.hw_max_sectors = mult_frac(queue_max_hw_sectors(q),
128 			SECTOR_SIZE,
129 			dev->dev_attrib.hw_block_size);
130 	dev->dev_attrib.hw_queue_depth = q->nr_requests;
131 
132 	/*
133 	 * Enable write same emulation for IBLOCK and use 0xFFFF as
134 	 * the smaller WRITE_SAME(10) only has a two-byte block count.
135 	 */
136 	max_write_zeroes_sectors = bdev_write_zeroes_sectors(bd);
137 	if (max_write_zeroes_sectors)
138 		dev->dev_attrib.max_write_same_len = max_write_zeroes_sectors;
139 	else
140 		dev->dev_attrib.max_write_same_len = 0xFFFF;
141 
142 	if (bdev_nonrot(bd))
143 		dev->dev_attrib.is_nonrot = 1;
144 
145 	bi = bdev_get_integrity(bd);
146 	if (bi) {
147 		struct bio_set *bs = &ib_dev->ibd_bio_set;
148 
149 		if (!strcmp(bi->profile->name, "T10-DIF-TYPE3-IP") ||
150 		    !strcmp(bi->profile->name, "T10-DIF-TYPE1-IP")) {
151 			pr_err("IBLOCK export of blk_integrity: %s not"
152 			       " supported\n", bi->profile->name);
153 			ret = -ENOSYS;
154 			goto out_blkdev_put;
155 		}
156 
157 		if (!strcmp(bi->profile->name, "T10-DIF-TYPE3-CRC")) {
158 			dev->dev_attrib.pi_prot_type = TARGET_DIF_TYPE3_PROT;
159 		} else if (!strcmp(bi->profile->name, "T10-DIF-TYPE1-CRC")) {
160 			dev->dev_attrib.pi_prot_type = TARGET_DIF_TYPE1_PROT;
161 		}
162 
163 		if (dev->dev_attrib.pi_prot_type) {
164 			if (bioset_integrity_create(bs, IBLOCK_BIO_POOL_SIZE) < 0) {
165 				pr_err("Unable to allocate bioset for PI\n");
166 				ret = -ENOMEM;
167 				goto out_blkdev_put;
168 			}
169 			pr_debug("IBLOCK setup BIP bs->bio_integrity_pool: %p\n",
170 				 &bs->bio_integrity_pool);
171 		}
172 		dev->dev_attrib.hw_pi_prot_type = dev->dev_attrib.pi_prot_type;
173 	}
174 
175 	return 0;
176 
177 out_blkdev_put:
178 	blkdev_put(ib_dev->ibd_bd, FMODE_WRITE|FMODE_READ|FMODE_EXCL);
179 out_free_bioset:
180 	bioset_exit(&ib_dev->ibd_bio_set);
181 out:
182 	return ret;
183 }
184 
185 static void iblock_dev_call_rcu(struct rcu_head *p)
186 {
187 	struct se_device *dev = container_of(p, struct se_device, rcu_head);
188 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
189 
190 	kfree(ib_dev->ibd_plug);
191 	kfree(ib_dev);
192 }
193 
194 static void iblock_free_device(struct se_device *dev)
195 {
196 	call_rcu(&dev->rcu_head, iblock_dev_call_rcu);
197 }
198 
199 static void iblock_destroy_device(struct se_device *dev)
200 {
201 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
202 
203 	if (ib_dev->ibd_bd != NULL)
204 		blkdev_put(ib_dev->ibd_bd, FMODE_WRITE|FMODE_READ|FMODE_EXCL);
205 	bioset_exit(&ib_dev->ibd_bio_set);
206 }
207 
208 static struct se_dev_plug *iblock_plug_device(struct se_device *se_dev)
209 {
210 	struct iblock_dev *ib_dev = IBLOCK_DEV(se_dev);
211 	struct iblock_dev_plug *ib_dev_plug;
212 
213 	/*
214 	 * Each se_device has a per cpu work this can be run from. We
215 	 * shouldn't have multiple threads on the same cpu calling this
216 	 * at the same time.
217 	 */
218 	ib_dev_plug = &ib_dev->ibd_plug[raw_smp_processor_id()];
219 	if (test_and_set_bit(IBD_PLUGF_PLUGGED, &ib_dev_plug->flags))
220 		return NULL;
221 
222 	blk_start_plug(&ib_dev_plug->blk_plug);
223 	return &ib_dev_plug->se_plug;
224 }
225 
226 static void iblock_unplug_device(struct se_dev_plug *se_plug)
227 {
228 	struct iblock_dev_plug *ib_dev_plug = container_of(se_plug,
229 					struct iblock_dev_plug, se_plug);
230 
231 	blk_finish_plug(&ib_dev_plug->blk_plug);
232 	clear_bit(IBD_PLUGF_PLUGGED, &ib_dev_plug->flags);
233 }
234 
235 static sector_t iblock_get_blocks(struct se_device *dev)
236 {
237 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
238 	u32 block_size = bdev_logical_block_size(ib_dev->ibd_bd);
239 	unsigned long long blocks_long =
240 		div_u64(bdev_nr_bytes(ib_dev->ibd_bd), block_size) - 1;
241 
242 	if (block_size == dev->dev_attrib.block_size)
243 		return blocks_long;
244 
245 	switch (block_size) {
246 	case 4096:
247 		switch (dev->dev_attrib.block_size) {
248 		case 2048:
249 			blocks_long <<= 1;
250 			break;
251 		case 1024:
252 			blocks_long <<= 2;
253 			break;
254 		case 512:
255 			blocks_long <<= 3;
256 			break;
257 		default:
258 			break;
259 		}
260 		break;
261 	case 2048:
262 		switch (dev->dev_attrib.block_size) {
263 		case 4096:
264 			blocks_long >>= 1;
265 			break;
266 		case 1024:
267 			blocks_long <<= 1;
268 			break;
269 		case 512:
270 			blocks_long <<= 2;
271 			break;
272 		default:
273 			break;
274 		}
275 		break;
276 	case 1024:
277 		switch (dev->dev_attrib.block_size) {
278 		case 4096:
279 			blocks_long >>= 2;
280 			break;
281 		case 2048:
282 			blocks_long >>= 1;
283 			break;
284 		case 512:
285 			blocks_long <<= 1;
286 			break;
287 		default:
288 			break;
289 		}
290 		break;
291 	case 512:
292 		switch (dev->dev_attrib.block_size) {
293 		case 4096:
294 			blocks_long >>= 3;
295 			break;
296 		case 2048:
297 			blocks_long >>= 2;
298 			break;
299 		case 1024:
300 			blocks_long >>= 1;
301 			break;
302 		default:
303 			break;
304 		}
305 		break;
306 	default:
307 		break;
308 	}
309 
310 	return blocks_long;
311 }
312 
313 static void iblock_complete_cmd(struct se_cmd *cmd)
314 {
315 	struct iblock_req *ibr = cmd->priv;
316 	u8 status;
317 
318 	if (!refcount_dec_and_test(&ibr->pending))
319 		return;
320 
321 	if (atomic_read(&ibr->ib_bio_err_cnt))
322 		status = SAM_STAT_CHECK_CONDITION;
323 	else
324 		status = SAM_STAT_GOOD;
325 
326 	target_complete_cmd(cmd, status);
327 	kfree(ibr);
328 }
329 
330 static void iblock_bio_done(struct bio *bio)
331 {
332 	struct se_cmd *cmd = bio->bi_private;
333 	struct iblock_req *ibr = cmd->priv;
334 
335 	if (bio->bi_status) {
336 		pr_err("bio error: %p,  err: %d\n", bio, bio->bi_status);
337 		/*
338 		 * Bump the ib_bio_err_cnt and release bio.
339 		 */
340 		atomic_inc(&ibr->ib_bio_err_cnt);
341 		smp_mb__after_atomic();
342 	}
343 
344 	bio_put(bio);
345 
346 	iblock_complete_cmd(cmd);
347 }
348 
349 static struct bio *iblock_get_bio(struct se_cmd *cmd, sector_t lba, u32 sg_num,
350 				  blk_opf_t opf)
351 {
352 	struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
353 	struct bio *bio;
354 
355 	/*
356 	 * Only allocate as many vector entries as the bio code allows us to,
357 	 * we'll loop later on until we have handled the whole request.
358 	 */
359 	bio = bio_alloc_bioset(ib_dev->ibd_bd, bio_max_segs(sg_num), opf,
360 			       GFP_NOIO, &ib_dev->ibd_bio_set);
361 	if (!bio) {
362 		pr_err("Unable to allocate memory for bio\n");
363 		return NULL;
364 	}
365 
366 	bio->bi_private = cmd;
367 	bio->bi_end_io = &iblock_bio_done;
368 	bio->bi_iter.bi_sector = lba;
369 
370 	return bio;
371 }
372 
373 static void iblock_submit_bios(struct bio_list *list)
374 {
375 	struct blk_plug plug;
376 	struct bio *bio;
377 	/*
378 	 * The block layer handles nested plugs, so just plug/unplug to handle
379 	 * fabric drivers that didn't support batching and multi bio cmds.
380 	 */
381 	blk_start_plug(&plug);
382 	while ((bio = bio_list_pop(list)))
383 		submit_bio(bio);
384 	blk_finish_plug(&plug);
385 }
386 
387 static void iblock_end_io_flush(struct bio *bio)
388 {
389 	struct se_cmd *cmd = bio->bi_private;
390 
391 	if (bio->bi_status)
392 		pr_err("IBLOCK: cache flush failed: %d\n", bio->bi_status);
393 
394 	if (cmd) {
395 		if (bio->bi_status)
396 			target_complete_cmd(cmd, SAM_STAT_CHECK_CONDITION);
397 		else
398 			target_complete_cmd(cmd, SAM_STAT_GOOD);
399 	}
400 
401 	bio_put(bio);
402 }
403 
404 /*
405  * Implement SYCHRONIZE CACHE.  Note that we can't handle lba ranges and must
406  * always flush the whole cache.
407  */
408 static sense_reason_t
409 iblock_execute_sync_cache(struct se_cmd *cmd)
410 {
411 	struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
412 	int immed = (cmd->t_task_cdb[1] & 0x2);
413 	struct bio *bio;
414 
415 	/*
416 	 * If the Immediate bit is set, queue up the GOOD response
417 	 * for this SYNCHRONIZE_CACHE op.
418 	 */
419 	if (immed)
420 		target_complete_cmd(cmd, SAM_STAT_GOOD);
421 
422 	bio = bio_alloc(ib_dev->ibd_bd, 0, REQ_OP_WRITE | REQ_PREFLUSH,
423 			GFP_KERNEL);
424 	bio->bi_end_io = iblock_end_io_flush;
425 	if (!immed)
426 		bio->bi_private = cmd;
427 	submit_bio(bio);
428 	return 0;
429 }
430 
431 static sense_reason_t
432 iblock_execute_unmap(struct se_cmd *cmd, sector_t lba, sector_t nolb)
433 {
434 	struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
435 	struct se_device *dev = cmd->se_dev;
436 	int ret;
437 
438 	ret = blkdev_issue_discard(bdev,
439 				   target_to_linux_sector(dev, lba),
440 				   target_to_linux_sector(dev,  nolb),
441 				   GFP_KERNEL);
442 	if (ret < 0) {
443 		pr_err("blkdev_issue_discard() failed: %d\n", ret);
444 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
445 	}
446 
447 	return 0;
448 }
449 
450 static sense_reason_t
451 iblock_execute_zero_out(struct block_device *bdev, struct se_cmd *cmd)
452 {
453 	struct se_device *dev = cmd->se_dev;
454 	struct scatterlist *sg = &cmd->t_data_sg[0];
455 	unsigned char *buf, *not_zero;
456 	int ret;
457 
458 	buf = kmap(sg_page(sg)) + sg->offset;
459 	if (!buf)
460 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
461 	/*
462 	 * Fall back to block_execute_write_same() slow-path if
463 	 * incoming WRITE_SAME payload does not contain zeros.
464 	 */
465 	not_zero = memchr_inv(buf, 0x00, cmd->data_length);
466 	kunmap(sg_page(sg));
467 
468 	if (not_zero)
469 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
470 
471 	ret = blkdev_issue_zeroout(bdev,
472 				target_to_linux_sector(dev, cmd->t_task_lba),
473 				target_to_linux_sector(dev,
474 					sbc_get_write_same_sectors(cmd)),
475 				GFP_KERNEL, BLKDEV_ZERO_NOUNMAP);
476 	if (ret)
477 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
478 
479 	target_complete_cmd(cmd, SAM_STAT_GOOD);
480 	return 0;
481 }
482 
483 static sense_reason_t
484 iblock_execute_write_same(struct se_cmd *cmd)
485 {
486 	struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
487 	struct iblock_req *ibr;
488 	struct scatterlist *sg;
489 	struct bio *bio;
490 	struct bio_list list;
491 	struct se_device *dev = cmd->se_dev;
492 	sector_t block_lba = target_to_linux_sector(dev, cmd->t_task_lba);
493 	sector_t sectors = target_to_linux_sector(dev,
494 					sbc_get_write_same_sectors(cmd));
495 
496 	if (cmd->prot_op) {
497 		pr_err("WRITE_SAME: Protection information with IBLOCK"
498 		       " backends not supported\n");
499 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
500 	}
501 
502 	if (!cmd->t_data_nents)
503 		return TCM_INVALID_CDB_FIELD;
504 
505 	sg = &cmd->t_data_sg[0];
506 
507 	if (cmd->t_data_nents > 1 ||
508 	    sg->length != cmd->se_dev->dev_attrib.block_size) {
509 		pr_err("WRITE_SAME: Illegal SGL t_data_nents: %u length: %u"
510 			" block_size: %u\n", cmd->t_data_nents, sg->length,
511 			cmd->se_dev->dev_attrib.block_size);
512 		return TCM_INVALID_CDB_FIELD;
513 	}
514 
515 	if (bdev_write_zeroes_sectors(bdev)) {
516 		if (!iblock_execute_zero_out(bdev, cmd))
517 			return 0;
518 	}
519 
520 	ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
521 	if (!ibr)
522 		goto fail;
523 	cmd->priv = ibr;
524 
525 	bio = iblock_get_bio(cmd, block_lba, 1, REQ_OP_WRITE);
526 	if (!bio)
527 		goto fail_free_ibr;
528 
529 	bio_list_init(&list);
530 	bio_list_add(&list, bio);
531 
532 	refcount_set(&ibr->pending, 1);
533 
534 	while (sectors) {
535 		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
536 				!= sg->length) {
537 
538 			bio = iblock_get_bio(cmd, block_lba, 1, REQ_OP_WRITE);
539 			if (!bio)
540 				goto fail_put_bios;
541 
542 			refcount_inc(&ibr->pending);
543 			bio_list_add(&list, bio);
544 		}
545 
546 		/* Always in 512 byte units for Linux/Block */
547 		block_lba += sg->length >> SECTOR_SHIFT;
548 		sectors -= sg->length >> SECTOR_SHIFT;
549 	}
550 
551 	iblock_submit_bios(&list);
552 	return 0;
553 
554 fail_put_bios:
555 	while ((bio = bio_list_pop(&list)))
556 		bio_put(bio);
557 fail_free_ibr:
558 	kfree(ibr);
559 fail:
560 	return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
561 }
562 
563 enum {
564 	Opt_udev_path, Opt_readonly, Opt_force, Opt_err
565 };
566 
567 static match_table_t tokens = {
568 	{Opt_udev_path, "udev_path=%s"},
569 	{Opt_readonly, "readonly=%d"},
570 	{Opt_force, "force=%d"},
571 	{Opt_err, NULL}
572 };
573 
574 static ssize_t iblock_set_configfs_dev_params(struct se_device *dev,
575 		const char *page, ssize_t count)
576 {
577 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
578 	char *orig, *ptr, *arg_p, *opts;
579 	substring_t args[MAX_OPT_ARGS];
580 	int ret = 0, token;
581 	unsigned long tmp_readonly;
582 
583 	opts = kstrdup(page, GFP_KERNEL);
584 	if (!opts)
585 		return -ENOMEM;
586 
587 	orig = opts;
588 
589 	while ((ptr = strsep(&opts, ",\n")) != NULL) {
590 		if (!*ptr)
591 			continue;
592 
593 		token = match_token(ptr, tokens, args);
594 		switch (token) {
595 		case Opt_udev_path:
596 			if (ib_dev->ibd_bd) {
597 				pr_err("Unable to set udev_path= while"
598 					" ib_dev->ibd_bd exists\n");
599 				ret = -EEXIST;
600 				goto out;
601 			}
602 			if (match_strlcpy(ib_dev->ibd_udev_path, &args[0],
603 				SE_UDEV_PATH_LEN) == 0) {
604 				ret = -EINVAL;
605 				break;
606 			}
607 			pr_debug("IBLOCK: Referencing UDEV path: %s\n",
608 					ib_dev->ibd_udev_path);
609 			ib_dev->ibd_flags |= IBDF_HAS_UDEV_PATH;
610 			break;
611 		case Opt_readonly:
612 			arg_p = match_strdup(&args[0]);
613 			if (!arg_p) {
614 				ret = -ENOMEM;
615 				break;
616 			}
617 			ret = kstrtoul(arg_p, 0, &tmp_readonly);
618 			kfree(arg_p);
619 			if (ret < 0) {
620 				pr_err("kstrtoul() failed for"
621 						" readonly=\n");
622 				goto out;
623 			}
624 			ib_dev->ibd_readonly = tmp_readonly;
625 			pr_debug("IBLOCK: readonly: %d\n", ib_dev->ibd_readonly);
626 			break;
627 		case Opt_force:
628 			break;
629 		default:
630 			break;
631 		}
632 	}
633 
634 out:
635 	kfree(orig);
636 	return (!ret) ? count : ret;
637 }
638 
639 static ssize_t iblock_show_configfs_dev_params(struct se_device *dev, char *b)
640 {
641 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
642 	struct block_device *bd = ib_dev->ibd_bd;
643 	ssize_t bl = 0;
644 
645 	if (bd)
646 		bl += sprintf(b + bl, "iBlock device: %pg", bd);
647 	if (ib_dev->ibd_flags & IBDF_HAS_UDEV_PATH)
648 		bl += sprintf(b + bl, "  UDEV PATH: %s",
649 				ib_dev->ibd_udev_path);
650 	bl += sprintf(b + bl, "  readonly: %d\n", ib_dev->ibd_readonly);
651 
652 	bl += sprintf(b + bl, "        ");
653 	if (bd) {
654 		bl += sprintf(b + bl, "Major: %d Minor: %d  %s\n",
655 			MAJOR(bd->bd_dev), MINOR(bd->bd_dev),
656 			"CLAIMED: IBLOCK");
657 	} else {
658 		bl += sprintf(b + bl, "Major: 0 Minor: 0\n");
659 	}
660 
661 	return bl;
662 }
663 
664 static int
665 iblock_alloc_bip(struct se_cmd *cmd, struct bio *bio,
666 		 struct sg_mapping_iter *miter)
667 {
668 	struct se_device *dev = cmd->se_dev;
669 	struct blk_integrity *bi;
670 	struct bio_integrity_payload *bip;
671 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
672 	int rc;
673 	size_t resid, len;
674 
675 	bi = bdev_get_integrity(ib_dev->ibd_bd);
676 	if (!bi) {
677 		pr_err("Unable to locate bio_integrity\n");
678 		return -ENODEV;
679 	}
680 
681 	bip = bio_integrity_alloc(bio, GFP_NOIO, bio_max_segs(cmd->t_prot_nents));
682 	if (IS_ERR(bip)) {
683 		pr_err("Unable to allocate bio_integrity_payload\n");
684 		return PTR_ERR(bip);
685 	}
686 
687 	bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
688 	/* virtual start sector must be in integrity interval units */
689 	bip_set_seed(bip, bio->bi_iter.bi_sector >>
690 				  (bi->interval_exp - SECTOR_SHIFT));
691 
692 	pr_debug("IBLOCK BIP Size: %u Sector: %llu\n", bip->bip_iter.bi_size,
693 		 (unsigned long long)bip->bip_iter.bi_sector);
694 
695 	resid = bip->bip_iter.bi_size;
696 	while (resid > 0 && sg_miter_next(miter)) {
697 
698 		len = min_t(size_t, miter->length, resid);
699 		rc = bio_integrity_add_page(bio, miter->page, len,
700 					    offset_in_page(miter->addr));
701 		if (rc != len) {
702 			pr_err("bio_integrity_add_page() failed; %d\n", rc);
703 			sg_miter_stop(miter);
704 			return -ENOMEM;
705 		}
706 
707 		pr_debug("Added bio integrity page: %p length: %zu offset: %lu\n",
708 			  miter->page, len, offset_in_page(miter->addr));
709 
710 		resid -= len;
711 		if (len < miter->length)
712 			miter->consumed -= miter->length - len;
713 	}
714 	sg_miter_stop(miter);
715 
716 	return 0;
717 }
718 
719 static sense_reason_t
720 iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
721 		  enum dma_data_direction data_direction)
722 {
723 	struct se_device *dev = cmd->se_dev;
724 	sector_t block_lba = target_to_linux_sector(dev, cmd->t_task_lba);
725 	struct iblock_req *ibr;
726 	struct bio *bio;
727 	struct bio_list list;
728 	struct scatterlist *sg;
729 	u32 sg_num = sgl_nents;
730 	blk_opf_t opf;
731 	unsigned bio_cnt;
732 	int i, rc;
733 	struct sg_mapping_iter prot_miter;
734 	unsigned int miter_dir;
735 
736 	if (data_direction == DMA_TO_DEVICE) {
737 		struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
738 		/*
739 		 * Force writethrough using REQ_FUA if a volatile write cache
740 		 * is not enabled, or if initiator set the Force Unit Access bit.
741 		 */
742 		opf = REQ_OP_WRITE;
743 		miter_dir = SG_MITER_TO_SG;
744 		if (bdev_fua(ib_dev->ibd_bd)) {
745 			if (cmd->se_cmd_flags & SCF_FUA)
746 				opf |= REQ_FUA;
747 			else if (!bdev_write_cache(ib_dev->ibd_bd))
748 				opf |= REQ_FUA;
749 		}
750 	} else {
751 		opf = REQ_OP_READ;
752 		miter_dir = SG_MITER_FROM_SG;
753 	}
754 
755 	ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
756 	if (!ibr)
757 		goto fail;
758 	cmd->priv = ibr;
759 
760 	if (!sgl_nents) {
761 		refcount_set(&ibr->pending, 1);
762 		iblock_complete_cmd(cmd);
763 		return 0;
764 	}
765 
766 	bio = iblock_get_bio(cmd, block_lba, sgl_nents, opf);
767 	if (!bio)
768 		goto fail_free_ibr;
769 
770 	bio_list_init(&list);
771 	bio_list_add(&list, bio);
772 
773 	refcount_set(&ibr->pending, 2);
774 	bio_cnt = 1;
775 
776 	if (cmd->prot_type && dev->dev_attrib.pi_prot_type)
777 		sg_miter_start(&prot_miter, cmd->t_prot_sg, cmd->t_prot_nents,
778 			       miter_dir);
779 
780 	for_each_sg(sgl, sg, sgl_nents, i) {
781 		/*
782 		 * XXX: if the length the device accepts is shorter than the
783 		 *	length of the S/G list entry this will cause and
784 		 *	endless loop.  Better hope no driver uses huge pages.
785 		 */
786 		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
787 				!= sg->length) {
788 			if (cmd->prot_type && dev->dev_attrib.pi_prot_type) {
789 				rc = iblock_alloc_bip(cmd, bio, &prot_miter);
790 				if (rc)
791 					goto fail_put_bios;
792 			}
793 
794 			if (bio_cnt >= IBLOCK_MAX_BIO_PER_TASK) {
795 				iblock_submit_bios(&list);
796 				bio_cnt = 0;
797 			}
798 
799 			bio = iblock_get_bio(cmd, block_lba, sg_num, opf);
800 			if (!bio)
801 				goto fail_put_bios;
802 
803 			refcount_inc(&ibr->pending);
804 			bio_list_add(&list, bio);
805 			bio_cnt++;
806 		}
807 
808 		/* Always in 512 byte units for Linux/Block */
809 		block_lba += sg->length >> SECTOR_SHIFT;
810 		sg_num--;
811 	}
812 
813 	if (cmd->prot_type && dev->dev_attrib.pi_prot_type) {
814 		rc = iblock_alloc_bip(cmd, bio, &prot_miter);
815 		if (rc)
816 			goto fail_put_bios;
817 	}
818 
819 	iblock_submit_bios(&list);
820 	iblock_complete_cmd(cmd);
821 	return 0;
822 
823 fail_put_bios:
824 	while ((bio = bio_list_pop(&list)))
825 		bio_put(bio);
826 fail_free_ibr:
827 	kfree(ibr);
828 fail:
829 	return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
830 }
831 
832 static sector_t iblock_get_alignment_offset_lbas(struct se_device *dev)
833 {
834 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
835 	struct block_device *bd = ib_dev->ibd_bd;
836 	int ret;
837 
838 	ret = bdev_alignment_offset(bd);
839 	if (ret == -1)
840 		return 0;
841 
842 	/* convert offset-bytes to offset-lbas */
843 	return ret / bdev_logical_block_size(bd);
844 }
845 
846 static unsigned int iblock_get_lbppbe(struct se_device *dev)
847 {
848 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
849 	struct block_device *bd = ib_dev->ibd_bd;
850 	unsigned int logs_per_phys =
851 		bdev_physical_block_size(bd) / bdev_logical_block_size(bd);
852 
853 	return ilog2(logs_per_phys);
854 }
855 
856 static unsigned int iblock_get_io_min(struct se_device *dev)
857 {
858 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
859 	struct block_device *bd = ib_dev->ibd_bd;
860 
861 	return bdev_io_min(bd);
862 }
863 
864 static unsigned int iblock_get_io_opt(struct se_device *dev)
865 {
866 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
867 	struct block_device *bd = ib_dev->ibd_bd;
868 
869 	return bdev_io_opt(bd);
870 }
871 
872 static struct sbc_ops iblock_sbc_ops = {
873 	.execute_rw		= iblock_execute_rw,
874 	.execute_sync_cache	= iblock_execute_sync_cache,
875 	.execute_write_same	= iblock_execute_write_same,
876 	.execute_unmap		= iblock_execute_unmap,
877 };
878 
879 static sense_reason_t
880 iblock_parse_cdb(struct se_cmd *cmd)
881 {
882 	return sbc_parse_cdb(cmd, &iblock_sbc_ops);
883 }
884 
885 static bool iblock_get_write_cache(struct se_device *dev)
886 {
887 	return bdev_write_cache(IBLOCK_DEV(dev)->ibd_bd);
888 }
889 
890 static const struct target_backend_ops iblock_ops = {
891 	.name			= "iblock",
892 	.inquiry_prod		= "IBLOCK",
893 	.inquiry_rev		= IBLOCK_VERSION,
894 	.owner			= THIS_MODULE,
895 	.attach_hba		= iblock_attach_hba,
896 	.detach_hba		= iblock_detach_hba,
897 	.alloc_device		= iblock_alloc_device,
898 	.configure_device	= iblock_configure_device,
899 	.destroy_device		= iblock_destroy_device,
900 	.free_device		= iblock_free_device,
901 	.configure_unmap	= iblock_configure_unmap,
902 	.plug_device		= iblock_plug_device,
903 	.unplug_device		= iblock_unplug_device,
904 	.parse_cdb		= iblock_parse_cdb,
905 	.set_configfs_dev_params = iblock_set_configfs_dev_params,
906 	.show_configfs_dev_params = iblock_show_configfs_dev_params,
907 	.get_device_type	= sbc_get_device_type,
908 	.get_blocks		= iblock_get_blocks,
909 	.get_alignment_offset_lbas = iblock_get_alignment_offset_lbas,
910 	.get_lbppbe		= iblock_get_lbppbe,
911 	.get_io_min		= iblock_get_io_min,
912 	.get_io_opt		= iblock_get_io_opt,
913 	.get_write_cache	= iblock_get_write_cache,
914 	.tb_dev_attrib_attrs	= sbc_attrib_attrs,
915 };
916 
917 static int __init iblock_module_init(void)
918 {
919 	return transport_backend_register(&iblock_ops);
920 }
921 
922 static void __exit iblock_module_exit(void)
923 {
924 	target_backend_unregister(&iblock_ops);
925 }
926 
927 MODULE_DESCRIPTION("TCM IBLOCK subsystem plugin");
928 MODULE_AUTHOR("nab@Linux-iSCSI.org");
929 MODULE_LICENSE("GPL");
930 
931 module_init(iblock_module_init);
932 module_exit(iblock_module_exit);
933