1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*******************************************************************************
3  * Filename:  target_core_iblock.c
4  *
5  * This file contains the Storage Engine  <-> Linux BlockIO transport
6  * specific functions.
7  *
8  * (c) Copyright 2003-2013 Datera, Inc.
9  *
10  * Nicholas A. Bellinger <nab@kernel.org>
11  *
12  ******************************************************************************/
13 
14 #include <linux/string.h>
15 #include <linux/parser.h>
16 #include <linux/timer.h>
17 #include <linux/fs.h>
18 #include <linux/blkdev.h>
19 #include <linux/slab.h>
20 #include <linux/spinlock.h>
21 #include <linux/bio.h>
22 #include <linux/genhd.h>
23 #include <linux/file.h>
24 #include <linux/module.h>
25 #include <scsi/scsi_proto.h>
26 #include <asm/unaligned.h>
27 
28 #include <target/target_core_base.h>
29 #include <target/target_core_backend.h>
30 
31 #include "target_core_iblock.h"
32 
33 #define IBLOCK_MAX_BIO_PER_TASK	 32	/* max # of bios to submit at a time */
34 #define IBLOCK_BIO_POOL_SIZE	128
35 
36 static inline struct iblock_dev *IBLOCK_DEV(struct se_device *dev)
37 {
38 	return container_of(dev, struct iblock_dev, dev);
39 }
40 
41 
42 static int iblock_attach_hba(struct se_hba *hba, u32 host_id)
43 {
44 	pr_debug("CORE_HBA[%d] - TCM iBlock HBA Driver %s on"
45 		" Generic Target Core Stack %s\n", hba->hba_id,
46 		IBLOCK_VERSION, TARGET_CORE_VERSION);
47 	return 0;
48 }
49 
50 static void iblock_detach_hba(struct se_hba *hba)
51 {
52 }
53 
54 static struct se_device *iblock_alloc_device(struct se_hba *hba, const char *name)
55 {
56 	struct iblock_dev *ib_dev = NULL;
57 
58 	ib_dev = kzalloc(sizeof(struct iblock_dev), GFP_KERNEL);
59 	if (!ib_dev) {
60 		pr_err("Unable to allocate struct iblock_dev\n");
61 		return NULL;
62 	}
63 
64 	pr_debug( "IBLOCK: Allocated ib_dev for %s\n", name);
65 
66 	return &ib_dev->dev;
67 }
68 
69 static int iblock_configure_device(struct se_device *dev)
70 {
71 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
72 	struct request_queue *q;
73 	struct block_device *bd = NULL;
74 	struct blk_integrity *bi;
75 	fmode_t mode;
76 	unsigned int max_write_zeroes_sectors;
77 	int ret = -ENOMEM;
78 
79 	if (!(ib_dev->ibd_flags & IBDF_HAS_UDEV_PATH)) {
80 		pr_err("Missing udev_path= parameters for IBLOCK\n");
81 		return -EINVAL;
82 	}
83 
84 	ret = bioset_init(&ib_dev->ibd_bio_set, IBLOCK_BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
85 	if (ret) {
86 		pr_err("IBLOCK: Unable to create bioset\n");
87 		goto out;
88 	}
89 
90 	pr_debug( "IBLOCK: Claiming struct block_device: %s\n",
91 			ib_dev->ibd_udev_path);
92 
93 	mode = FMODE_READ|FMODE_EXCL;
94 	if (!ib_dev->ibd_readonly)
95 		mode |= FMODE_WRITE;
96 	else
97 		dev->dev_flags |= DF_READ_ONLY;
98 
99 	bd = blkdev_get_by_path(ib_dev->ibd_udev_path, mode, ib_dev);
100 	if (IS_ERR(bd)) {
101 		ret = PTR_ERR(bd);
102 		goto out_free_bioset;
103 	}
104 	ib_dev->ibd_bd = bd;
105 
106 	q = bdev_get_queue(bd);
107 
108 	dev->dev_attrib.hw_block_size = bdev_logical_block_size(bd);
109 	dev->dev_attrib.hw_max_sectors = queue_max_hw_sectors(q);
110 	dev->dev_attrib.hw_queue_depth = q->nr_requests;
111 
112 	if (target_configure_unmap_from_queue(&dev->dev_attrib, q))
113 		pr_debug("IBLOCK: BLOCK Discard support available,"
114 			 " disabled by default\n");
115 
116 	/*
117 	 * Enable write same emulation for IBLOCK and use 0xFFFF as
118 	 * the smaller WRITE_SAME(10) only has a two-byte block count.
119 	 */
120 	max_write_zeroes_sectors = bdev_write_zeroes_sectors(bd);
121 	if (max_write_zeroes_sectors)
122 		dev->dev_attrib.max_write_same_len = max_write_zeroes_sectors;
123 	else
124 		dev->dev_attrib.max_write_same_len = 0xFFFF;
125 
126 	if (blk_queue_nonrot(q))
127 		dev->dev_attrib.is_nonrot = 1;
128 
129 	bi = bdev_get_integrity(bd);
130 	if (bi) {
131 		struct bio_set *bs = &ib_dev->ibd_bio_set;
132 
133 		if (!strcmp(bi->profile->name, "T10-DIF-TYPE3-IP") ||
134 		    !strcmp(bi->profile->name, "T10-DIF-TYPE1-IP")) {
135 			pr_err("IBLOCK export of blk_integrity: %s not"
136 			       " supported\n", bi->profile->name);
137 			ret = -ENOSYS;
138 			goto out_blkdev_put;
139 		}
140 
141 		if (!strcmp(bi->profile->name, "T10-DIF-TYPE3-CRC")) {
142 			dev->dev_attrib.pi_prot_type = TARGET_DIF_TYPE3_PROT;
143 		} else if (!strcmp(bi->profile->name, "T10-DIF-TYPE1-CRC")) {
144 			dev->dev_attrib.pi_prot_type = TARGET_DIF_TYPE1_PROT;
145 		}
146 
147 		if (dev->dev_attrib.pi_prot_type) {
148 			if (bioset_integrity_create(bs, IBLOCK_BIO_POOL_SIZE) < 0) {
149 				pr_err("Unable to allocate bioset for PI\n");
150 				ret = -ENOMEM;
151 				goto out_blkdev_put;
152 			}
153 			pr_debug("IBLOCK setup BIP bs->bio_integrity_pool: %p\n",
154 				 &bs->bio_integrity_pool);
155 		}
156 		dev->dev_attrib.hw_pi_prot_type = dev->dev_attrib.pi_prot_type;
157 	}
158 
159 	return 0;
160 
161 out_blkdev_put:
162 	blkdev_put(ib_dev->ibd_bd, FMODE_WRITE|FMODE_READ|FMODE_EXCL);
163 out_free_bioset:
164 	bioset_exit(&ib_dev->ibd_bio_set);
165 out:
166 	return ret;
167 }
168 
169 static void iblock_dev_call_rcu(struct rcu_head *p)
170 {
171 	struct se_device *dev = container_of(p, struct se_device, rcu_head);
172 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
173 
174 	kfree(ib_dev);
175 }
176 
177 static void iblock_free_device(struct se_device *dev)
178 {
179 	call_rcu(&dev->rcu_head, iblock_dev_call_rcu);
180 }
181 
182 static void iblock_destroy_device(struct se_device *dev)
183 {
184 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
185 
186 	if (ib_dev->ibd_bd != NULL)
187 		blkdev_put(ib_dev->ibd_bd, FMODE_WRITE|FMODE_READ|FMODE_EXCL);
188 	bioset_exit(&ib_dev->ibd_bio_set);
189 }
190 
191 static unsigned long long iblock_emulate_read_cap_with_block_size(
192 	struct se_device *dev,
193 	struct block_device *bd,
194 	struct request_queue *q)
195 {
196 	unsigned long long blocks_long = (div_u64(i_size_read(bd->bd_inode),
197 					bdev_logical_block_size(bd)) - 1);
198 	u32 block_size = bdev_logical_block_size(bd);
199 
200 	if (block_size == dev->dev_attrib.block_size)
201 		return blocks_long;
202 
203 	switch (block_size) {
204 	case 4096:
205 		switch (dev->dev_attrib.block_size) {
206 		case 2048:
207 			blocks_long <<= 1;
208 			break;
209 		case 1024:
210 			blocks_long <<= 2;
211 			break;
212 		case 512:
213 			blocks_long <<= 3;
214 			break;
215 		default:
216 			break;
217 		}
218 		break;
219 	case 2048:
220 		switch (dev->dev_attrib.block_size) {
221 		case 4096:
222 			blocks_long >>= 1;
223 			break;
224 		case 1024:
225 			blocks_long <<= 1;
226 			break;
227 		case 512:
228 			blocks_long <<= 2;
229 			break;
230 		default:
231 			break;
232 		}
233 		break;
234 	case 1024:
235 		switch (dev->dev_attrib.block_size) {
236 		case 4096:
237 			blocks_long >>= 2;
238 			break;
239 		case 2048:
240 			blocks_long >>= 1;
241 			break;
242 		case 512:
243 			blocks_long <<= 1;
244 			break;
245 		default:
246 			break;
247 		}
248 		break;
249 	case 512:
250 		switch (dev->dev_attrib.block_size) {
251 		case 4096:
252 			blocks_long >>= 3;
253 			break;
254 		case 2048:
255 			blocks_long >>= 2;
256 			break;
257 		case 1024:
258 			blocks_long >>= 1;
259 			break;
260 		default:
261 			break;
262 		}
263 		break;
264 	default:
265 		break;
266 	}
267 
268 	return blocks_long;
269 }
270 
271 static void iblock_complete_cmd(struct se_cmd *cmd)
272 {
273 	struct iblock_req *ibr = cmd->priv;
274 	u8 status;
275 
276 	if (!refcount_dec_and_test(&ibr->pending))
277 		return;
278 
279 	if (atomic_read(&ibr->ib_bio_err_cnt))
280 		status = SAM_STAT_CHECK_CONDITION;
281 	else
282 		status = SAM_STAT_GOOD;
283 
284 	target_complete_cmd(cmd, status);
285 	kfree(ibr);
286 }
287 
288 static void iblock_bio_done(struct bio *bio)
289 {
290 	struct se_cmd *cmd = bio->bi_private;
291 	struct iblock_req *ibr = cmd->priv;
292 
293 	if (bio->bi_status) {
294 		pr_err("bio error: %p,  err: %d\n", bio, bio->bi_status);
295 		/*
296 		 * Bump the ib_bio_err_cnt and release bio.
297 		 */
298 		atomic_inc(&ibr->ib_bio_err_cnt);
299 		smp_mb__after_atomic();
300 	}
301 
302 	bio_put(bio);
303 
304 	iblock_complete_cmd(cmd);
305 }
306 
307 static struct bio *
308 iblock_get_bio(struct se_cmd *cmd, sector_t lba, u32 sg_num, int op,
309 	       int op_flags)
310 {
311 	struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
312 	struct bio *bio;
313 
314 	/*
315 	 * Only allocate as many vector entries as the bio code allows us to,
316 	 * we'll loop later on until we have handled the whole request.
317 	 */
318 	if (sg_num > BIO_MAX_PAGES)
319 		sg_num = BIO_MAX_PAGES;
320 
321 	bio = bio_alloc_bioset(GFP_NOIO, sg_num, &ib_dev->ibd_bio_set);
322 	if (!bio) {
323 		pr_err("Unable to allocate memory for bio\n");
324 		return NULL;
325 	}
326 
327 	bio_set_dev(bio, ib_dev->ibd_bd);
328 	bio->bi_private = cmd;
329 	bio->bi_end_io = &iblock_bio_done;
330 	bio->bi_iter.bi_sector = lba;
331 	bio_set_op_attrs(bio, op, op_flags);
332 
333 	return bio;
334 }
335 
336 static void iblock_submit_bios(struct bio_list *list)
337 {
338 	struct blk_plug plug;
339 	struct bio *bio;
340 
341 	blk_start_plug(&plug);
342 	while ((bio = bio_list_pop(list)))
343 		submit_bio(bio);
344 	blk_finish_plug(&plug);
345 }
346 
347 static void iblock_end_io_flush(struct bio *bio)
348 {
349 	struct se_cmd *cmd = bio->bi_private;
350 
351 	if (bio->bi_status)
352 		pr_err("IBLOCK: cache flush failed: %d\n", bio->bi_status);
353 
354 	if (cmd) {
355 		if (bio->bi_status)
356 			target_complete_cmd(cmd, SAM_STAT_CHECK_CONDITION);
357 		else
358 			target_complete_cmd(cmd, SAM_STAT_GOOD);
359 	}
360 
361 	bio_put(bio);
362 }
363 
364 /*
365  * Implement SYCHRONIZE CACHE.  Note that we can't handle lba ranges and must
366  * always flush the whole cache.
367  */
368 static sense_reason_t
369 iblock_execute_sync_cache(struct se_cmd *cmd)
370 {
371 	struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev);
372 	int immed = (cmd->t_task_cdb[1] & 0x2);
373 	struct bio *bio;
374 
375 	/*
376 	 * If the Immediate bit is set, queue up the GOOD response
377 	 * for this SYNCHRONIZE_CACHE op.
378 	 */
379 	if (immed)
380 		target_complete_cmd(cmd, SAM_STAT_GOOD);
381 
382 	bio = bio_alloc(GFP_KERNEL, 0);
383 	bio->bi_end_io = iblock_end_io_flush;
384 	bio_set_dev(bio, ib_dev->ibd_bd);
385 	bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
386 	if (!immed)
387 		bio->bi_private = cmd;
388 	submit_bio(bio);
389 	return 0;
390 }
391 
392 static sense_reason_t
393 iblock_execute_unmap(struct se_cmd *cmd, sector_t lba, sector_t nolb)
394 {
395 	struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
396 	struct se_device *dev = cmd->se_dev;
397 	int ret;
398 
399 	ret = blkdev_issue_discard(bdev,
400 				   target_to_linux_sector(dev, lba),
401 				   target_to_linux_sector(dev,  nolb),
402 				   GFP_KERNEL, 0);
403 	if (ret < 0) {
404 		pr_err("blkdev_issue_discard() failed: %d\n", ret);
405 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
406 	}
407 
408 	return 0;
409 }
410 
411 static sense_reason_t
412 iblock_execute_zero_out(struct block_device *bdev, struct se_cmd *cmd)
413 {
414 	struct se_device *dev = cmd->se_dev;
415 	struct scatterlist *sg = &cmd->t_data_sg[0];
416 	unsigned char *buf, *not_zero;
417 	int ret;
418 
419 	buf = kmap(sg_page(sg)) + sg->offset;
420 	if (!buf)
421 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
422 	/*
423 	 * Fall back to block_execute_write_same() slow-path if
424 	 * incoming WRITE_SAME payload does not contain zeros.
425 	 */
426 	not_zero = memchr_inv(buf, 0x00, cmd->data_length);
427 	kunmap(sg_page(sg));
428 
429 	if (not_zero)
430 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
431 
432 	ret = blkdev_issue_zeroout(bdev,
433 				target_to_linux_sector(dev, cmd->t_task_lba),
434 				target_to_linux_sector(dev,
435 					sbc_get_write_same_sectors(cmd)),
436 				GFP_KERNEL, BLKDEV_ZERO_NOUNMAP);
437 	if (ret)
438 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
439 
440 	target_complete_cmd(cmd, GOOD);
441 	return 0;
442 }
443 
444 static sense_reason_t
445 iblock_execute_write_same(struct se_cmd *cmd)
446 {
447 	struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd;
448 	struct iblock_req *ibr;
449 	struct scatterlist *sg;
450 	struct bio *bio;
451 	struct bio_list list;
452 	struct se_device *dev = cmd->se_dev;
453 	sector_t block_lba = target_to_linux_sector(dev, cmd->t_task_lba);
454 	sector_t sectors = target_to_linux_sector(dev,
455 					sbc_get_write_same_sectors(cmd));
456 
457 	if (cmd->prot_op) {
458 		pr_err("WRITE_SAME: Protection information with IBLOCK"
459 		       " backends not supported\n");
460 		return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
461 	}
462 	sg = &cmd->t_data_sg[0];
463 
464 	if (cmd->t_data_nents > 1 ||
465 	    sg->length != cmd->se_dev->dev_attrib.block_size) {
466 		pr_err("WRITE_SAME: Illegal SGL t_data_nents: %u length: %u"
467 			" block_size: %u\n", cmd->t_data_nents, sg->length,
468 			cmd->se_dev->dev_attrib.block_size);
469 		return TCM_INVALID_CDB_FIELD;
470 	}
471 
472 	if (bdev_write_zeroes_sectors(bdev)) {
473 		if (!iblock_execute_zero_out(bdev, cmd))
474 			return 0;
475 	}
476 
477 	ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
478 	if (!ibr)
479 		goto fail;
480 	cmd->priv = ibr;
481 
482 	bio = iblock_get_bio(cmd, block_lba, 1, REQ_OP_WRITE, 0);
483 	if (!bio)
484 		goto fail_free_ibr;
485 
486 	bio_list_init(&list);
487 	bio_list_add(&list, bio);
488 
489 	refcount_set(&ibr->pending, 1);
490 
491 	while (sectors) {
492 		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
493 				!= sg->length) {
494 
495 			bio = iblock_get_bio(cmd, block_lba, 1, REQ_OP_WRITE,
496 					     0);
497 			if (!bio)
498 				goto fail_put_bios;
499 
500 			refcount_inc(&ibr->pending);
501 			bio_list_add(&list, bio);
502 		}
503 
504 		/* Always in 512 byte units for Linux/Block */
505 		block_lba += sg->length >> SECTOR_SHIFT;
506 		sectors -= sg->length >> SECTOR_SHIFT;
507 	}
508 
509 	iblock_submit_bios(&list);
510 	return 0;
511 
512 fail_put_bios:
513 	while ((bio = bio_list_pop(&list)))
514 		bio_put(bio);
515 fail_free_ibr:
516 	kfree(ibr);
517 fail:
518 	return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
519 }
520 
521 enum {
522 	Opt_udev_path, Opt_readonly, Opt_force, Opt_err
523 };
524 
525 static match_table_t tokens = {
526 	{Opt_udev_path, "udev_path=%s"},
527 	{Opt_readonly, "readonly=%d"},
528 	{Opt_force, "force=%d"},
529 	{Opt_err, NULL}
530 };
531 
532 static ssize_t iblock_set_configfs_dev_params(struct se_device *dev,
533 		const char *page, ssize_t count)
534 {
535 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
536 	char *orig, *ptr, *arg_p, *opts;
537 	substring_t args[MAX_OPT_ARGS];
538 	int ret = 0, token;
539 	unsigned long tmp_readonly;
540 
541 	opts = kstrdup(page, GFP_KERNEL);
542 	if (!opts)
543 		return -ENOMEM;
544 
545 	orig = opts;
546 
547 	while ((ptr = strsep(&opts, ",\n")) != NULL) {
548 		if (!*ptr)
549 			continue;
550 
551 		token = match_token(ptr, tokens, args);
552 		switch (token) {
553 		case Opt_udev_path:
554 			if (ib_dev->ibd_bd) {
555 				pr_err("Unable to set udev_path= while"
556 					" ib_dev->ibd_bd exists\n");
557 				ret = -EEXIST;
558 				goto out;
559 			}
560 			if (match_strlcpy(ib_dev->ibd_udev_path, &args[0],
561 				SE_UDEV_PATH_LEN) == 0) {
562 				ret = -EINVAL;
563 				break;
564 			}
565 			pr_debug("IBLOCK: Referencing UDEV path: %s\n",
566 					ib_dev->ibd_udev_path);
567 			ib_dev->ibd_flags |= IBDF_HAS_UDEV_PATH;
568 			break;
569 		case Opt_readonly:
570 			arg_p = match_strdup(&args[0]);
571 			if (!arg_p) {
572 				ret = -ENOMEM;
573 				break;
574 			}
575 			ret = kstrtoul(arg_p, 0, &tmp_readonly);
576 			kfree(arg_p);
577 			if (ret < 0) {
578 				pr_err("kstrtoul() failed for"
579 						" readonly=\n");
580 				goto out;
581 			}
582 			ib_dev->ibd_readonly = tmp_readonly;
583 			pr_debug("IBLOCK: readonly: %d\n", ib_dev->ibd_readonly);
584 			break;
585 		case Opt_force:
586 			break;
587 		default:
588 			break;
589 		}
590 	}
591 
592 out:
593 	kfree(orig);
594 	return (!ret) ? count : ret;
595 }
596 
597 static ssize_t iblock_show_configfs_dev_params(struct se_device *dev, char *b)
598 {
599 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
600 	struct block_device *bd = ib_dev->ibd_bd;
601 	char buf[BDEVNAME_SIZE];
602 	ssize_t bl = 0;
603 
604 	if (bd)
605 		bl += sprintf(b + bl, "iBlock device: %s",
606 				bdevname(bd, buf));
607 	if (ib_dev->ibd_flags & IBDF_HAS_UDEV_PATH)
608 		bl += sprintf(b + bl, "  UDEV PATH: %s",
609 				ib_dev->ibd_udev_path);
610 	bl += sprintf(b + bl, "  readonly: %d\n", ib_dev->ibd_readonly);
611 
612 	bl += sprintf(b + bl, "        ");
613 	if (bd) {
614 		bl += sprintf(b + bl, "Major: %d Minor: %d  %s\n",
615 			MAJOR(bd->bd_dev), MINOR(bd->bd_dev),
616 			"CLAIMED: IBLOCK");
617 	} else {
618 		bl += sprintf(b + bl, "Major: 0 Minor: 0\n");
619 	}
620 
621 	return bl;
622 }
623 
624 static int
625 iblock_alloc_bip(struct se_cmd *cmd, struct bio *bio,
626 		 struct sg_mapping_iter *miter)
627 {
628 	struct se_device *dev = cmd->se_dev;
629 	struct blk_integrity *bi;
630 	struct bio_integrity_payload *bip;
631 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
632 	int rc;
633 	size_t resid, len;
634 
635 	bi = bdev_get_integrity(ib_dev->ibd_bd);
636 	if (!bi) {
637 		pr_err("Unable to locate bio_integrity\n");
638 		return -ENODEV;
639 	}
640 
641 	bip = bio_integrity_alloc(bio, GFP_NOIO,
642 			min_t(unsigned int, cmd->t_prot_nents, BIO_MAX_PAGES));
643 	if (IS_ERR(bip)) {
644 		pr_err("Unable to allocate bio_integrity_payload\n");
645 		return PTR_ERR(bip);
646 	}
647 
648 	bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
649 	/* virtual start sector must be in integrity interval units */
650 	bip_set_seed(bip, bio->bi_iter.bi_sector >>
651 				  (bi->interval_exp - SECTOR_SHIFT));
652 
653 	pr_debug("IBLOCK BIP Size: %u Sector: %llu\n", bip->bip_iter.bi_size,
654 		 (unsigned long long)bip->bip_iter.bi_sector);
655 
656 	resid = bip->bip_iter.bi_size;
657 	while (resid > 0 && sg_miter_next(miter)) {
658 
659 		len = min_t(size_t, miter->length, resid);
660 		rc = bio_integrity_add_page(bio, miter->page, len,
661 					    offset_in_page(miter->addr));
662 		if (rc != len) {
663 			pr_err("bio_integrity_add_page() failed; %d\n", rc);
664 			sg_miter_stop(miter);
665 			return -ENOMEM;
666 		}
667 
668 		pr_debug("Added bio integrity page: %p length: %zu offset: %lu\n",
669 			  miter->page, len, offset_in_page(miter->addr));
670 
671 		resid -= len;
672 		if (len < miter->length)
673 			miter->consumed -= miter->length - len;
674 	}
675 	sg_miter_stop(miter);
676 
677 	return 0;
678 }
679 
680 static sense_reason_t
681 iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
682 		  enum dma_data_direction data_direction)
683 {
684 	struct se_device *dev = cmd->se_dev;
685 	sector_t block_lba = target_to_linux_sector(dev, cmd->t_task_lba);
686 	struct iblock_req *ibr;
687 	struct bio *bio;
688 	struct bio_list list;
689 	struct scatterlist *sg;
690 	u32 sg_num = sgl_nents;
691 	unsigned bio_cnt;
692 	int i, rc, op, op_flags = 0;
693 	struct sg_mapping_iter prot_miter;
694 
695 	if (data_direction == DMA_TO_DEVICE) {
696 		struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
697 		struct request_queue *q = bdev_get_queue(ib_dev->ibd_bd);
698 		/*
699 		 * Force writethrough using REQ_FUA if a volatile write cache
700 		 * is not enabled, or if initiator set the Force Unit Access bit.
701 		 */
702 		op = REQ_OP_WRITE;
703 		if (test_bit(QUEUE_FLAG_FUA, &q->queue_flags)) {
704 			if (cmd->se_cmd_flags & SCF_FUA)
705 				op_flags = REQ_FUA;
706 			else if (!test_bit(QUEUE_FLAG_WC, &q->queue_flags))
707 				op_flags = REQ_FUA;
708 		}
709 	} else {
710 		op = REQ_OP_READ;
711 	}
712 
713 	ibr = kzalloc(sizeof(struct iblock_req), GFP_KERNEL);
714 	if (!ibr)
715 		goto fail;
716 	cmd->priv = ibr;
717 
718 	if (!sgl_nents) {
719 		refcount_set(&ibr->pending, 1);
720 		iblock_complete_cmd(cmd);
721 		return 0;
722 	}
723 
724 	bio = iblock_get_bio(cmd, block_lba, sgl_nents, op, op_flags);
725 	if (!bio)
726 		goto fail_free_ibr;
727 
728 	bio_list_init(&list);
729 	bio_list_add(&list, bio);
730 
731 	refcount_set(&ibr->pending, 2);
732 	bio_cnt = 1;
733 
734 	if (cmd->prot_type && dev->dev_attrib.pi_prot_type)
735 		sg_miter_start(&prot_miter, cmd->t_prot_sg, cmd->t_prot_nents,
736 			       op == REQ_OP_READ ? SG_MITER_FROM_SG :
737 						   SG_MITER_TO_SG);
738 
739 	for_each_sg(sgl, sg, sgl_nents, i) {
740 		/*
741 		 * XXX: if the length the device accepts is shorter than the
742 		 *	length of the S/G list entry this will cause and
743 		 *	endless loop.  Better hope no driver uses huge pages.
744 		 */
745 		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)
746 				!= sg->length) {
747 			if (cmd->prot_type && dev->dev_attrib.pi_prot_type) {
748 				rc = iblock_alloc_bip(cmd, bio, &prot_miter);
749 				if (rc)
750 					goto fail_put_bios;
751 			}
752 
753 			if (bio_cnt >= IBLOCK_MAX_BIO_PER_TASK) {
754 				iblock_submit_bios(&list);
755 				bio_cnt = 0;
756 			}
757 
758 			bio = iblock_get_bio(cmd, block_lba, sg_num, op,
759 					     op_flags);
760 			if (!bio)
761 				goto fail_put_bios;
762 
763 			refcount_inc(&ibr->pending);
764 			bio_list_add(&list, bio);
765 			bio_cnt++;
766 		}
767 
768 		/* Always in 512 byte units for Linux/Block */
769 		block_lba += sg->length >> SECTOR_SHIFT;
770 		sg_num--;
771 	}
772 
773 	if (cmd->prot_type && dev->dev_attrib.pi_prot_type) {
774 		rc = iblock_alloc_bip(cmd, bio, &prot_miter);
775 		if (rc)
776 			goto fail_put_bios;
777 	}
778 
779 	iblock_submit_bios(&list);
780 	iblock_complete_cmd(cmd);
781 	return 0;
782 
783 fail_put_bios:
784 	while ((bio = bio_list_pop(&list)))
785 		bio_put(bio);
786 fail_free_ibr:
787 	kfree(ibr);
788 fail:
789 	return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
790 }
791 
792 static sector_t iblock_get_blocks(struct se_device *dev)
793 {
794 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
795 	struct block_device *bd = ib_dev->ibd_bd;
796 	struct request_queue *q = bdev_get_queue(bd);
797 
798 	return iblock_emulate_read_cap_with_block_size(dev, bd, q);
799 }
800 
801 static sector_t iblock_get_alignment_offset_lbas(struct se_device *dev)
802 {
803 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
804 	struct block_device *bd = ib_dev->ibd_bd;
805 	int ret;
806 
807 	ret = bdev_alignment_offset(bd);
808 	if (ret == -1)
809 		return 0;
810 
811 	/* convert offset-bytes to offset-lbas */
812 	return ret / bdev_logical_block_size(bd);
813 }
814 
815 static unsigned int iblock_get_lbppbe(struct se_device *dev)
816 {
817 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
818 	struct block_device *bd = ib_dev->ibd_bd;
819 	int logs_per_phys = bdev_physical_block_size(bd) / bdev_logical_block_size(bd);
820 
821 	return ilog2(logs_per_phys);
822 }
823 
824 static unsigned int iblock_get_io_min(struct se_device *dev)
825 {
826 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
827 	struct block_device *bd = ib_dev->ibd_bd;
828 
829 	return bdev_io_min(bd);
830 }
831 
832 static unsigned int iblock_get_io_opt(struct se_device *dev)
833 {
834 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
835 	struct block_device *bd = ib_dev->ibd_bd;
836 
837 	return bdev_io_opt(bd);
838 }
839 
840 static struct sbc_ops iblock_sbc_ops = {
841 	.execute_rw		= iblock_execute_rw,
842 	.execute_sync_cache	= iblock_execute_sync_cache,
843 	.execute_write_same	= iblock_execute_write_same,
844 	.execute_unmap		= iblock_execute_unmap,
845 };
846 
847 static sense_reason_t
848 iblock_parse_cdb(struct se_cmd *cmd)
849 {
850 	return sbc_parse_cdb(cmd, &iblock_sbc_ops);
851 }
852 
853 static bool iblock_get_write_cache(struct se_device *dev)
854 {
855 	struct iblock_dev *ib_dev = IBLOCK_DEV(dev);
856 	struct block_device *bd = ib_dev->ibd_bd;
857 	struct request_queue *q = bdev_get_queue(bd);
858 
859 	return test_bit(QUEUE_FLAG_WC, &q->queue_flags);
860 }
861 
862 static const struct target_backend_ops iblock_ops = {
863 	.name			= "iblock",
864 	.inquiry_prod		= "IBLOCK",
865 	.inquiry_rev		= IBLOCK_VERSION,
866 	.owner			= THIS_MODULE,
867 	.attach_hba		= iblock_attach_hba,
868 	.detach_hba		= iblock_detach_hba,
869 	.alloc_device		= iblock_alloc_device,
870 	.configure_device	= iblock_configure_device,
871 	.destroy_device		= iblock_destroy_device,
872 	.free_device		= iblock_free_device,
873 	.parse_cdb		= iblock_parse_cdb,
874 	.set_configfs_dev_params = iblock_set_configfs_dev_params,
875 	.show_configfs_dev_params = iblock_show_configfs_dev_params,
876 	.get_device_type	= sbc_get_device_type,
877 	.get_blocks		= iblock_get_blocks,
878 	.get_alignment_offset_lbas = iblock_get_alignment_offset_lbas,
879 	.get_lbppbe		= iblock_get_lbppbe,
880 	.get_io_min		= iblock_get_io_min,
881 	.get_io_opt		= iblock_get_io_opt,
882 	.get_write_cache	= iblock_get_write_cache,
883 	.tb_dev_attrib_attrs	= sbc_attrib_attrs,
884 };
885 
886 static int __init iblock_module_init(void)
887 {
888 	return transport_backend_register(&iblock_ops);
889 }
890 
891 static void __exit iblock_module_exit(void)
892 {
893 	target_backend_unregister(&iblock_ops);
894 }
895 
896 MODULE_DESCRIPTION("TCM IBLOCK subsystem plugin");
897 MODULE_AUTHOR("nab@Linux-iSCSI.org");
898 MODULE_LICENSE("GPL");
899 
900 module_init(iblock_module_init);
901 module_exit(iblock_module_exit);
902