xref: /openbmc/linux/drivers/md/dm-integrity.c (revision da2ef666)
1 /*
2  * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved.
3  * Copyright (C) 2016-2017 Milan Broz
4  * Copyright (C) 2016-2017 Mikulas Patocka
5  *
6  * This file is released under the GPL.
7  */
8 
9 #include <linux/compiler.h>
10 #include <linux/module.h>
11 #include <linux/device-mapper.h>
12 #include <linux/dm-io.h>
13 #include <linux/vmalloc.h>
14 #include <linux/sort.h>
15 #include <linux/rbtree.h>
16 #include <linux/delay.h>
17 #include <linux/random.h>
18 #include <crypto/hash.h>
19 #include <crypto/skcipher.h>
20 #include <linux/async_tx.h>
21 #include <linux/dm-bufio.h>
22 
23 #define DM_MSG_PREFIX "integrity"
24 
25 #define DEFAULT_INTERLEAVE_SECTORS	32768
26 #define DEFAULT_JOURNAL_SIZE_FACTOR	7
27 #define DEFAULT_BUFFER_SECTORS		128
28 #define DEFAULT_JOURNAL_WATERMARK	50
29 #define DEFAULT_SYNC_MSEC		10000
30 #define DEFAULT_MAX_JOURNAL_SECTORS	131072
31 #define MIN_LOG2_INTERLEAVE_SECTORS	3
32 #define MAX_LOG2_INTERLEAVE_SECTORS	31
33 #define METADATA_WORKQUEUE_MAX_ACTIVE	16
34 #define RECALC_SECTORS			8192
35 #define RECALC_WRITE_SUPER		16
36 
37 /*
38  * Warning - DEBUG_PRINT prints security-sensitive data to the log,
39  * so it should not be enabled in the official kernel
40  */
41 //#define DEBUG_PRINT
42 //#define INTERNAL_VERIFY
43 
44 /*
45  * On disk structures
46  */
47 
48 #define SB_MAGIC			"integrt"
49 #define SB_VERSION_1			1
50 #define SB_VERSION_2			2
51 #define SB_SECTORS			8
52 #define MAX_SECTORS_PER_BLOCK		8
53 
54 struct superblock {
55 	__u8 magic[8];
56 	__u8 version;
57 	__u8 log2_interleave_sectors;
58 	__u16 integrity_tag_size;
59 	__u32 journal_sections;
60 	__u64 provided_data_sectors;	/* userspace uses this value */
61 	__u32 flags;
62 	__u8 log2_sectors_per_block;
63 	__u8 pad[3];
64 	__u64 recalc_sector;
65 };
66 
67 #define SB_FLAG_HAVE_JOURNAL_MAC	0x1
68 #define SB_FLAG_RECALCULATING		0x2
69 
70 #define	JOURNAL_ENTRY_ROUNDUP		8
71 
72 typedef __u64 commit_id_t;
73 #define JOURNAL_MAC_PER_SECTOR		8
74 
75 struct journal_entry {
76 	union {
77 		struct {
78 			__u32 sector_lo;
79 			__u32 sector_hi;
80 		} s;
81 		__u64 sector;
82 	} u;
83 	commit_id_t last_bytes[0];
84 	/* __u8 tag[0]; */
85 };
86 
87 #define journal_entry_tag(ic, je)		((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block])
88 
89 #if BITS_PER_LONG == 64
90 #define journal_entry_set_sector(je, x)		do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0)
91 #define journal_entry_get_sector(je)		le64_to_cpu((je)->u.sector)
92 #elif defined(CONFIG_LBDAF)
93 #define journal_entry_set_sector(je, x)		do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0)
94 #define journal_entry_get_sector(je)		le64_to_cpu((je)->u.sector)
95 #else
96 #define journal_entry_set_sector(je, x)		do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32(0)); } while (0)
97 #define journal_entry_get_sector(je)		le32_to_cpu((je)->u.s.sector_lo)
98 #endif
99 #define journal_entry_is_unused(je)		((je)->u.s.sector_hi == cpu_to_le32(-1))
100 #define journal_entry_set_unused(je)		do { ((je)->u.s.sector_hi = cpu_to_le32(-1)); } while (0)
101 #define journal_entry_is_inprogress(je)		((je)->u.s.sector_hi == cpu_to_le32(-2))
102 #define journal_entry_set_inprogress(je)	do { ((je)->u.s.sector_hi = cpu_to_le32(-2)); } while (0)
103 
104 #define JOURNAL_BLOCK_SECTORS		8
105 #define JOURNAL_SECTOR_DATA		((1 << SECTOR_SHIFT) - sizeof(commit_id_t))
106 #define JOURNAL_MAC_SIZE		(JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS)
107 
108 struct journal_sector {
109 	__u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR];
110 	__u8 mac[JOURNAL_MAC_PER_SECTOR];
111 	commit_id_t commit_id;
112 };
113 
114 #define MAX_TAG_SIZE			(JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK]))
115 
116 #define METADATA_PADDING_SECTORS	8
117 
118 #define N_COMMIT_IDS			4
119 
120 static unsigned char prev_commit_seq(unsigned char seq)
121 {
122 	return (seq + N_COMMIT_IDS - 1) % N_COMMIT_IDS;
123 }
124 
125 static unsigned char next_commit_seq(unsigned char seq)
126 {
127 	return (seq + 1) % N_COMMIT_IDS;
128 }
129 
130 /*
131  * In-memory structures
132  */
133 
134 struct journal_node {
135 	struct rb_node node;
136 	sector_t sector;
137 };
138 
139 struct alg_spec {
140 	char *alg_string;
141 	char *key_string;
142 	__u8 *key;
143 	unsigned key_size;
144 };
145 
146 struct dm_integrity_c {
147 	struct dm_dev *dev;
148 	struct dm_dev *meta_dev;
149 	unsigned tag_size;
150 	__s8 log2_tag_size;
151 	sector_t start;
152 	mempool_t journal_io_mempool;
153 	struct dm_io_client *io;
154 	struct dm_bufio_client *bufio;
155 	struct workqueue_struct *metadata_wq;
156 	struct superblock *sb;
157 	unsigned journal_pages;
158 	struct page_list *journal;
159 	struct page_list *journal_io;
160 	struct page_list *journal_xor;
161 
162 	struct crypto_skcipher *journal_crypt;
163 	struct scatterlist **journal_scatterlist;
164 	struct scatterlist **journal_io_scatterlist;
165 	struct skcipher_request **sk_requests;
166 
167 	struct crypto_shash *journal_mac;
168 
169 	struct journal_node *journal_tree;
170 	struct rb_root journal_tree_root;
171 
172 	sector_t provided_data_sectors;
173 
174 	unsigned short journal_entry_size;
175 	unsigned char journal_entries_per_sector;
176 	unsigned char journal_section_entries;
177 	unsigned short journal_section_sectors;
178 	unsigned journal_sections;
179 	unsigned journal_entries;
180 	sector_t data_device_sectors;
181 	sector_t meta_device_sectors;
182 	unsigned initial_sectors;
183 	unsigned metadata_run;
184 	__s8 log2_metadata_run;
185 	__u8 log2_buffer_sectors;
186 	__u8 sectors_per_block;
187 
188 	unsigned char mode;
189 	int suspending;
190 
191 	int failed;
192 
193 	struct crypto_shash *internal_hash;
194 
195 	/* these variables are locked with endio_wait.lock */
196 	struct rb_root in_progress;
197 	struct list_head wait_list;
198 	wait_queue_head_t endio_wait;
199 	struct workqueue_struct *wait_wq;
200 
201 	unsigned char commit_seq;
202 	commit_id_t commit_ids[N_COMMIT_IDS];
203 
204 	unsigned committed_section;
205 	unsigned n_committed_sections;
206 
207 	unsigned uncommitted_section;
208 	unsigned n_uncommitted_sections;
209 
210 	unsigned free_section;
211 	unsigned char free_section_entry;
212 	unsigned free_sectors;
213 
214 	unsigned free_sectors_threshold;
215 
216 	struct workqueue_struct *commit_wq;
217 	struct work_struct commit_work;
218 
219 	struct workqueue_struct *writer_wq;
220 	struct work_struct writer_work;
221 
222 	struct workqueue_struct *recalc_wq;
223 	struct work_struct recalc_work;
224 	u8 *recalc_buffer;
225 	u8 *recalc_tags;
226 
227 	struct bio_list flush_bio_list;
228 
229 	unsigned long autocommit_jiffies;
230 	struct timer_list autocommit_timer;
231 	unsigned autocommit_msec;
232 
233 	wait_queue_head_t copy_to_journal_wait;
234 
235 	struct completion crypto_backoff;
236 
237 	bool journal_uptodate;
238 	bool just_formatted;
239 
240 	struct alg_spec internal_hash_alg;
241 	struct alg_spec journal_crypt_alg;
242 	struct alg_spec journal_mac_alg;
243 
244 	atomic64_t number_of_mismatches;
245 };
246 
247 struct dm_integrity_range {
248 	sector_t logical_sector;
249 	unsigned n_sectors;
250 	bool waiting;
251 	union {
252 		struct rb_node node;
253 		struct {
254 			struct task_struct *task;
255 			struct list_head wait_entry;
256 		};
257 	};
258 };
259 
260 struct dm_integrity_io {
261 	struct work_struct work;
262 
263 	struct dm_integrity_c *ic;
264 	bool write;
265 	bool fua;
266 
267 	struct dm_integrity_range range;
268 
269 	sector_t metadata_block;
270 	unsigned metadata_offset;
271 
272 	atomic_t in_flight;
273 	blk_status_t bi_status;
274 
275 	struct completion *completion;
276 
277 	struct gendisk *orig_bi_disk;
278 	u8 orig_bi_partno;
279 	bio_end_io_t *orig_bi_end_io;
280 	struct bio_integrity_payload *orig_bi_integrity;
281 	struct bvec_iter orig_bi_iter;
282 };
283 
284 struct journal_completion {
285 	struct dm_integrity_c *ic;
286 	atomic_t in_flight;
287 	struct completion comp;
288 };
289 
290 struct journal_io {
291 	struct dm_integrity_range range;
292 	struct journal_completion *comp;
293 };
294 
295 static struct kmem_cache *journal_io_cache;
296 
297 #define JOURNAL_IO_MEMPOOL	32
298 
299 #ifdef DEBUG_PRINT
300 #define DEBUG_print(x, ...)	printk(KERN_DEBUG x, ##__VA_ARGS__)
301 static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...)
302 {
303 	va_list args;
304 	va_start(args, msg);
305 	vprintk(msg, args);
306 	va_end(args);
307 	if (len)
308 		pr_cont(":");
309 	while (len) {
310 		pr_cont(" %02x", *bytes);
311 		bytes++;
312 		len--;
313 	}
314 	pr_cont("\n");
315 }
316 #define DEBUG_bytes(bytes, len, msg, ...)	__DEBUG_bytes(bytes, len, KERN_DEBUG msg, ##__VA_ARGS__)
317 #else
318 #define DEBUG_print(x, ...)			do { } while (0)
319 #define DEBUG_bytes(bytes, len, msg, ...)	do { } while (0)
320 #endif
321 
322 /*
323  * DM Integrity profile, protection is performed layer above (dm-crypt)
324  */
325 static const struct blk_integrity_profile dm_integrity_profile = {
326 	.name			= "DM-DIF-EXT-TAG",
327 	.generate_fn		= NULL,
328 	.verify_fn		= NULL,
329 };
330 
331 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
332 static void integrity_bio_wait(struct work_struct *w);
333 static void dm_integrity_dtr(struct dm_target *ti);
334 
335 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err)
336 {
337 	if (err == -EILSEQ)
338 		atomic64_inc(&ic->number_of_mismatches);
339 	if (!cmpxchg(&ic->failed, 0, err))
340 		DMERR("Error on %s: %d", msg, err);
341 }
342 
343 static int dm_integrity_failed(struct dm_integrity_c *ic)
344 {
345 	return READ_ONCE(ic->failed);
346 }
347 
348 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i,
349 					  unsigned j, unsigned char seq)
350 {
351 	/*
352 	 * Xor the number with section and sector, so that if a piece of
353 	 * journal is written at wrong place, it is detected.
354 	 */
355 	return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j);
356 }
357 
358 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector,
359 				sector_t *area, sector_t *offset)
360 {
361 	if (!ic->meta_dev) {
362 		__u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors;
363 		*area = data_sector >> log2_interleave_sectors;
364 		*offset = (unsigned)data_sector & ((1U << log2_interleave_sectors) - 1);
365 	} else {
366 		*area = 0;
367 		*offset = data_sector;
368 	}
369 }
370 
371 #define sector_to_block(ic, n)						\
372 do {									\
373 	BUG_ON((n) & (unsigned)((ic)->sectors_per_block - 1));		\
374 	(n) >>= (ic)->sb->log2_sectors_per_block;			\
375 } while (0)
376 
377 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area,
378 					    sector_t offset, unsigned *metadata_offset)
379 {
380 	__u64 ms;
381 	unsigned mo;
382 
383 	ms = area << ic->sb->log2_interleave_sectors;
384 	if (likely(ic->log2_metadata_run >= 0))
385 		ms += area << ic->log2_metadata_run;
386 	else
387 		ms += area * ic->metadata_run;
388 	ms >>= ic->log2_buffer_sectors;
389 
390 	sector_to_block(ic, offset);
391 
392 	if (likely(ic->log2_tag_size >= 0)) {
393 		ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size);
394 		mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
395 	} else {
396 		ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors);
397 		mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
398 	}
399 	*metadata_offset = mo;
400 	return ms;
401 }
402 
403 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset)
404 {
405 	sector_t result;
406 
407 	if (ic->meta_dev)
408 		return offset;
409 
410 	result = area << ic->sb->log2_interleave_sectors;
411 	if (likely(ic->log2_metadata_run >= 0))
412 		result += (area + 1) << ic->log2_metadata_run;
413 	else
414 		result += (area + 1) * ic->metadata_run;
415 
416 	result += (sector_t)ic->initial_sectors + offset;
417 	result += ic->start;
418 
419 	return result;
420 }
421 
422 static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr)
423 {
424 	if (unlikely(*sec_ptr >= ic->journal_sections))
425 		*sec_ptr -= ic->journal_sections;
426 }
427 
428 static void sb_set_version(struct dm_integrity_c *ic)
429 {
430 	if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
431 		ic->sb->version = SB_VERSION_2;
432 	else
433 		ic->sb->version = SB_VERSION_1;
434 }
435 
436 static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags)
437 {
438 	struct dm_io_request io_req;
439 	struct dm_io_region io_loc;
440 
441 	io_req.bi_op = op;
442 	io_req.bi_op_flags = op_flags;
443 	io_req.mem.type = DM_IO_KMEM;
444 	io_req.mem.ptr.addr = ic->sb;
445 	io_req.notify.fn = NULL;
446 	io_req.client = ic->io;
447 	io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
448 	io_loc.sector = ic->start;
449 	io_loc.count = SB_SECTORS;
450 
451 	return dm_io(&io_req, 1, &io_loc, NULL);
452 }
453 
454 static void access_journal_check(struct dm_integrity_c *ic, unsigned section, unsigned offset,
455 				 bool e, const char *function)
456 {
457 #if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY)
458 	unsigned limit = e ? ic->journal_section_entries : ic->journal_section_sectors;
459 
460 	if (unlikely(section >= ic->journal_sections) ||
461 	    unlikely(offset >= limit)) {
462 		printk(KERN_CRIT "%s: invalid access at (%u,%u), limit (%u,%u)\n",
463 			function, section, offset, ic->journal_sections, limit);
464 		BUG();
465 	}
466 #endif
467 }
468 
469 static void page_list_location(struct dm_integrity_c *ic, unsigned section, unsigned offset,
470 			       unsigned *pl_index, unsigned *pl_offset)
471 {
472 	unsigned sector;
473 
474 	access_journal_check(ic, section, offset, false, "page_list_location");
475 
476 	sector = section * ic->journal_section_sectors + offset;
477 
478 	*pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
479 	*pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
480 }
481 
482 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl,
483 					       unsigned section, unsigned offset, unsigned *n_sectors)
484 {
485 	unsigned pl_index, pl_offset;
486 	char *va;
487 
488 	page_list_location(ic, section, offset, &pl_index, &pl_offset);
489 
490 	if (n_sectors)
491 		*n_sectors = (PAGE_SIZE - pl_offset) >> SECTOR_SHIFT;
492 
493 	va = lowmem_page_address(pl[pl_index].page);
494 
495 	return (struct journal_sector *)(va + pl_offset);
496 }
497 
498 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset)
499 {
500 	return access_page_list(ic, ic->journal, section, offset, NULL);
501 }
502 
503 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned section, unsigned n)
504 {
505 	unsigned rel_sector, offset;
506 	struct journal_sector *js;
507 
508 	access_journal_check(ic, section, n, true, "access_journal_entry");
509 
510 	rel_sector = n % JOURNAL_BLOCK_SECTORS;
511 	offset = n / JOURNAL_BLOCK_SECTORS;
512 
513 	js = access_journal(ic, section, rel_sector);
514 	return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size);
515 }
516 
517 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned section, unsigned n)
518 {
519 	n <<= ic->sb->log2_sectors_per_block;
520 
521 	n += JOURNAL_BLOCK_SECTORS;
522 
523 	access_journal_check(ic, section, n, false, "access_journal_data");
524 
525 	return access_journal(ic, section, n);
526 }
527 
528 static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result[JOURNAL_MAC_SIZE])
529 {
530 	SHASH_DESC_ON_STACK(desc, ic->journal_mac);
531 	int r;
532 	unsigned j, size;
533 
534 	desc->tfm = ic->journal_mac;
535 	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
536 
537 	r = crypto_shash_init(desc);
538 	if (unlikely(r)) {
539 		dm_integrity_io_error(ic, "crypto_shash_init", r);
540 		goto err;
541 	}
542 
543 	for (j = 0; j < ic->journal_section_entries; j++) {
544 		struct journal_entry *je = access_journal_entry(ic, section, j);
545 		r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof je->u.sector);
546 		if (unlikely(r)) {
547 			dm_integrity_io_error(ic, "crypto_shash_update", r);
548 			goto err;
549 		}
550 	}
551 
552 	size = crypto_shash_digestsize(ic->journal_mac);
553 
554 	if (likely(size <= JOURNAL_MAC_SIZE)) {
555 		r = crypto_shash_final(desc, result);
556 		if (unlikely(r)) {
557 			dm_integrity_io_error(ic, "crypto_shash_final", r);
558 			goto err;
559 		}
560 		memset(result + size, 0, JOURNAL_MAC_SIZE - size);
561 	} else {
562 		__u8 digest[size];
563 		r = crypto_shash_final(desc, digest);
564 		if (unlikely(r)) {
565 			dm_integrity_io_error(ic, "crypto_shash_final", r);
566 			goto err;
567 		}
568 		memcpy(result, digest, JOURNAL_MAC_SIZE);
569 	}
570 
571 	return;
572 err:
573 	memset(result, 0, JOURNAL_MAC_SIZE);
574 }
575 
576 static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr)
577 {
578 	__u8 result[JOURNAL_MAC_SIZE];
579 	unsigned j;
580 
581 	if (!ic->journal_mac)
582 		return;
583 
584 	section_mac(ic, section, result);
585 
586 	for (j = 0; j < JOURNAL_BLOCK_SECTORS; j++) {
587 		struct journal_sector *js = access_journal(ic, section, j);
588 
589 		if (likely(wr))
590 			memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR);
591 		else {
592 			if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR))
593 				dm_integrity_io_error(ic, "journal mac", -EILSEQ);
594 		}
595 	}
596 }
597 
598 static void complete_journal_op(void *context)
599 {
600 	struct journal_completion *comp = context;
601 	BUG_ON(!atomic_read(&comp->in_flight));
602 	if (likely(atomic_dec_and_test(&comp->in_flight)))
603 		complete(&comp->comp);
604 }
605 
606 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
607 			unsigned n_sections, struct journal_completion *comp)
608 {
609 	struct async_submit_ctl submit;
610 	size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT;
611 	unsigned pl_index, pl_offset, section_index;
612 	struct page_list *source_pl, *target_pl;
613 
614 	if (likely(encrypt)) {
615 		source_pl = ic->journal;
616 		target_pl = ic->journal_io;
617 	} else {
618 		source_pl = ic->journal_io;
619 		target_pl = ic->journal;
620 	}
621 
622 	page_list_location(ic, section, 0, &pl_index, &pl_offset);
623 
624 	atomic_add(roundup(pl_offset + n_bytes, PAGE_SIZE) >> PAGE_SHIFT, &comp->in_flight);
625 
626 	init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, complete_journal_op, comp, NULL);
627 
628 	section_index = pl_index;
629 
630 	do {
631 		size_t this_step;
632 		struct page *src_pages[2];
633 		struct page *dst_page;
634 
635 		while (unlikely(pl_index == section_index)) {
636 			unsigned dummy;
637 			if (likely(encrypt))
638 				rw_section_mac(ic, section, true);
639 			section++;
640 			n_sections--;
641 			if (!n_sections)
642 				break;
643 			page_list_location(ic, section, 0, &section_index, &dummy);
644 		}
645 
646 		this_step = min(n_bytes, (size_t)PAGE_SIZE - pl_offset);
647 		dst_page = target_pl[pl_index].page;
648 		src_pages[0] = source_pl[pl_index].page;
649 		src_pages[1] = ic->journal_xor[pl_index].page;
650 
651 		async_xor(dst_page, src_pages, pl_offset, 2, this_step, &submit);
652 
653 		pl_index++;
654 		pl_offset = 0;
655 		n_bytes -= this_step;
656 	} while (n_bytes);
657 
658 	BUG_ON(n_sections);
659 
660 	async_tx_issue_pending_all();
661 }
662 
663 static void complete_journal_encrypt(struct crypto_async_request *req, int err)
664 {
665 	struct journal_completion *comp = req->data;
666 	if (unlikely(err)) {
667 		if (likely(err == -EINPROGRESS)) {
668 			complete(&comp->ic->crypto_backoff);
669 			return;
670 		}
671 		dm_integrity_io_error(comp->ic, "asynchronous encrypt", err);
672 	}
673 	complete_journal_op(comp);
674 }
675 
676 static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp)
677 {
678 	int r;
679 	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
680 				      complete_journal_encrypt, comp);
681 	if (likely(encrypt))
682 		r = crypto_skcipher_encrypt(req);
683 	else
684 		r = crypto_skcipher_decrypt(req);
685 	if (likely(!r))
686 		return false;
687 	if (likely(r == -EINPROGRESS))
688 		return true;
689 	if (likely(r == -EBUSY)) {
690 		wait_for_completion(&comp->ic->crypto_backoff);
691 		reinit_completion(&comp->ic->crypto_backoff);
692 		return true;
693 	}
694 	dm_integrity_io_error(comp->ic, "encrypt", r);
695 	return false;
696 }
697 
698 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
699 			  unsigned n_sections, struct journal_completion *comp)
700 {
701 	struct scatterlist **source_sg;
702 	struct scatterlist **target_sg;
703 
704 	atomic_add(2, &comp->in_flight);
705 
706 	if (likely(encrypt)) {
707 		source_sg = ic->journal_scatterlist;
708 		target_sg = ic->journal_io_scatterlist;
709 	} else {
710 		source_sg = ic->journal_io_scatterlist;
711 		target_sg = ic->journal_scatterlist;
712 	}
713 
714 	do {
715 		struct skcipher_request *req;
716 		unsigned ivsize;
717 		char *iv;
718 
719 		if (likely(encrypt))
720 			rw_section_mac(ic, section, true);
721 
722 		req = ic->sk_requests[section];
723 		ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
724 		iv = req->iv;
725 
726 		memcpy(iv, iv + ivsize, ivsize);
727 
728 		req->src = source_sg[section];
729 		req->dst = target_sg[section];
730 
731 		if (unlikely(do_crypt(encrypt, req, comp)))
732 			atomic_inc(&comp->in_flight);
733 
734 		section++;
735 		n_sections--;
736 	} while (n_sections);
737 
738 	atomic_dec(&comp->in_flight);
739 	complete_journal_op(comp);
740 }
741 
742 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
743 			    unsigned n_sections, struct journal_completion *comp)
744 {
745 	if (ic->journal_xor)
746 		return xor_journal(ic, encrypt, section, n_sections, comp);
747 	else
748 		return crypt_journal(ic, encrypt, section, n_sections, comp);
749 }
750 
751 static void complete_journal_io(unsigned long error, void *context)
752 {
753 	struct journal_completion *comp = context;
754 	if (unlikely(error != 0))
755 		dm_integrity_io_error(comp->ic, "writing journal", -EIO);
756 	complete_journal_op(comp);
757 }
758 
759 static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned section,
760 		       unsigned n_sections, struct journal_completion *comp)
761 {
762 	struct dm_io_request io_req;
763 	struct dm_io_region io_loc;
764 	unsigned sector, n_sectors, pl_index, pl_offset;
765 	int r;
766 
767 	if (unlikely(dm_integrity_failed(ic))) {
768 		if (comp)
769 			complete_journal_io(-1UL, comp);
770 		return;
771 	}
772 
773 	sector = section * ic->journal_section_sectors;
774 	n_sectors = n_sections * ic->journal_section_sectors;
775 
776 	pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
777 	pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
778 
779 	io_req.bi_op = op;
780 	io_req.bi_op_flags = op_flags;
781 	io_req.mem.type = DM_IO_PAGE_LIST;
782 	if (ic->journal_io)
783 		io_req.mem.ptr.pl = &ic->journal_io[pl_index];
784 	else
785 		io_req.mem.ptr.pl = &ic->journal[pl_index];
786 	io_req.mem.offset = pl_offset;
787 	if (likely(comp != NULL)) {
788 		io_req.notify.fn = complete_journal_io;
789 		io_req.notify.context = comp;
790 	} else {
791 		io_req.notify.fn = NULL;
792 	}
793 	io_req.client = ic->io;
794 	io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
795 	io_loc.sector = ic->start + SB_SECTORS + sector;
796 	io_loc.count = n_sectors;
797 
798 	r = dm_io(&io_req, 1, &io_loc, NULL);
799 	if (unlikely(r)) {
800 		dm_integrity_io_error(ic, op == REQ_OP_READ ? "reading journal" : "writing journal", r);
801 		if (comp) {
802 			WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
803 			complete_journal_io(-1UL, comp);
804 		}
805 	}
806 }
807 
808 static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsigned commit_sections)
809 {
810 	struct journal_completion io_comp;
811 	struct journal_completion crypt_comp_1;
812 	struct journal_completion crypt_comp_2;
813 	unsigned i;
814 
815 	io_comp.ic = ic;
816 	init_completion(&io_comp.comp);
817 
818 	if (commit_start + commit_sections <= ic->journal_sections) {
819 		io_comp.in_flight = (atomic_t)ATOMIC_INIT(1);
820 		if (ic->journal_io) {
821 			crypt_comp_1.ic = ic;
822 			init_completion(&crypt_comp_1.comp);
823 			crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
824 			encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1);
825 			wait_for_completion_io(&crypt_comp_1.comp);
826 		} else {
827 			for (i = 0; i < commit_sections; i++)
828 				rw_section_mac(ic, commit_start + i, true);
829 		}
830 		rw_journal(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, commit_start,
831 			   commit_sections, &io_comp);
832 	} else {
833 		unsigned to_end;
834 		io_comp.in_flight = (atomic_t)ATOMIC_INIT(2);
835 		to_end = ic->journal_sections - commit_start;
836 		if (ic->journal_io) {
837 			crypt_comp_1.ic = ic;
838 			init_completion(&crypt_comp_1.comp);
839 			crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
840 			encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
841 			if (try_wait_for_completion(&crypt_comp_1.comp)) {
842 				rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
843 				reinit_completion(&crypt_comp_1.comp);
844 				crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
845 				encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
846 				wait_for_completion_io(&crypt_comp_1.comp);
847 			} else {
848 				crypt_comp_2.ic = ic;
849 				init_completion(&crypt_comp_2.comp);
850 				crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
851 				encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
852 				wait_for_completion_io(&crypt_comp_1.comp);
853 				rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
854 				wait_for_completion_io(&crypt_comp_2.comp);
855 			}
856 		} else {
857 			for (i = 0; i < to_end; i++)
858 				rw_section_mac(ic, commit_start + i, true);
859 			rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
860 			for (i = 0; i < commit_sections - to_end; i++)
861 				rw_section_mac(ic, i, true);
862 		}
863 		rw_journal(ic, REQ_OP_WRITE, REQ_FUA, 0, commit_sections - to_end, &io_comp);
864 	}
865 
866 	wait_for_completion_io(&io_comp.comp);
867 }
868 
869 static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset,
870 			      unsigned n_sectors, sector_t target, io_notify_fn fn, void *data)
871 {
872 	struct dm_io_request io_req;
873 	struct dm_io_region io_loc;
874 	int r;
875 	unsigned sector, pl_index, pl_offset;
876 
877 	BUG_ON((target | n_sectors | offset) & (unsigned)(ic->sectors_per_block - 1));
878 
879 	if (unlikely(dm_integrity_failed(ic))) {
880 		fn(-1UL, data);
881 		return;
882 	}
883 
884 	sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset;
885 
886 	pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
887 	pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
888 
889 	io_req.bi_op = REQ_OP_WRITE;
890 	io_req.bi_op_flags = 0;
891 	io_req.mem.type = DM_IO_PAGE_LIST;
892 	io_req.mem.ptr.pl = &ic->journal[pl_index];
893 	io_req.mem.offset = pl_offset;
894 	io_req.notify.fn = fn;
895 	io_req.notify.context = data;
896 	io_req.client = ic->io;
897 	io_loc.bdev = ic->dev->bdev;
898 	io_loc.sector = target;
899 	io_loc.count = n_sectors;
900 
901 	r = dm_io(&io_req, 1, &io_loc, NULL);
902 	if (unlikely(r)) {
903 		WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
904 		fn(-1UL, data);
905 	}
906 }
907 
908 static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2)
909 {
910 	return range1->logical_sector < range2->logical_sector + range2->n_sectors &&
911 	       range2->logical_sector + range2->n_sectors > range2->logical_sector;
912 }
913 
914 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting)
915 {
916 	struct rb_node **n = &ic->in_progress.rb_node;
917 	struct rb_node *parent;
918 
919 	BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned)(ic->sectors_per_block - 1));
920 
921 	if (likely(check_waiting)) {
922 		struct dm_integrity_range *range;
923 		list_for_each_entry(range, &ic->wait_list, wait_entry) {
924 			if (unlikely(ranges_overlap(range, new_range)))
925 				return false;
926 		}
927 	}
928 
929 	parent = NULL;
930 
931 	while (*n) {
932 		struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node);
933 
934 		parent = *n;
935 		if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector) {
936 			n = &range->node.rb_left;
937 		} else if (new_range->logical_sector >= range->logical_sector + range->n_sectors) {
938 			n = &range->node.rb_right;
939 		} else {
940 			return false;
941 		}
942 	}
943 
944 	rb_link_node(&new_range->node, parent, n);
945 	rb_insert_color(&new_range->node, &ic->in_progress);
946 
947 	return true;
948 }
949 
950 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range)
951 {
952 	rb_erase(&range->node, &ic->in_progress);
953 	while (unlikely(!list_empty(&ic->wait_list))) {
954 		struct dm_integrity_range *last_range =
955 			list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry);
956 		struct task_struct *last_range_task;
957 		if (!ranges_overlap(range, last_range))
958 			break;
959 		last_range_task = last_range->task;
960 		list_del(&last_range->wait_entry);
961 		if (!add_new_range(ic, last_range, false)) {
962 			last_range->task = last_range_task;
963 			list_add(&last_range->wait_entry, &ic->wait_list);
964 			break;
965 		}
966 		last_range->waiting = false;
967 		wake_up_process(last_range_task);
968 	}
969 }
970 
971 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range)
972 {
973 	unsigned long flags;
974 
975 	spin_lock_irqsave(&ic->endio_wait.lock, flags);
976 	remove_range_unlocked(ic, range);
977 	spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
978 }
979 
980 static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
981 {
982 	new_range->waiting = true;
983 	list_add_tail(&new_range->wait_entry, &ic->wait_list);
984 	new_range->task = current;
985 	do {
986 		__set_current_state(TASK_UNINTERRUPTIBLE);
987 		spin_unlock_irq(&ic->endio_wait.lock);
988 		io_schedule();
989 		spin_lock_irq(&ic->endio_wait.lock);
990 	} while (unlikely(new_range->waiting));
991 }
992 
993 static void init_journal_node(struct journal_node *node)
994 {
995 	RB_CLEAR_NODE(&node->node);
996 	node->sector = (sector_t)-1;
997 }
998 
999 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector)
1000 {
1001 	struct rb_node **link;
1002 	struct rb_node *parent;
1003 
1004 	node->sector = sector;
1005 	BUG_ON(!RB_EMPTY_NODE(&node->node));
1006 
1007 	link = &ic->journal_tree_root.rb_node;
1008 	parent = NULL;
1009 
1010 	while (*link) {
1011 		struct journal_node *j;
1012 		parent = *link;
1013 		j = container_of(parent, struct journal_node, node);
1014 		if (sector < j->sector)
1015 			link = &j->node.rb_left;
1016 		else
1017 			link = &j->node.rb_right;
1018 	}
1019 
1020 	rb_link_node(&node->node, parent, link);
1021 	rb_insert_color(&node->node, &ic->journal_tree_root);
1022 }
1023 
1024 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node)
1025 {
1026 	BUG_ON(RB_EMPTY_NODE(&node->node));
1027 	rb_erase(&node->node, &ic->journal_tree_root);
1028 	init_journal_node(node);
1029 }
1030 
1031 #define NOT_FOUND	(-1U)
1032 
1033 static unsigned find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector)
1034 {
1035 	struct rb_node *n = ic->journal_tree_root.rb_node;
1036 	unsigned found = NOT_FOUND;
1037 	*next_sector = (sector_t)-1;
1038 	while (n) {
1039 		struct journal_node *j = container_of(n, struct journal_node, node);
1040 		if (sector == j->sector) {
1041 			found = j - ic->journal_tree;
1042 		}
1043 		if (sector < j->sector) {
1044 			*next_sector = j->sector;
1045 			n = j->node.rb_left;
1046 		} else {
1047 			n = j->node.rb_right;
1048 		}
1049 	}
1050 
1051 	return found;
1052 }
1053 
1054 static bool test_journal_node(struct dm_integrity_c *ic, unsigned pos, sector_t sector)
1055 {
1056 	struct journal_node *node, *next_node;
1057 	struct rb_node *next;
1058 
1059 	if (unlikely(pos >= ic->journal_entries))
1060 		return false;
1061 	node = &ic->journal_tree[pos];
1062 	if (unlikely(RB_EMPTY_NODE(&node->node)))
1063 		return false;
1064 	if (unlikely(node->sector != sector))
1065 		return false;
1066 
1067 	next = rb_next(&node->node);
1068 	if (unlikely(!next))
1069 		return true;
1070 
1071 	next_node = container_of(next, struct journal_node, node);
1072 	return next_node->sector != sector;
1073 }
1074 
1075 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node)
1076 {
1077 	struct rb_node *next;
1078 	struct journal_node *next_node;
1079 	unsigned next_section;
1080 
1081 	BUG_ON(RB_EMPTY_NODE(&node->node));
1082 
1083 	next = rb_next(&node->node);
1084 	if (unlikely(!next))
1085 		return false;
1086 
1087 	next_node = container_of(next, struct journal_node, node);
1088 
1089 	if (next_node->sector != node->sector)
1090 		return false;
1091 
1092 	next_section = (unsigned)(next_node - ic->journal_tree) / ic->journal_section_entries;
1093 	if (next_section >= ic->committed_section &&
1094 	    next_section < ic->committed_section + ic->n_committed_sections)
1095 		return true;
1096 	if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections)
1097 		return true;
1098 
1099 	return false;
1100 }
1101 
1102 #define TAG_READ	0
1103 #define TAG_WRITE	1
1104 #define TAG_CMP		2
1105 
1106 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block,
1107 			       unsigned *metadata_offset, unsigned total_size, int op)
1108 {
1109 	do {
1110 		unsigned char *data, *dp;
1111 		struct dm_buffer *b;
1112 		unsigned to_copy;
1113 		int r;
1114 
1115 		r = dm_integrity_failed(ic);
1116 		if (unlikely(r))
1117 			return r;
1118 
1119 		data = dm_bufio_read(ic->bufio, *metadata_block, &b);
1120 		if (unlikely(IS_ERR(data)))
1121 			return PTR_ERR(data);
1122 
1123 		to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size);
1124 		dp = data + *metadata_offset;
1125 		if (op == TAG_READ) {
1126 			memcpy(tag, dp, to_copy);
1127 		} else if (op == TAG_WRITE) {
1128 			memcpy(dp, tag, to_copy);
1129 			dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy);
1130 		} else  {
1131 			/* e.g.: op == TAG_CMP */
1132 			if (unlikely(memcmp(dp, tag, to_copy))) {
1133 				unsigned i;
1134 
1135 				for (i = 0; i < to_copy; i++) {
1136 					if (dp[i] != tag[i])
1137 						break;
1138 					total_size--;
1139 				}
1140 				dm_bufio_release(b);
1141 				return total_size;
1142 			}
1143 		}
1144 		dm_bufio_release(b);
1145 
1146 		tag += to_copy;
1147 		*metadata_offset += to_copy;
1148 		if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) {
1149 			(*metadata_block)++;
1150 			*metadata_offset = 0;
1151 		}
1152 		total_size -= to_copy;
1153 	} while (unlikely(total_size));
1154 
1155 	return 0;
1156 }
1157 
1158 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic)
1159 {
1160 	int r;
1161 	r = dm_bufio_write_dirty_buffers(ic->bufio);
1162 	if (unlikely(r))
1163 		dm_integrity_io_error(ic, "writing tags", r);
1164 }
1165 
1166 static void sleep_on_endio_wait(struct dm_integrity_c *ic)
1167 {
1168 	DECLARE_WAITQUEUE(wait, current);
1169 	__add_wait_queue(&ic->endio_wait, &wait);
1170 	__set_current_state(TASK_UNINTERRUPTIBLE);
1171 	spin_unlock_irq(&ic->endio_wait.lock);
1172 	io_schedule();
1173 	spin_lock_irq(&ic->endio_wait.lock);
1174 	__remove_wait_queue(&ic->endio_wait, &wait);
1175 }
1176 
1177 static void autocommit_fn(struct timer_list *t)
1178 {
1179 	struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer);
1180 
1181 	if (likely(!dm_integrity_failed(ic)))
1182 		queue_work(ic->commit_wq, &ic->commit_work);
1183 }
1184 
1185 static void schedule_autocommit(struct dm_integrity_c *ic)
1186 {
1187 	if (!timer_pending(&ic->autocommit_timer))
1188 		mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies);
1189 }
1190 
1191 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1192 {
1193 	struct bio *bio;
1194 	unsigned long flags;
1195 
1196 	spin_lock_irqsave(&ic->endio_wait.lock, flags);
1197 	bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1198 	bio_list_add(&ic->flush_bio_list, bio);
1199 	spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1200 
1201 	queue_work(ic->commit_wq, &ic->commit_work);
1202 }
1203 
1204 static void do_endio(struct dm_integrity_c *ic, struct bio *bio)
1205 {
1206 	int r = dm_integrity_failed(ic);
1207 	if (unlikely(r) && !bio->bi_status)
1208 		bio->bi_status = errno_to_blk_status(r);
1209 	bio_endio(bio);
1210 }
1211 
1212 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1213 {
1214 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1215 
1216 	if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))
1217 		submit_flush_bio(ic, dio);
1218 	else
1219 		do_endio(ic, bio);
1220 }
1221 
1222 static void dec_in_flight(struct dm_integrity_io *dio)
1223 {
1224 	if (atomic_dec_and_test(&dio->in_flight)) {
1225 		struct dm_integrity_c *ic = dio->ic;
1226 		struct bio *bio;
1227 
1228 		remove_range(ic, &dio->range);
1229 
1230 		if (unlikely(dio->write))
1231 			schedule_autocommit(ic);
1232 
1233 		bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1234 
1235 		if (unlikely(dio->bi_status) && !bio->bi_status)
1236 			bio->bi_status = dio->bi_status;
1237 		if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {
1238 			dio->range.logical_sector += dio->range.n_sectors;
1239 			bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);
1240 			INIT_WORK(&dio->work, integrity_bio_wait);
1241 			queue_work(ic->wait_wq, &dio->work);
1242 			return;
1243 		}
1244 		do_endio_flush(ic, dio);
1245 	}
1246 }
1247 
1248 static void integrity_end_io(struct bio *bio)
1249 {
1250 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1251 
1252 	bio->bi_iter = dio->orig_bi_iter;
1253 	bio->bi_disk = dio->orig_bi_disk;
1254 	bio->bi_partno = dio->orig_bi_partno;
1255 	if (dio->orig_bi_integrity) {
1256 		bio->bi_integrity = dio->orig_bi_integrity;
1257 		bio->bi_opf |= REQ_INTEGRITY;
1258 	}
1259 	bio->bi_end_io = dio->orig_bi_end_io;
1260 
1261 	if (dio->completion)
1262 		complete(dio->completion);
1263 
1264 	dec_in_flight(dio);
1265 }
1266 
1267 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector,
1268 				      const char *data, char *result)
1269 {
1270 	__u64 sector_le = cpu_to_le64(sector);
1271 	SHASH_DESC_ON_STACK(req, ic->internal_hash);
1272 	int r;
1273 	unsigned digest_size;
1274 
1275 	req->tfm = ic->internal_hash;
1276 	req->flags = 0;
1277 
1278 	r = crypto_shash_init(req);
1279 	if (unlikely(r < 0)) {
1280 		dm_integrity_io_error(ic, "crypto_shash_init", r);
1281 		goto failed;
1282 	}
1283 
1284 	r = crypto_shash_update(req, (const __u8 *)&sector_le, sizeof sector_le);
1285 	if (unlikely(r < 0)) {
1286 		dm_integrity_io_error(ic, "crypto_shash_update", r);
1287 		goto failed;
1288 	}
1289 
1290 	r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT);
1291 	if (unlikely(r < 0)) {
1292 		dm_integrity_io_error(ic, "crypto_shash_update", r);
1293 		goto failed;
1294 	}
1295 
1296 	r = crypto_shash_final(req, result);
1297 	if (unlikely(r < 0)) {
1298 		dm_integrity_io_error(ic, "crypto_shash_final", r);
1299 		goto failed;
1300 	}
1301 
1302 	digest_size = crypto_shash_digestsize(ic->internal_hash);
1303 	if (unlikely(digest_size < ic->tag_size))
1304 		memset(result + digest_size, 0, ic->tag_size - digest_size);
1305 
1306 	return;
1307 
1308 failed:
1309 	/* this shouldn't happen anyway, the hash functions have no reason to fail */
1310 	get_random_bytes(result, ic->tag_size);
1311 }
1312 
1313 static void integrity_metadata(struct work_struct *w)
1314 {
1315 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1316 	struct dm_integrity_c *ic = dio->ic;
1317 
1318 	int r;
1319 
1320 	if (ic->internal_hash) {
1321 		struct bvec_iter iter;
1322 		struct bio_vec bv;
1323 		unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1324 		struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1325 		char *checksums;
1326 		unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
1327 		char checksums_onstack[ic->tag_size + extra_space];
1328 		unsigned sectors_to_process = dio->range.n_sectors;
1329 		sector_t sector = dio->range.logical_sector;
1330 
1331 		if (unlikely(ic->mode == 'R'))
1332 			goto skip_io;
1333 
1334 		checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
1335 				    GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1336 		if (!checksums)
1337 			checksums = checksums_onstack;
1338 
1339 		__bio_for_each_segment(bv, bio, iter, dio->orig_bi_iter) {
1340 			unsigned pos;
1341 			char *mem, *checksums_ptr;
1342 
1343 again:
1344 			mem = (char *)kmap_atomic(bv.bv_page) + bv.bv_offset;
1345 			pos = 0;
1346 			checksums_ptr = checksums;
1347 			do {
1348 				integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr);
1349 				checksums_ptr += ic->tag_size;
1350 				sectors_to_process -= ic->sectors_per_block;
1351 				pos += ic->sectors_per_block << SECTOR_SHIFT;
1352 				sector += ic->sectors_per_block;
1353 			} while (pos < bv.bv_len && sectors_to_process && checksums != checksums_onstack);
1354 			kunmap_atomic(mem);
1355 
1356 			r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1357 						checksums_ptr - checksums, !dio->write ? TAG_CMP : TAG_WRITE);
1358 			if (unlikely(r)) {
1359 				if (r > 0) {
1360 					DMERR("Checksum failed at sector 0x%llx",
1361 					      (unsigned long long)(sector - ((r + ic->tag_size - 1) / ic->tag_size)));
1362 					r = -EILSEQ;
1363 					atomic64_inc(&ic->number_of_mismatches);
1364 				}
1365 				if (likely(checksums != checksums_onstack))
1366 					kfree(checksums);
1367 				goto error;
1368 			}
1369 
1370 			if (!sectors_to_process)
1371 				break;
1372 
1373 			if (unlikely(pos < bv.bv_len)) {
1374 				bv.bv_offset += pos;
1375 				bv.bv_len -= pos;
1376 				goto again;
1377 			}
1378 		}
1379 
1380 		if (likely(checksums != checksums_onstack))
1381 			kfree(checksums);
1382 	} else {
1383 		struct bio_integrity_payload *bip = dio->orig_bi_integrity;
1384 
1385 		if (bip) {
1386 			struct bio_vec biv;
1387 			struct bvec_iter iter;
1388 			unsigned data_to_process = dio->range.n_sectors;
1389 			sector_to_block(ic, data_to_process);
1390 			data_to_process *= ic->tag_size;
1391 
1392 			bip_for_each_vec(biv, bip, iter) {
1393 				unsigned char *tag;
1394 				unsigned this_len;
1395 
1396 				BUG_ON(PageHighMem(biv.bv_page));
1397 				tag = lowmem_page_address(biv.bv_page) + biv.bv_offset;
1398 				this_len = min(biv.bv_len, data_to_process);
1399 				r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,
1400 							this_len, !dio->write ? TAG_READ : TAG_WRITE);
1401 				if (unlikely(r))
1402 					goto error;
1403 				data_to_process -= this_len;
1404 				if (!data_to_process)
1405 					break;
1406 			}
1407 		}
1408 	}
1409 skip_io:
1410 	dec_in_flight(dio);
1411 	return;
1412 error:
1413 	dio->bi_status = errno_to_blk_status(r);
1414 	dec_in_flight(dio);
1415 }
1416 
1417 static int dm_integrity_map(struct dm_target *ti, struct bio *bio)
1418 {
1419 	struct dm_integrity_c *ic = ti->private;
1420 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1421 	struct bio_integrity_payload *bip;
1422 
1423 	sector_t area, offset;
1424 
1425 	dio->ic = ic;
1426 	dio->bi_status = 0;
1427 
1428 	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1429 		submit_flush_bio(ic, dio);
1430 		return DM_MAPIO_SUBMITTED;
1431 	}
1432 
1433 	dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1434 	dio->write = bio_op(bio) == REQ_OP_WRITE;
1435 	dio->fua = dio->write && bio->bi_opf & REQ_FUA;
1436 	if (unlikely(dio->fua)) {
1437 		/*
1438 		 * Don't pass down the FUA flag because we have to flush
1439 		 * disk cache anyway.
1440 		 */
1441 		bio->bi_opf &= ~REQ_FUA;
1442 	}
1443 	if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {
1444 		DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx",
1445 		      (unsigned long long)dio->range.logical_sector, bio_sectors(bio),
1446 		      (unsigned long long)ic->provided_data_sectors);
1447 		return DM_MAPIO_KILL;
1448 	}
1449 	if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1))) {
1450 		DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x",
1451 		      ic->sectors_per_block,
1452 		      (unsigned long long)dio->range.logical_sector, bio_sectors(bio));
1453 		return DM_MAPIO_KILL;
1454 	}
1455 
1456 	if (ic->sectors_per_block > 1) {
1457 		struct bvec_iter iter;
1458 		struct bio_vec bv;
1459 		bio_for_each_segment(bv, bio, iter) {
1460 			if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) {
1461 				DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary",
1462 					bv.bv_offset, bv.bv_len, ic->sectors_per_block);
1463 				return DM_MAPIO_KILL;
1464 			}
1465 		}
1466 	}
1467 
1468 	bip = bio_integrity(bio);
1469 	if (!ic->internal_hash) {
1470 		if (bip) {
1471 			unsigned wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block;
1472 			if (ic->log2_tag_size >= 0)
1473 				wanted_tag_size <<= ic->log2_tag_size;
1474 			else
1475 				wanted_tag_size *= ic->tag_size;
1476 			if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) {
1477 				DMERR("Invalid integrity data size %u, expected %u", bip->bip_iter.bi_size, wanted_tag_size);
1478 				return DM_MAPIO_KILL;
1479 			}
1480 		}
1481 	} else {
1482 		if (unlikely(bip != NULL)) {
1483 			DMERR("Unexpected integrity data when using internal hash");
1484 			return DM_MAPIO_KILL;
1485 		}
1486 	}
1487 
1488 	if (unlikely(ic->mode == 'R') && unlikely(dio->write))
1489 		return DM_MAPIO_KILL;
1490 
1491 	get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1492 	dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1493 	bio->bi_iter.bi_sector = get_data_sector(ic, area, offset);
1494 
1495 	dm_integrity_map_continue(dio, true);
1496 	return DM_MAPIO_SUBMITTED;
1497 }
1498 
1499 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,
1500 				 unsigned journal_section, unsigned journal_entry)
1501 {
1502 	struct dm_integrity_c *ic = dio->ic;
1503 	sector_t logical_sector;
1504 	unsigned n_sectors;
1505 
1506 	logical_sector = dio->range.logical_sector;
1507 	n_sectors = dio->range.n_sectors;
1508 	do {
1509 		struct bio_vec bv = bio_iovec(bio);
1510 		char *mem;
1511 
1512 		if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors))
1513 			bv.bv_len = n_sectors << SECTOR_SHIFT;
1514 		n_sectors -= bv.bv_len >> SECTOR_SHIFT;
1515 		bio_advance_iter(bio, &bio->bi_iter, bv.bv_len);
1516 retry_kmap:
1517 		mem = kmap_atomic(bv.bv_page);
1518 		if (likely(dio->write))
1519 			flush_dcache_page(bv.bv_page);
1520 
1521 		do {
1522 			struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry);
1523 
1524 			if (unlikely(!dio->write)) {
1525 				struct journal_sector *js;
1526 				char *mem_ptr;
1527 				unsigned s;
1528 
1529 				if (unlikely(journal_entry_is_inprogress(je))) {
1530 					flush_dcache_page(bv.bv_page);
1531 					kunmap_atomic(mem);
1532 
1533 					__io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
1534 					goto retry_kmap;
1535 				}
1536 				smp_rmb();
1537 				BUG_ON(journal_entry_get_sector(je) != logical_sector);
1538 				js = access_journal_data(ic, journal_section, journal_entry);
1539 				mem_ptr = mem + bv.bv_offset;
1540 				s = 0;
1541 				do {
1542 					memcpy(mem_ptr, js, JOURNAL_SECTOR_DATA);
1543 					*(commit_id_t *)(mem_ptr + JOURNAL_SECTOR_DATA) = je->last_bytes[s];
1544 					js++;
1545 					mem_ptr += 1 << SECTOR_SHIFT;
1546 				} while (++s < ic->sectors_per_block);
1547 #ifdef INTERNAL_VERIFY
1548 				if (ic->internal_hash) {
1549 					char checksums_onstack[max(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)];
1550 
1551 					integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack);
1552 					if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) {
1553 						DMERR("Checksum failed when reading from journal, at sector 0x%llx",
1554 						      (unsigned long long)logical_sector);
1555 					}
1556 				}
1557 #endif
1558 			}
1559 
1560 			if (!ic->internal_hash) {
1561 				struct bio_integrity_payload *bip = bio_integrity(bio);
1562 				unsigned tag_todo = ic->tag_size;
1563 				char *tag_ptr = journal_entry_tag(ic, je);
1564 
1565 				if (bip) do {
1566 					struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter);
1567 					unsigned tag_now = min(biv.bv_len, tag_todo);
1568 					char *tag_addr;
1569 					BUG_ON(PageHighMem(biv.bv_page));
1570 					tag_addr = lowmem_page_address(biv.bv_page) + biv.bv_offset;
1571 					if (likely(dio->write))
1572 						memcpy(tag_ptr, tag_addr, tag_now);
1573 					else
1574 						memcpy(tag_addr, tag_ptr, tag_now);
1575 					bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now);
1576 					tag_ptr += tag_now;
1577 					tag_todo -= tag_now;
1578 				} while (unlikely(tag_todo)); else {
1579 					if (likely(dio->write))
1580 						memset(tag_ptr, 0, tag_todo);
1581 				}
1582 			}
1583 
1584 			if (likely(dio->write)) {
1585 				struct journal_sector *js;
1586 				unsigned s;
1587 
1588 				js = access_journal_data(ic, journal_section, journal_entry);
1589 				memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT);
1590 
1591 				s = 0;
1592 				do {
1593 					je->last_bytes[s] = js[s].commit_id;
1594 				} while (++s < ic->sectors_per_block);
1595 
1596 				if (ic->internal_hash) {
1597 					unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1598 					if (unlikely(digest_size > ic->tag_size)) {
1599 						char checksums_onstack[digest_size];
1600 						integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack);
1601 						memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size);
1602 					} else
1603 						integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je));
1604 				}
1605 
1606 				journal_entry_set_sector(je, logical_sector);
1607 			}
1608 			logical_sector += ic->sectors_per_block;
1609 
1610 			journal_entry++;
1611 			if (unlikely(journal_entry == ic->journal_section_entries)) {
1612 				journal_entry = 0;
1613 				journal_section++;
1614 				wraparound_section(ic, &journal_section);
1615 			}
1616 
1617 			bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT;
1618 		} while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT);
1619 
1620 		if (unlikely(!dio->write))
1621 			flush_dcache_page(bv.bv_page);
1622 		kunmap_atomic(mem);
1623 	} while (n_sectors);
1624 
1625 	if (likely(dio->write)) {
1626 		smp_mb();
1627 		if (unlikely(waitqueue_active(&ic->copy_to_journal_wait)))
1628 			wake_up(&ic->copy_to_journal_wait);
1629 		if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) {
1630 			queue_work(ic->commit_wq, &ic->commit_work);
1631 		} else {
1632 			schedule_autocommit(ic);
1633 		}
1634 	} else {
1635 		remove_range(ic, &dio->range);
1636 	}
1637 
1638 	if (unlikely(bio->bi_iter.bi_size)) {
1639 		sector_t area, offset;
1640 
1641 		dio->range.logical_sector = logical_sector;
1642 		get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1643 		dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1644 		return true;
1645 	}
1646 
1647 	return false;
1648 }
1649 
1650 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)
1651 {
1652 	struct dm_integrity_c *ic = dio->ic;
1653 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1654 	unsigned journal_section, journal_entry;
1655 	unsigned journal_read_pos;
1656 	struct completion read_comp;
1657 	bool need_sync_io = ic->internal_hash && !dio->write;
1658 
1659 	if (need_sync_io && from_map) {
1660 		INIT_WORK(&dio->work, integrity_bio_wait);
1661 		queue_work(ic->metadata_wq, &dio->work);
1662 		return;
1663 	}
1664 
1665 lock_retry:
1666 	spin_lock_irq(&ic->endio_wait.lock);
1667 retry:
1668 	if (unlikely(dm_integrity_failed(ic))) {
1669 		spin_unlock_irq(&ic->endio_wait.lock);
1670 		do_endio(ic, bio);
1671 		return;
1672 	}
1673 	dio->range.n_sectors = bio_sectors(bio);
1674 	journal_read_pos = NOT_FOUND;
1675 	if (likely(ic->mode == 'J')) {
1676 		if (dio->write) {
1677 			unsigned next_entry, i, pos;
1678 			unsigned ws, we, range_sectors;
1679 
1680 			dio->range.n_sectors = min(dio->range.n_sectors,
1681 						   ic->free_sectors << ic->sb->log2_sectors_per_block);
1682 			if (unlikely(!dio->range.n_sectors)) {
1683 				if (from_map)
1684 					goto offload_to_thread;
1685 				sleep_on_endio_wait(ic);
1686 				goto retry;
1687 			}
1688 			range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;
1689 			ic->free_sectors -= range_sectors;
1690 			journal_section = ic->free_section;
1691 			journal_entry = ic->free_section_entry;
1692 
1693 			next_entry = ic->free_section_entry + range_sectors;
1694 			ic->free_section_entry = next_entry % ic->journal_section_entries;
1695 			ic->free_section += next_entry / ic->journal_section_entries;
1696 			ic->n_uncommitted_sections += next_entry / ic->journal_section_entries;
1697 			wraparound_section(ic, &ic->free_section);
1698 
1699 			pos = journal_section * ic->journal_section_entries + journal_entry;
1700 			ws = journal_section;
1701 			we = journal_entry;
1702 			i = 0;
1703 			do {
1704 				struct journal_entry *je;
1705 
1706 				add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);
1707 				pos++;
1708 				if (unlikely(pos >= ic->journal_entries))
1709 					pos = 0;
1710 
1711 				je = access_journal_entry(ic, ws, we);
1712 				BUG_ON(!journal_entry_is_unused(je));
1713 				journal_entry_set_inprogress(je);
1714 				we++;
1715 				if (unlikely(we == ic->journal_section_entries)) {
1716 					we = 0;
1717 					ws++;
1718 					wraparound_section(ic, &ws);
1719 				}
1720 			} while ((i += ic->sectors_per_block) < dio->range.n_sectors);
1721 
1722 			spin_unlock_irq(&ic->endio_wait.lock);
1723 			goto journal_read_write;
1724 		} else {
1725 			sector_t next_sector;
1726 			journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
1727 			if (likely(journal_read_pos == NOT_FOUND)) {
1728 				if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))
1729 					dio->range.n_sectors = next_sector - dio->range.logical_sector;
1730 			} else {
1731 				unsigned i;
1732 				unsigned jp = journal_read_pos + 1;
1733 				for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {
1734 					if (!test_journal_node(ic, jp, dio->range.logical_sector + i))
1735 						break;
1736 				}
1737 				dio->range.n_sectors = i;
1738 			}
1739 		}
1740 	}
1741 	if (unlikely(!add_new_range(ic, &dio->range, true))) {
1742 		/*
1743 		 * We must not sleep in the request routine because it could
1744 		 * stall bios on current->bio_list.
1745 		 * So, we offload the bio to a workqueue if we have to sleep.
1746 		 */
1747 		if (from_map) {
1748 offload_to_thread:
1749 			spin_unlock_irq(&ic->endio_wait.lock);
1750 			INIT_WORK(&dio->work, integrity_bio_wait);
1751 			queue_work(ic->wait_wq, &dio->work);
1752 			return;
1753 		}
1754 		wait_and_add_new_range(ic, &dio->range);
1755 	}
1756 	spin_unlock_irq(&ic->endio_wait.lock);
1757 
1758 	if (unlikely(journal_read_pos != NOT_FOUND)) {
1759 		journal_section = journal_read_pos / ic->journal_section_entries;
1760 		journal_entry = journal_read_pos % ic->journal_section_entries;
1761 		goto journal_read_write;
1762 	}
1763 
1764 	dio->in_flight = (atomic_t)ATOMIC_INIT(2);
1765 
1766 	if (need_sync_io) {
1767 		init_completion(&read_comp);
1768 		dio->completion = &read_comp;
1769 	} else
1770 		dio->completion = NULL;
1771 
1772 	dio->orig_bi_iter = bio->bi_iter;
1773 
1774 	dio->orig_bi_disk = bio->bi_disk;
1775 	dio->orig_bi_partno = bio->bi_partno;
1776 	bio_set_dev(bio, ic->dev->bdev);
1777 
1778 	dio->orig_bi_integrity = bio_integrity(bio);
1779 	bio->bi_integrity = NULL;
1780 	bio->bi_opf &= ~REQ_INTEGRITY;
1781 
1782 	dio->orig_bi_end_io = bio->bi_end_io;
1783 	bio->bi_end_io = integrity_end_io;
1784 
1785 	bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
1786 	generic_make_request(bio);
1787 
1788 	if (need_sync_io) {
1789 		wait_for_completion_io(&read_comp);
1790 		if (unlikely(ic->recalc_wq != NULL) &&
1791 		    ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
1792 		    dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))
1793 			goto skip_check;
1794 		if (likely(!bio->bi_status))
1795 			integrity_metadata(&dio->work);
1796 		else
1797 skip_check:
1798 			dec_in_flight(dio);
1799 
1800 	} else {
1801 		INIT_WORK(&dio->work, integrity_metadata);
1802 		queue_work(ic->metadata_wq, &dio->work);
1803 	}
1804 
1805 	return;
1806 
1807 journal_read_write:
1808 	if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))
1809 		goto lock_retry;
1810 
1811 	do_endio_flush(ic, dio);
1812 }
1813 
1814 
1815 static void integrity_bio_wait(struct work_struct *w)
1816 {
1817 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1818 
1819 	dm_integrity_map_continue(dio, false);
1820 }
1821 
1822 static void pad_uncommitted(struct dm_integrity_c *ic)
1823 {
1824 	if (ic->free_section_entry) {
1825 		ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry;
1826 		ic->free_section_entry = 0;
1827 		ic->free_section++;
1828 		wraparound_section(ic, &ic->free_section);
1829 		ic->n_uncommitted_sections++;
1830 	}
1831 	WARN_ON(ic->journal_sections * ic->journal_section_entries !=
1832 		(ic->n_uncommitted_sections + ic->n_committed_sections) * ic->journal_section_entries + ic->free_sectors);
1833 }
1834 
1835 static void integrity_commit(struct work_struct *w)
1836 {
1837 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work);
1838 	unsigned commit_start, commit_sections;
1839 	unsigned i, j, n;
1840 	struct bio *flushes;
1841 
1842 	del_timer(&ic->autocommit_timer);
1843 
1844 	spin_lock_irq(&ic->endio_wait.lock);
1845 	flushes = bio_list_get(&ic->flush_bio_list);
1846 	if (unlikely(ic->mode != 'J')) {
1847 		spin_unlock_irq(&ic->endio_wait.lock);
1848 		dm_integrity_flush_buffers(ic);
1849 		goto release_flush_bios;
1850 	}
1851 
1852 	pad_uncommitted(ic);
1853 	commit_start = ic->uncommitted_section;
1854 	commit_sections = ic->n_uncommitted_sections;
1855 	spin_unlock_irq(&ic->endio_wait.lock);
1856 
1857 	if (!commit_sections)
1858 		goto release_flush_bios;
1859 
1860 	i = commit_start;
1861 	for (n = 0; n < commit_sections; n++) {
1862 		for (j = 0; j < ic->journal_section_entries; j++) {
1863 			struct journal_entry *je;
1864 			je = access_journal_entry(ic, i, j);
1865 			io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
1866 		}
1867 		for (j = 0; j < ic->journal_section_sectors; j++) {
1868 			struct journal_sector *js;
1869 			js = access_journal(ic, i, j);
1870 			js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq);
1871 		}
1872 		i++;
1873 		if (unlikely(i >= ic->journal_sections))
1874 			ic->commit_seq = next_commit_seq(ic->commit_seq);
1875 		wraparound_section(ic, &i);
1876 	}
1877 	smp_rmb();
1878 
1879 	write_journal(ic, commit_start, commit_sections);
1880 
1881 	spin_lock_irq(&ic->endio_wait.lock);
1882 	ic->uncommitted_section += commit_sections;
1883 	wraparound_section(ic, &ic->uncommitted_section);
1884 	ic->n_uncommitted_sections -= commit_sections;
1885 	ic->n_committed_sections += commit_sections;
1886 	spin_unlock_irq(&ic->endio_wait.lock);
1887 
1888 	if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold)
1889 		queue_work(ic->writer_wq, &ic->writer_work);
1890 
1891 release_flush_bios:
1892 	while (flushes) {
1893 		struct bio *next = flushes->bi_next;
1894 		flushes->bi_next = NULL;
1895 		do_endio(ic, flushes);
1896 		flushes = next;
1897 	}
1898 }
1899 
1900 static void complete_copy_from_journal(unsigned long error, void *context)
1901 {
1902 	struct journal_io *io = context;
1903 	struct journal_completion *comp = io->comp;
1904 	struct dm_integrity_c *ic = comp->ic;
1905 	remove_range(ic, &io->range);
1906 	mempool_free(io, &ic->journal_io_mempool);
1907 	if (unlikely(error != 0))
1908 		dm_integrity_io_error(ic, "copying from journal", -EIO);
1909 	complete_journal_op(comp);
1910 }
1911 
1912 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js,
1913 			       struct journal_entry *je)
1914 {
1915 	unsigned s = 0;
1916 	do {
1917 		js->commit_id = je->last_bytes[s];
1918 		js++;
1919 	} while (++s < ic->sectors_per_block);
1920 }
1921 
1922 static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
1923 			     unsigned write_sections, bool from_replay)
1924 {
1925 	unsigned i, j, n;
1926 	struct journal_completion comp;
1927 	struct blk_plug plug;
1928 
1929 	blk_start_plug(&plug);
1930 
1931 	comp.ic = ic;
1932 	comp.in_flight = (atomic_t)ATOMIC_INIT(1);
1933 	init_completion(&comp.comp);
1934 
1935 	i = write_start;
1936 	for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) {
1937 #ifndef INTERNAL_VERIFY
1938 		if (unlikely(from_replay))
1939 #endif
1940 			rw_section_mac(ic, i, false);
1941 		for (j = 0; j < ic->journal_section_entries; j++) {
1942 			struct journal_entry *je = access_journal_entry(ic, i, j);
1943 			sector_t sec, area, offset;
1944 			unsigned k, l, next_loop;
1945 			sector_t metadata_block;
1946 			unsigned metadata_offset;
1947 			struct journal_io *io;
1948 
1949 			if (journal_entry_is_unused(je))
1950 				continue;
1951 			BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay);
1952 			sec = journal_entry_get_sector(je);
1953 			if (unlikely(from_replay)) {
1954 				if (unlikely(sec & (unsigned)(ic->sectors_per_block - 1))) {
1955 					dm_integrity_io_error(ic, "invalid sector in journal", -EIO);
1956 					sec &= ~(sector_t)(ic->sectors_per_block - 1);
1957 				}
1958 			}
1959 			get_area_and_offset(ic, sec, &area, &offset);
1960 			restore_last_bytes(ic, access_journal_data(ic, i, j), je);
1961 			for (k = j + 1; k < ic->journal_section_entries; k++) {
1962 				struct journal_entry *je2 = access_journal_entry(ic, i, k);
1963 				sector_t sec2, area2, offset2;
1964 				if (journal_entry_is_unused(je2))
1965 					break;
1966 				BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay);
1967 				sec2 = journal_entry_get_sector(je2);
1968 				get_area_and_offset(ic, sec2, &area2, &offset2);
1969 				if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block))
1970 					break;
1971 				restore_last_bytes(ic, access_journal_data(ic, i, k), je2);
1972 			}
1973 			next_loop = k - 1;
1974 
1975 			io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO);
1976 			io->comp = &comp;
1977 			io->range.logical_sector = sec;
1978 			io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block;
1979 
1980 			spin_lock_irq(&ic->endio_wait.lock);
1981 			if (unlikely(!add_new_range(ic, &io->range, true)))
1982 				wait_and_add_new_range(ic, &io->range);
1983 
1984 			if (likely(!from_replay)) {
1985 				struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries];
1986 
1987 				/* don't write if there is newer committed sector */
1988 				while (j < k && find_newer_committed_node(ic, &section_node[j])) {
1989 					struct journal_entry *je2 = access_journal_entry(ic, i, j);
1990 
1991 					journal_entry_set_unused(je2);
1992 					remove_journal_node(ic, &section_node[j]);
1993 					j++;
1994 					sec += ic->sectors_per_block;
1995 					offset += ic->sectors_per_block;
1996 				}
1997 				while (j < k && find_newer_committed_node(ic, &section_node[k - 1])) {
1998 					struct journal_entry *je2 = access_journal_entry(ic, i, k - 1);
1999 
2000 					journal_entry_set_unused(je2);
2001 					remove_journal_node(ic, &section_node[k - 1]);
2002 					k--;
2003 				}
2004 				if (j == k) {
2005 					remove_range_unlocked(ic, &io->range);
2006 					spin_unlock_irq(&ic->endio_wait.lock);
2007 					mempool_free(io, &ic->journal_io_mempool);
2008 					goto skip_io;
2009 				}
2010 				for (l = j; l < k; l++) {
2011 					remove_journal_node(ic, &section_node[l]);
2012 				}
2013 			}
2014 			spin_unlock_irq(&ic->endio_wait.lock);
2015 
2016 			metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
2017 			for (l = j; l < k; l++) {
2018 				int r;
2019 				struct journal_entry *je2 = access_journal_entry(ic, i, l);
2020 
2021 				if (
2022 #ifndef INTERNAL_VERIFY
2023 				    unlikely(from_replay) &&
2024 #endif
2025 				    ic->internal_hash) {
2026 					char test_tag[max(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)];
2027 
2028 					integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block),
2029 								  (char *)access_journal_data(ic, i, l), test_tag);
2030 					if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size)))
2031 						dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ);
2032 				}
2033 
2034 				journal_entry_set_unused(je2);
2035 				r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset,
2036 							ic->tag_size, TAG_WRITE);
2037 				if (unlikely(r)) {
2038 					dm_integrity_io_error(ic, "reading tags", r);
2039 				}
2040 			}
2041 
2042 			atomic_inc(&comp.in_flight);
2043 			copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block,
2044 					  (k - j) << ic->sb->log2_sectors_per_block,
2045 					  get_data_sector(ic, area, offset),
2046 					  complete_copy_from_journal, io);
2047 skip_io:
2048 			j = next_loop;
2049 		}
2050 	}
2051 
2052 	dm_bufio_write_dirty_buffers_async(ic->bufio);
2053 
2054 	blk_finish_plug(&plug);
2055 
2056 	complete_journal_op(&comp);
2057 	wait_for_completion_io(&comp.comp);
2058 
2059 	dm_integrity_flush_buffers(ic);
2060 }
2061 
2062 static void integrity_writer(struct work_struct *w)
2063 {
2064 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work);
2065 	unsigned write_start, write_sections;
2066 
2067 	unsigned prev_free_sectors;
2068 
2069 	/* the following test is not needed, but it tests the replay code */
2070 	if (READ_ONCE(ic->suspending) && !ic->meta_dev)
2071 		return;
2072 
2073 	spin_lock_irq(&ic->endio_wait.lock);
2074 	write_start = ic->committed_section;
2075 	write_sections = ic->n_committed_sections;
2076 	spin_unlock_irq(&ic->endio_wait.lock);
2077 
2078 	if (!write_sections)
2079 		return;
2080 
2081 	do_journal_write(ic, write_start, write_sections, false);
2082 
2083 	spin_lock_irq(&ic->endio_wait.lock);
2084 
2085 	ic->committed_section += write_sections;
2086 	wraparound_section(ic, &ic->committed_section);
2087 	ic->n_committed_sections -= write_sections;
2088 
2089 	prev_free_sectors = ic->free_sectors;
2090 	ic->free_sectors += write_sections * ic->journal_section_entries;
2091 	if (unlikely(!prev_free_sectors))
2092 		wake_up_locked(&ic->endio_wait);
2093 
2094 	spin_unlock_irq(&ic->endio_wait.lock);
2095 }
2096 
2097 static void recalc_write_super(struct dm_integrity_c *ic)
2098 {
2099 	int r;
2100 
2101 	dm_integrity_flush_buffers(ic);
2102 	if (dm_integrity_failed(ic))
2103 		return;
2104 
2105 	sb_set_version(ic);
2106 	r = sync_rw_sb(ic, REQ_OP_WRITE, 0);
2107 	if (unlikely(r))
2108 		dm_integrity_io_error(ic, "writing superblock", r);
2109 }
2110 
2111 static void integrity_recalc(struct work_struct *w)
2112 {
2113 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work);
2114 	struct dm_integrity_range range;
2115 	struct dm_io_request io_req;
2116 	struct dm_io_region io_loc;
2117 	sector_t area, offset;
2118 	sector_t metadata_block;
2119 	unsigned metadata_offset;
2120 	__u8 *t;
2121 	unsigned i;
2122 	int r;
2123 	unsigned super_counter = 0;
2124 
2125 	spin_lock_irq(&ic->endio_wait.lock);
2126 
2127 next_chunk:
2128 
2129 	if (unlikely(READ_ONCE(ic->suspending)))
2130 		goto unlock_ret;
2131 
2132 	range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
2133 	if (unlikely(range.logical_sector >= ic->provided_data_sectors))
2134 		goto unlock_ret;
2135 
2136 	get_area_and_offset(ic, range.logical_sector, &area, &offset);
2137 	range.n_sectors = min((sector_t)RECALC_SECTORS, ic->provided_data_sectors - range.logical_sector);
2138 	if (!ic->meta_dev)
2139 		range.n_sectors = min(range.n_sectors, (1U << ic->sb->log2_interleave_sectors) - (unsigned)offset);
2140 
2141 	if (unlikely(!add_new_range(ic, &range, true)))
2142 		wait_and_add_new_range(ic, &range);
2143 
2144 	spin_unlock_irq(&ic->endio_wait.lock);
2145 
2146 	if (unlikely(++super_counter == RECALC_WRITE_SUPER)) {
2147 		recalc_write_super(ic);
2148 		super_counter = 0;
2149 	}
2150 
2151 	if (unlikely(dm_integrity_failed(ic)))
2152 		goto err;
2153 
2154 	io_req.bi_op = REQ_OP_READ;
2155 	io_req.bi_op_flags = 0;
2156 	io_req.mem.type = DM_IO_VMA;
2157 	io_req.mem.ptr.addr = ic->recalc_buffer;
2158 	io_req.notify.fn = NULL;
2159 	io_req.client = ic->io;
2160 	io_loc.bdev = ic->dev->bdev;
2161 	io_loc.sector = get_data_sector(ic, area, offset);
2162 	io_loc.count = range.n_sectors;
2163 
2164 	r = dm_io(&io_req, 1, &io_loc, NULL);
2165 	if (unlikely(r)) {
2166 		dm_integrity_io_error(ic, "reading data", r);
2167 		goto err;
2168 	}
2169 
2170 	t = ic->recalc_tags;
2171 	for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) {
2172 		integrity_sector_checksum(ic, range.logical_sector + i, ic->recalc_buffer + (i << SECTOR_SHIFT), t);
2173 		t += ic->tag_size;
2174 	}
2175 
2176 	metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
2177 
2178 	r = dm_integrity_rw_tag(ic, ic->recalc_tags, &metadata_block, &metadata_offset, t - ic->recalc_tags, TAG_WRITE);
2179 	if (unlikely(r)) {
2180 		dm_integrity_io_error(ic, "writing tags", r);
2181 		goto err;
2182 	}
2183 
2184 	spin_lock_irq(&ic->endio_wait.lock);
2185 	remove_range_unlocked(ic, &range);
2186 	ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors);
2187 	goto next_chunk;
2188 
2189 err:
2190 	remove_range(ic, &range);
2191 	return;
2192 
2193 unlock_ret:
2194 	spin_unlock_irq(&ic->endio_wait.lock);
2195 
2196 	recalc_write_super(ic);
2197 }
2198 
2199 static void init_journal(struct dm_integrity_c *ic, unsigned start_section,
2200 			 unsigned n_sections, unsigned char commit_seq)
2201 {
2202 	unsigned i, j, n;
2203 
2204 	if (!n_sections)
2205 		return;
2206 
2207 	for (n = 0; n < n_sections; n++) {
2208 		i = start_section + n;
2209 		wraparound_section(ic, &i);
2210 		for (j = 0; j < ic->journal_section_sectors; j++) {
2211 			struct journal_sector *js = access_journal(ic, i, j);
2212 			memset(&js->entries, 0, JOURNAL_SECTOR_DATA);
2213 			js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq);
2214 		}
2215 		for (j = 0; j < ic->journal_section_entries; j++) {
2216 			struct journal_entry *je = access_journal_entry(ic, i, j);
2217 			journal_entry_set_unused(je);
2218 		}
2219 	}
2220 
2221 	write_journal(ic, start_section, n_sections);
2222 }
2223 
2224 static int find_commit_seq(struct dm_integrity_c *ic, unsigned i, unsigned j, commit_id_t id)
2225 {
2226 	unsigned char k;
2227 	for (k = 0; k < N_COMMIT_IDS; k++) {
2228 		if (dm_integrity_commit_id(ic, i, j, k) == id)
2229 			return k;
2230 	}
2231 	dm_integrity_io_error(ic, "journal commit id", -EIO);
2232 	return -EIO;
2233 }
2234 
2235 static void replay_journal(struct dm_integrity_c *ic)
2236 {
2237 	unsigned i, j;
2238 	bool used_commit_ids[N_COMMIT_IDS];
2239 	unsigned max_commit_id_sections[N_COMMIT_IDS];
2240 	unsigned write_start, write_sections;
2241 	unsigned continue_section;
2242 	bool journal_empty;
2243 	unsigned char unused, last_used, want_commit_seq;
2244 
2245 	if (ic->mode == 'R')
2246 		return;
2247 
2248 	if (ic->journal_uptodate)
2249 		return;
2250 
2251 	last_used = 0;
2252 	write_start = 0;
2253 
2254 	if (!ic->just_formatted) {
2255 		DEBUG_print("reading journal\n");
2256 		rw_journal(ic, REQ_OP_READ, 0, 0, ic->journal_sections, NULL);
2257 		if (ic->journal_io)
2258 			DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal");
2259 		if (ic->journal_io) {
2260 			struct journal_completion crypt_comp;
2261 			crypt_comp.ic = ic;
2262 			init_completion(&crypt_comp.comp);
2263 			crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0);
2264 			encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp);
2265 			wait_for_completion(&crypt_comp.comp);
2266 		}
2267 		DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal");
2268 	}
2269 
2270 	if (dm_integrity_failed(ic))
2271 		goto clear_journal;
2272 
2273 	journal_empty = true;
2274 	memset(used_commit_ids, 0, sizeof used_commit_ids);
2275 	memset(max_commit_id_sections, 0, sizeof max_commit_id_sections);
2276 	for (i = 0; i < ic->journal_sections; i++) {
2277 		for (j = 0; j < ic->journal_section_sectors; j++) {
2278 			int k;
2279 			struct journal_sector *js = access_journal(ic, i, j);
2280 			k = find_commit_seq(ic, i, j, js->commit_id);
2281 			if (k < 0)
2282 				goto clear_journal;
2283 			used_commit_ids[k] = true;
2284 			max_commit_id_sections[k] = i;
2285 		}
2286 		if (journal_empty) {
2287 			for (j = 0; j < ic->journal_section_entries; j++) {
2288 				struct journal_entry *je = access_journal_entry(ic, i, j);
2289 				if (!journal_entry_is_unused(je)) {
2290 					journal_empty = false;
2291 					break;
2292 				}
2293 			}
2294 		}
2295 	}
2296 
2297 	if (!used_commit_ids[N_COMMIT_IDS - 1]) {
2298 		unused = N_COMMIT_IDS - 1;
2299 		while (unused && !used_commit_ids[unused - 1])
2300 			unused--;
2301 	} else {
2302 		for (unused = 0; unused < N_COMMIT_IDS; unused++)
2303 			if (!used_commit_ids[unused])
2304 				break;
2305 		if (unused == N_COMMIT_IDS) {
2306 			dm_integrity_io_error(ic, "journal commit ids", -EIO);
2307 			goto clear_journal;
2308 		}
2309 	}
2310 	DEBUG_print("first unused commit seq %d [%d,%d,%d,%d]\n",
2311 		    unused, used_commit_ids[0], used_commit_ids[1],
2312 		    used_commit_ids[2], used_commit_ids[3]);
2313 
2314 	last_used = prev_commit_seq(unused);
2315 	want_commit_seq = prev_commit_seq(last_used);
2316 
2317 	if (!used_commit_ids[want_commit_seq] && used_commit_ids[prev_commit_seq(want_commit_seq)])
2318 		journal_empty = true;
2319 
2320 	write_start = max_commit_id_sections[last_used] + 1;
2321 	if (unlikely(write_start >= ic->journal_sections))
2322 		want_commit_seq = next_commit_seq(want_commit_seq);
2323 	wraparound_section(ic, &write_start);
2324 
2325 	i = write_start;
2326 	for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) {
2327 		for (j = 0; j < ic->journal_section_sectors; j++) {
2328 			struct journal_sector *js = access_journal(ic, i, j);
2329 
2330 			if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) {
2331 				/*
2332 				 * This could be caused by crash during writing.
2333 				 * We won't replay the inconsistent part of the
2334 				 * journal.
2335 				 */
2336 				DEBUG_print("commit id mismatch at position (%u, %u): %d != %d\n",
2337 					    i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq);
2338 				goto brk;
2339 			}
2340 		}
2341 		i++;
2342 		if (unlikely(i >= ic->journal_sections))
2343 			want_commit_seq = next_commit_seq(want_commit_seq);
2344 		wraparound_section(ic, &i);
2345 	}
2346 brk:
2347 
2348 	if (!journal_empty) {
2349 		DEBUG_print("replaying %u sections, starting at %u, commit seq %d\n",
2350 			    write_sections, write_start, want_commit_seq);
2351 		do_journal_write(ic, write_start, write_sections, true);
2352 	}
2353 
2354 	if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) {
2355 		continue_section = write_start;
2356 		ic->commit_seq = want_commit_seq;
2357 		DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq);
2358 	} else {
2359 		unsigned s;
2360 		unsigned char erase_seq;
2361 clear_journal:
2362 		DEBUG_print("clearing journal\n");
2363 
2364 		erase_seq = prev_commit_seq(prev_commit_seq(last_used));
2365 		s = write_start;
2366 		init_journal(ic, s, 1, erase_seq);
2367 		s++;
2368 		wraparound_section(ic, &s);
2369 		if (ic->journal_sections >= 2) {
2370 			init_journal(ic, s, ic->journal_sections - 2, erase_seq);
2371 			s += ic->journal_sections - 2;
2372 			wraparound_section(ic, &s);
2373 			init_journal(ic, s, 1, erase_seq);
2374 		}
2375 
2376 		continue_section = 0;
2377 		ic->commit_seq = next_commit_seq(erase_seq);
2378 	}
2379 
2380 	ic->committed_section = continue_section;
2381 	ic->n_committed_sections = 0;
2382 
2383 	ic->uncommitted_section = continue_section;
2384 	ic->n_uncommitted_sections = 0;
2385 
2386 	ic->free_section = continue_section;
2387 	ic->free_section_entry = 0;
2388 	ic->free_sectors = ic->journal_entries;
2389 
2390 	ic->journal_tree_root = RB_ROOT;
2391 	for (i = 0; i < ic->journal_entries; i++)
2392 		init_journal_node(&ic->journal_tree[i]);
2393 }
2394 
2395 static void dm_integrity_postsuspend(struct dm_target *ti)
2396 {
2397 	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2398 
2399 	del_timer_sync(&ic->autocommit_timer);
2400 
2401 	WRITE_ONCE(ic->suspending, 1);
2402 
2403 	if (ic->recalc_wq)
2404 		drain_workqueue(ic->recalc_wq);
2405 
2406 	queue_work(ic->commit_wq, &ic->commit_work);
2407 	drain_workqueue(ic->commit_wq);
2408 
2409 	if (ic->mode == 'J') {
2410 		if (ic->meta_dev)
2411 			queue_work(ic->writer_wq, &ic->writer_work);
2412 		drain_workqueue(ic->writer_wq);
2413 		dm_integrity_flush_buffers(ic);
2414 	}
2415 
2416 	WRITE_ONCE(ic->suspending, 0);
2417 
2418 	BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
2419 
2420 	ic->journal_uptodate = true;
2421 }
2422 
2423 static void dm_integrity_resume(struct dm_target *ti)
2424 {
2425 	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2426 
2427 	replay_journal(ic);
2428 
2429 	if (ic->recalc_wq && ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
2430 		__u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector);
2431 		if (recalc_pos < ic->provided_data_sectors) {
2432 			queue_work(ic->recalc_wq, &ic->recalc_work);
2433 		} else if (recalc_pos > ic->provided_data_sectors) {
2434 			ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors);
2435 			recalc_write_super(ic);
2436 		}
2437 	}
2438 }
2439 
2440 static void dm_integrity_status(struct dm_target *ti, status_type_t type,
2441 				unsigned status_flags, char *result, unsigned maxlen)
2442 {
2443 	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2444 	unsigned arg_count;
2445 	size_t sz = 0;
2446 
2447 	switch (type) {
2448 	case STATUSTYPE_INFO:
2449 		DMEMIT("%llu %llu",
2450 			(unsigned long long)atomic64_read(&ic->number_of_mismatches),
2451 			(unsigned long long)ic->provided_data_sectors);
2452 		if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
2453 			DMEMIT(" %llu", (unsigned long long)le64_to_cpu(ic->sb->recalc_sector));
2454 		else
2455 			DMEMIT(" -");
2456 		break;
2457 
2458 	case STATUSTYPE_TABLE: {
2459 		__u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100;
2460 		watermark_percentage += ic->journal_entries / 2;
2461 		do_div(watermark_percentage, ic->journal_entries);
2462 		arg_count = 5;
2463 		arg_count += !!ic->meta_dev;
2464 		arg_count += ic->sectors_per_block != 1;
2465 		arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING));
2466 		arg_count += !!ic->internal_hash_alg.alg_string;
2467 		arg_count += !!ic->journal_crypt_alg.alg_string;
2468 		arg_count += !!ic->journal_mac_alg.alg_string;
2469 		DMEMIT("%s %llu %u %c %u", ic->dev->name, (unsigned long long)ic->start,
2470 		       ic->tag_size, ic->mode, arg_count);
2471 		if (ic->meta_dev)
2472 			DMEMIT(" meta_device:%s", ic->meta_dev->name);
2473 		if (ic->sectors_per_block != 1)
2474 			DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT);
2475 		if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
2476 			DMEMIT(" recalculate");
2477 		DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS);
2478 		DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors);
2479 		DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
2480 		DMEMIT(" journal_watermark:%u", (unsigned)watermark_percentage);
2481 		DMEMIT(" commit_time:%u", ic->autocommit_msec);
2482 
2483 #define EMIT_ALG(a, n)							\
2484 		do {							\
2485 			if (ic->a.alg_string) {				\
2486 				DMEMIT(" %s:%s", n, ic->a.alg_string);	\
2487 				if (ic->a.key_string)			\
2488 					DMEMIT(":%s", ic->a.key_string);\
2489 			}						\
2490 		} while (0)
2491 		EMIT_ALG(internal_hash_alg, "internal_hash");
2492 		EMIT_ALG(journal_crypt_alg, "journal_crypt");
2493 		EMIT_ALG(journal_mac_alg, "journal_mac");
2494 		break;
2495 	}
2496 	}
2497 }
2498 
2499 static int dm_integrity_iterate_devices(struct dm_target *ti,
2500 					iterate_devices_callout_fn fn, void *data)
2501 {
2502 	struct dm_integrity_c *ic = ti->private;
2503 
2504 	if (!ic->meta_dev)
2505 		return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data);
2506 	else
2507 		return fn(ti, ic->dev, 0, ti->len, data);
2508 }
2509 
2510 static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits)
2511 {
2512 	struct dm_integrity_c *ic = ti->private;
2513 
2514 	if (ic->sectors_per_block > 1) {
2515 		limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
2516 		limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
2517 		blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT);
2518 	}
2519 }
2520 
2521 static void calculate_journal_section_size(struct dm_integrity_c *ic)
2522 {
2523 	unsigned sector_space = JOURNAL_SECTOR_DATA;
2524 
2525 	ic->journal_sections = le32_to_cpu(ic->sb->journal_sections);
2526 	ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size,
2527 					 JOURNAL_ENTRY_ROUNDUP);
2528 
2529 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC))
2530 		sector_space -= JOURNAL_MAC_PER_SECTOR;
2531 	ic->journal_entries_per_sector = sector_space / ic->journal_entry_size;
2532 	ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS;
2533 	ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JOURNAL_BLOCK_SECTORS;
2534 	ic->journal_entries = ic->journal_section_entries * ic->journal_sections;
2535 }
2536 
2537 static int calculate_device_limits(struct dm_integrity_c *ic)
2538 {
2539 	__u64 initial_sectors;
2540 
2541 	calculate_journal_section_size(ic);
2542 	initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections;
2543 	if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UINT_MAX)
2544 		return -EINVAL;
2545 	ic->initial_sectors = initial_sectors;
2546 
2547 	if (!ic->meta_dev) {
2548 		sector_t last_sector, last_area, last_offset;
2549 
2550 		ic->metadata_run = roundup((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block),
2551 					   (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS)) >> SECTOR_SHIFT;
2552 		if (!(ic->metadata_run & (ic->metadata_run - 1)))
2553 			ic->log2_metadata_run = __ffs(ic->metadata_run);
2554 		else
2555 			ic->log2_metadata_run = -1;
2556 
2557 		get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset);
2558 		last_sector = get_data_sector(ic, last_area, last_offset);
2559 		if (last_sector < ic->start || last_sector >= ic->meta_device_sectors)
2560 			return -EINVAL;
2561 	} else {
2562 		__u64 meta_size = ic->provided_data_sectors * ic->tag_size;
2563 		meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1))
2564 				>> (ic->log2_buffer_sectors + SECTOR_SHIFT);
2565 		meta_size <<= ic->log2_buffer_sectors;
2566 		if (ic->initial_sectors + meta_size < ic->initial_sectors ||
2567 		    ic->initial_sectors + meta_size > ic->meta_device_sectors)
2568 			return -EINVAL;
2569 		ic->metadata_run = 1;
2570 		ic->log2_metadata_run = 0;
2571 	}
2572 
2573 	return 0;
2574 }
2575 
2576 static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sectors, unsigned interleave_sectors)
2577 {
2578 	unsigned journal_sections;
2579 	int test_bit;
2580 
2581 	memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT);
2582 	memcpy(ic->sb->magic, SB_MAGIC, 8);
2583 	ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size);
2584 	ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block);
2585 	if (ic->journal_mac_alg.alg_string)
2586 		ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC);
2587 
2588 	calculate_journal_section_size(ic);
2589 	journal_sections = journal_sectors / ic->journal_section_sectors;
2590 	if (!journal_sections)
2591 		journal_sections = 1;
2592 
2593 	if (!ic->meta_dev) {
2594 		ic->sb->journal_sections = cpu_to_le32(journal_sections);
2595 		if (!interleave_sectors)
2596 			interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
2597 		ic->sb->log2_interleave_sectors = __fls(interleave_sectors);
2598 		ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
2599 		ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
2600 
2601 		ic->provided_data_sectors = 0;
2602 		for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) {
2603 			__u64 prev_data_sectors = ic->provided_data_sectors;
2604 
2605 			ic->provided_data_sectors |= (sector_t)1 << test_bit;
2606 			if (calculate_device_limits(ic))
2607 				ic->provided_data_sectors = prev_data_sectors;
2608 		}
2609 		if (!ic->provided_data_sectors)
2610 			return -EINVAL;
2611 	} else {
2612 		ic->sb->log2_interleave_sectors = 0;
2613 		ic->provided_data_sectors = ic->data_device_sectors;
2614 		ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1);
2615 
2616 try_smaller_buffer:
2617 		ic->sb->journal_sections = cpu_to_le32(0);
2618 		for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) {
2619 			__u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections);
2620 			__u32 test_journal_sections = prev_journal_sections | (1U << test_bit);
2621 			if (test_journal_sections > journal_sections)
2622 				continue;
2623 			ic->sb->journal_sections = cpu_to_le32(test_journal_sections);
2624 			if (calculate_device_limits(ic))
2625 				ic->sb->journal_sections = cpu_to_le32(prev_journal_sections);
2626 
2627 		}
2628 		if (!le32_to_cpu(ic->sb->journal_sections)) {
2629 			if (ic->log2_buffer_sectors > 3) {
2630 				ic->log2_buffer_sectors--;
2631 				goto try_smaller_buffer;
2632 			}
2633 			return -EINVAL;
2634 		}
2635 	}
2636 
2637 	ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
2638 
2639 	sb_set_version(ic);
2640 
2641 	return 0;
2642 }
2643 
2644 static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic)
2645 {
2646 	struct gendisk *disk = dm_disk(dm_table_get_md(ti->table));
2647 	struct blk_integrity bi;
2648 
2649 	memset(&bi, 0, sizeof(bi));
2650 	bi.profile = &dm_integrity_profile;
2651 	bi.tuple_size = ic->tag_size;
2652 	bi.tag_size = bi.tuple_size;
2653 	bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT;
2654 
2655 	blk_integrity_register(disk, &bi);
2656 	blk_queue_max_integrity_segments(disk->queue, UINT_MAX);
2657 }
2658 
2659 static void dm_integrity_free_page_list(struct dm_integrity_c *ic, struct page_list *pl)
2660 {
2661 	unsigned i;
2662 
2663 	if (!pl)
2664 		return;
2665 	for (i = 0; i < ic->journal_pages; i++)
2666 		if (pl[i].page)
2667 			__free_page(pl[i].page);
2668 	kvfree(pl);
2669 }
2670 
2671 static struct page_list *dm_integrity_alloc_page_list(struct dm_integrity_c *ic)
2672 {
2673 	size_t page_list_desc_size = ic->journal_pages * sizeof(struct page_list);
2674 	struct page_list *pl;
2675 	unsigned i;
2676 
2677 	pl = kvmalloc(page_list_desc_size, GFP_KERNEL | __GFP_ZERO);
2678 	if (!pl)
2679 		return NULL;
2680 
2681 	for (i = 0; i < ic->journal_pages; i++) {
2682 		pl[i].page = alloc_page(GFP_KERNEL);
2683 		if (!pl[i].page) {
2684 			dm_integrity_free_page_list(ic, pl);
2685 			return NULL;
2686 		}
2687 		if (i)
2688 			pl[i - 1].next = &pl[i];
2689 	}
2690 
2691 	return pl;
2692 }
2693 
2694 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl)
2695 {
2696 	unsigned i;
2697 	for (i = 0; i < ic->journal_sections; i++)
2698 		kvfree(sl[i]);
2699 	kvfree(sl);
2700 }
2701 
2702 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, struct page_list *pl)
2703 {
2704 	struct scatterlist **sl;
2705 	unsigned i;
2706 
2707 	sl = kvmalloc_array(ic->journal_sections,
2708 			    sizeof(struct scatterlist *),
2709 			    GFP_KERNEL | __GFP_ZERO);
2710 	if (!sl)
2711 		return NULL;
2712 
2713 	for (i = 0; i < ic->journal_sections; i++) {
2714 		struct scatterlist *s;
2715 		unsigned start_index, start_offset;
2716 		unsigned end_index, end_offset;
2717 		unsigned n_pages;
2718 		unsigned idx;
2719 
2720 		page_list_location(ic, i, 0, &start_index, &start_offset);
2721 		page_list_location(ic, i, ic->journal_section_sectors - 1, &end_index, &end_offset);
2722 
2723 		n_pages = (end_index - start_index + 1);
2724 
2725 		s = kvmalloc_array(n_pages, sizeof(struct scatterlist),
2726 				   GFP_KERNEL);
2727 		if (!s) {
2728 			dm_integrity_free_journal_scatterlist(ic, sl);
2729 			return NULL;
2730 		}
2731 
2732 		sg_init_table(s, n_pages);
2733 		for (idx = start_index; idx <= end_index; idx++) {
2734 			char *va = lowmem_page_address(pl[idx].page);
2735 			unsigned start = 0, end = PAGE_SIZE;
2736 			if (idx == start_index)
2737 				start = start_offset;
2738 			if (idx == end_index)
2739 				end = end_offset + (1 << SECTOR_SHIFT);
2740 			sg_set_buf(&s[idx - start_index], va + start, end - start);
2741 		}
2742 
2743 		sl[i] = s;
2744 	}
2745 
2746 	return sl;
2747 }
2748 
2749 static void free_alg(struct alg_spec *a)
2750 {
2751 	kzfree(a->alg_string);
2752 	kzfree(a->key);
2753 	memset(a, 0, sizeof *a);
2754 }
2755 
2756 static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval)
2757 {
2758 	char *k;
2759 
2760 	free_alg(a);
2761 
2762 	a->alg_string = kstrdup(strchr(arg, ':') + 1, GFP_KERNEL);
2763 	if (!a->alg_string)
2764 		goto nomem;
2765 
2766 	k = strchr(a->alg_string, ':');
2767 	if (k) {
2768 		*k = 0;
2769 		a->key_string = k + 1;
2770 		if (strlen(a->key_string) & 1)
2771 			goto inval;
2772 
2773 		a->key_size = strlen(a->key_string) / 2;
2774 		a->key = kmalloc(a->key_size, GFP_KERNEL);
2775 		if (!a->key)
2776 			goto nomem;
2777 		if (hex2bin(a->key, a->key_string, a->key_size))
2778 			goto inval;
2779 	}
2780 
2781 	return 0;
2782 inval:
2783 	*error = error_inval;
2784 	return -EINVAL;
2785 nomem:
2786 	*error = "Out of memory for an argument";
2787 	return -ENOMEM;
2788 }
2789 
2790 static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error,
2791 		   char *error_alg, char *error_key)
2792 {
2793 	int r;
2794 
2795 	if (a->alg_string) {
2796 		*hash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ASYNC);
2797 		if (IS_ERR(*hash)) {
2798 			*error = error_alg;
2799 			r = PTR_ERR(*hash);
2800 			*hash = NULL;
2801 			return r;
2802 		}
2803 
2804 		if (a->key) {
2805 			r = crypto_shash_setkey(*hash, a->key, a->key_size);
2806 			if (r) {
2807 				*error = error_key;
2808 				return r;
2809 			}
2810 		} else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) {
2811 			*error = error_key;
2812 			return -ENOKEY;
2813 		}
2814 	}
2815 
2816 	return 0;
2817 }
2818 
2819 static int create_journal(struct dm_integrity_c *ic, char **error)
2820 {
2821 	int r = 0;
2822 	unsigned i;
2823 	__u64 journal_pages, journal_desc_size, journal_tree_size;
2824 	unsigned char *crypt_data = NULL, *crypt_iv = NULL;
2825 	struct skcipher_request *req = NULL;
2826 
2827 	ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL);
2828 	ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL);
2829 	ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL);
2830 	ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL);
2831 
2832 	journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors,
2833 				PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT);
2834 	journal_desc_size = journal_pages * sizeof(struct page_list);
2835 	if (journal_pages >= totalram_pages - totalhigh_pages || journal_desc_size > ULONG_MAX) {
2836 		*error = "Journal doesn't fit into memory";
2837 		r = -ENOMEM;
2838 		goto bad;
2839 	}
2840 	ic->journal_pages = journal_pages;
2841 
2842 	ic->journal = dm_integrity_alloc_page_list(ic);
2843 	if (!ic->journal) {
2844 		*error = "Could not allocate memory for journal";
2845 		r = -ENOMEM;
2846 		goto bad;
2847 	}
2848 	if (ic->journal_crypt_alg.alg_string) {
2849 		unsigned ivsize, blocksize;
2850 		struct journal_completion comp;
2851 
2852 		comp.ic = ic;
2853 		ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, 0);
2854 		if (IS_ERR(ic->journal_crypt)) {
2855 			*error = "Invalid journal cipher";
2856 			r = PTR_ERR(ic->journal_crypt);
2857 			ic->journal_crypt = NULL;
2858 			goto bad;
2859 		}
2860 		ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
2861 		blocksize = crypto_skcipher_blocksize(ic->journal_crypt);
2862 
2863 		if (ic->journal_crypt_alg.key) {
2864 			r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key,
2865 						   ic->journal_crypt_alg.key_size);
2866 			if (r) {
2867 				*error = "Error setting encryption key";
2868 				goto bad;
2869 			}
2870 		}
2871 		DEBUG_print("cipher %s, block size %u iv size %u\n",
2872 			    ic->journal_crypt_alg.alg_string, blocksize, ivsize);
2873 
2874 		ic->journal_io = dm_integrity_alloc_page_list(ic);
2875 		if (!ic->journal_io) {
2876 			*error = "Could not allocate memory for journal io";
2877 			r = -ENOMEM;
2878 			goto bad;
2879 		}
2880 
2881 		if (blocksize == 1) {
2882 			struct scatterlist *sg;
2883 
2884 			req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
2885 			if (!req) {
2886 				*error = "Could not allocate crypt request";
2887 				r = -ENOMEM;
2888 				goto bad;
2889 			}
2890 
2891 			crypt_iv = kmalloc(ivsize, GFP_KERNEL);
2892 			if (!crypt_iv) {
2893 				*error = "Could not allocate iv";
2894 				r = -ENOMEM;
2895 				goto bad;
2896 			}
2897 
2898 			ic->journal_xor = dm_integrity_alloc_page_list(ic);
2899 			if (!ic->journal_xor) {
2900 				*error = "Could not allocate memory for journal xor";
2901 				r = -ENOMEM;
2902 				goto bad;
2903 			}
2904 
2905 			sg = kvmalloc_array(ic->journal_pages + 1,
2906 					    sizeof(struct scatterlist),
2907 					    GFP_KERNEL);
2908 			if (!sg) {
2909 				*error = "Unable to allocate sg list";
2910 				r = -ENOMEM;
2911 				goto bad;
2912 			}
2913 			sg_init_table(sg, ic->journal_pages + 1);
2914 			for (i = 0; i < ic->journal_pages; i++) {
2915 				char *va = lowmem_page_address(ic->journal_xor[i].page);
2916 				clear_page(va);
2917 				sg_set_buf(&sg[i], va, PAGE_SIZE);
2918 			}
2919 			sg_set_buf(&sg[i], &ic->commit_ids, sizeof ic->commit_ids);
2920 			memset(crypt_iv, 0x00, ivsize);
2921 
2922 			skcipher_request_set_crypt(req, sg, sg, PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, crypt_iv);
2923 			init_completion(&comp.comp);
2924 			comp.in_flight = (atomic_t)ATOMIC_INIT(1);
2925 			if (do_crypt(true, req, &comp))
2926 				wait_for_completion(&comp.comp);
2927 			kvfree(sg);
2928 			r = dm_integrity_failed(ic);
2929 			if (r) {
2930 				*error = "Unable to encrypt journal";
2931 				goto bad;
2932 			}
2933 			DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data");
2934 
2935 			crypto_free_skcipher(ic->journal_crypt);
2936 			ic->journal_crypt = NULL;
2937 		} else {
2938 			unsigned crypt_len = roundup(ivsize, blocksize);
2939 
2940 			req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
2941 			if (!req) {
2942 				*error = "Could not allocate crypt request";
2943 				r = -ENOMEM;
2944 				goto bad;
2945 			}
2946 
2947 			crypt_iv = kmalloc(ivsize, GFP_KERNEL);
2948 			if (!crypt_iv) {
2949 				*error = "Could not allocate iv";
2950 				r = -ENOMEM;
2951 				goto bad;
2952 			}
2953 
2954 			crypt_data = kmalloc(crypt_len, GFP_KERNEL);
2955 			if (!crypt_data) {
2956 				*error = "Unable to allocate crypt data";
2957 				r = -ENOMEM;
2958 				goto bad;
2959 			}
2960 
2961 			ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal);
2962 			if (!ic->journal_scatterlist) {
2963 				*error = "Unable to allocate sg list";
2964 				r = -ENOMEM;
2965 				goto bad;
2966 			}
2967 			ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io);
2968 			if (!ic->journal_io_scatterlist) {
2969 				*error = "Unable to allocate sg list";
2970 				r = -ENOMEM;
2971 				goto bad;
2972 			}
2973 			ic->sk_requests = kvmalloc_array(ic->journal_sections,
2974 							 sizeof(struct skcipher_request *),
2975 							 GFP_KERNEL | __GFP_ZERO);
2976 			if (!ic->sk_requests) {
2977 				*error = "Unable to allocate sk requests";
2978 				r = -ENOMEM;
2979 				goto bad;
2980 			}
2981 			for (i = 0; i < ic->journal_sections; i++) {
2982 				struct scatterlist sg;
2983 				struct skcipher_request *section_req;
2984 				__u32 section_le = cpu_to_le32(i);
2985 
2986 				memset(crypt_iv, 0x00, ivsize);
2987 				memset(crypt_data, 0x00, crypt_len);
2988 				memcpy(crypt_data, &section_le, min((size_t)crypt_len, sizeof(section_le)));
2989 
2990 				sg_init_one(&sg, crypt_data, crypt_len);
2991 				skcipher_request_set_crypt(req, &sg, &sg, crypt_len, crypt_iv);
2992 				init_completion(&comp.comp);
2993 				comp.in_flight = (atomic_t)ATOMIC_INIT(1);
2994 				if (do_crypt(true, req, &comp))
2995 					wait_for_completion(&comp.comp);
2996 
2997 				r = dm_integrity_failed(ic);
2998 				if (r) {
2999 					*error = "Unable to generate iv";
3000 					goto bad;
3001 				}
3002 
3003 				section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
3004 				if (!section_req) {
3005 					*error = "Unable to allocate crypt request";
3006 					r = -ENOMEM;
3007 					goto bad;
3008 				}
3009 				section_req->iv = kmalloc_array(ivsize, 2,
3010 								GFP_KERNEL);
3011 				if (!section_req->iv) {
3012 					skcipher_request_free(section_req);
3013 					*error = "Unable to allocate iv";
3014 					r = -ENOMEM;
3015 					goto bad;
3016 				}
3017 				memcpy(section_req->iv + ivsize, crypt_data, ivsize);
3018 				section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT;
3019 				ic->sk_requests[i] = section_req;
3020 				DEBUG_bytes(crypt_data, ivsize, "iv(%u)", i);
3021 			}
3022 		}
3023 	}
3024 
3025 	for (i = 0; i < N_COMMIT_IDS; i++) {
3026 		unsigned j;
3027 retest_commit_id:
3028 		for (j = 0; j < i; j++) {
3029 			if (ic->commit_ids[j] == ic->commit_ids[i]) {
3030 				ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1);
3031 				goto retest_commit_id;
3032 			}
3033 		}
3034 		DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]);
3035 	}
3036 
3037 	journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node);
3038 	if (journal_tree_size > ULONG_MAX) {
3039 		*error = "Journal doesn't fit into memory";
3040 		r = -ENOMEM;
3041 		goto bad;
3042 	}
3043 	ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL);
3044 	if (!ic->journal_tree) {
3045 		*error = "Could not allocate memory for journal tree";
3046 		r = -ENOMEM;
3047 	}
3048 bad:
3049 	kfree(crypt_data);
3050 	kfree(crypt_iv);
3051 	skcipher_request_free(req);
3052 
3053 	return r;
3054 }
3055 
3056 /*
3057  * Construct a integrity mapping
3058  *
3059  * Arguments:
3060  *	device
3061  *	offset from the start of the device
3062  *	tag size
3063  *	D - direct writes, J - journal writes, R - recovery mode
3064  *	number of optional arguments
3065  *	optional arguments:
3066  *		journal_sectors
3067  *		interleave_sectors
3068  *		buffer_sectors
3069  *		journal_watermark
3070  *		commit_time
3071  *		internal_hash
3072  *		journal_crypt
3073  *		journal_mac
3074  *		block_size
3075  */
3076 static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
3077 {
3078 	struct dm_integrity_c *ic;
3079 	char dummy;
3080 	int r;
3081 	unsigned extra_args;
3082 	struct dm_arg_set as;
3083 	static const struct dm_arg _args[] = {
3084 		{0, 9, "Invalid number of feature args"},
3085 	};
3086 	unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
3087 	bool recalculate;
3088 	bool should_write_sb;
3089 	__u64 threshold;
3090 	unsigned long long start;
3091 
3092 #define DIRECT_ARGUMENTS	4
3093 
3094 	if (argc <= DIRECT_ARGUMENTS) {
3095 		ti->error = "Invalid argument count";
3096 		return -EINVAL;
3097 	}
3098 
3099 	ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL);
3100 	if (!ic) {
3101 		ti->error = "Cannot allocate integrity context";
3102 		return -ENOMEM;
3103 	}
3104 	ti->private = ic;
3105 	ti->per_io_data_size = sizeof(struct dm_integrity_io);
3106 
3107 	ic->in_progress = RB_ROOT;
3108 	INIT_LIST_HEAD(&ic->wait_list);
3109 	init_waitqueue_head(&ic->endio_wait);
3110 	bio_list_init(&ic->flush_bio_list);
3111 	init_waitqueue_head(&ic->copy_to_journal_wait);
3112 	init_completion(&ic->crypto_backoff);
3113 	atomic64_set(&ic->number_of_mismatches, 0);
3114 
3115 	r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev);
3116 	if (r) {
3117 		ti->error = "Device lookup failed";
3118 		goto bad;
3119 	}
3120 
3121 	if (sscanf(argv[1], "%llu%c", &start, &dummy) != 1 || start != (sector_t)start) {
3122 		ti->error = "Invalid starting offset";
3123 		r = -EINVAL;
3124 		goto bad;
3125 	}
3126 	ic->start = start;
3127 
3128 	if (strcmp(argv[2], "-")) {
3129 		if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) {
3130 			ti->error = "Invalid tag size";
3131 			r = -EINVAL;
3132 			goto bad;
3133 		}
3134 	}
3135 
3136 	if (!strcmp(argv[3], "J") || !strcmp(argv[3], "D") || !strcmp(argv[3], "R"))
3137 		ic->mode = argv[3][0];
3138 	else {
3139 		ti->error = "Invalid mode (expecting J, D, R)";
3140 		r = -EINVAL;
3141 		goto bad;
3142 	}
3143 
3144 	journal_sectors = 0;
3145 	interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
3146 	buffer_sectors = DEFAULT_BUFFER_SECTORS;
3147 	journal_watermark = DEFAULT_JOURNAL_WATERMARK;
3148 	sync_msec = DEFAULT_SYNC_MSEC;
3149 	recalculate = false;
3150 	ic->sectors_per_block = 1;
3151 
3152 	as.argc = argc - DIRECT_ARGUMENTS;
3153 	as.argv = argv + DIRECT_ARGUMENTS;
3154 	r = dm_read_arg_group(_args, &as, &extra_args, &ti->error);
3155 	if (r)
3156 		goto bad;
3157 
3158 	while (extra_args--) {
3159 		const char *opt_string;
3160 		unsigned val;
3161 		opt_string = dm_shift_arg(&as);
3162 		if (!opt_string) {
3163 			r = -EINVAL;
3164 			ti->error = "Not enough feature arguments";
3165 			goto bad;
3166 		}
3167 		if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1)
3168 			journal_sectors = val ? val : 1;
3169 		else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1)
3170 			interleave_sectors = val;
3171 		else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1)
3172 			buffer_sectors = val;
3173 		else if (sscanf(opt_string, "journal_watermark:%u%c", &val, &dummy) == 1 && val <= 100)
3174 			journal_watermark = val;
3175 		else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1)
3176 			sync_msec = val;
3177 		else if (!memcmp(opt_string, "meta_device:", strlen("meta_device:"))) {
3178 			if (ic->meta_dev) {
3179 				dm_put_device(ti, ic->meta_dev);
3180 				ic->meta_dev = NULL;
3181 			}
3182 			r = dm_get_device(ti, strchr(opt_string, ':') + 1, dm_table_get_mode(ti->table), &ic->meta_dev);
3183 			if (r) {
3184 				ti->error = "Device lookup failed";
3185 				goto bad;
3186 			}
3187 		} else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) {
3188 			if (val < 1 << SECTOR_SHIFT ||
3189 			    val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT ||
3190 			    (val & (val -1))) {
3191 				r = -EINVAL;
3192 				ti->error = "Invalid block_size argument";
3193 				goto bad;
3194 			}
3195 			ic->sectors_per_block = val >> SECTOR_SHIFT;
3196 		} else if (!memcmp(opt_string, "internal_hash:", strlen("internal_hash:"))) {
3197 			r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error,
3198 					    "Invalid internal_hash argument");
3199 			if (r)
3200 				goto bad;
3201 		} else if (!memcmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) {
3202 			r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error,
3203 					    "Invalid journal_crypt argument");
3204 			if (r)
3205 				goto bad;
3206 		} else if (!memcmp(opt_string, "journal_mac:", strlen("journal_mac:"))) {
3207 			r = get_alg_and_key(opt_string, &ic->journal_mac_alg,  &ti->error,
3208 					    "Invalid journal_mac argument");
3209 			if (r)
3210 				goto bad;
3211 		} else if (!strcmp(opt_string, "recalculate")) {
3212 			recalculate = true;
3213 		} else {
3214 			r = -EINVAL;
3215 			ti->error = "Invalid argument";
3216 			goto bad;
3217 		}
3218 	}
3219 
3220 	ic->data_device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT;
3221 	if (!ic->meta_dev)
3222 		ic->meta_device_sectors = ic->data_device_sectors;
3223 	else
3224 		ic->meta_device_sectors = i_size_read(ic->meta_dev->bdev->bd_inode) >> SECTOR_SHIFT;
3225 
3226 	if (!journal_sectors) {
3227 		journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS,
3228 			ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR);
3229 	}
3230 
3231 	if (!buffer_sectors)
3232 		buffer_sectors = 1;
3233 	ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT);
3234 
3235 	r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error,
3236 		    "Invalid internal hash", "Error setting internal hash key");
3237 	if (r)
3238 		goto bad;
3239 
3240 	r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error,
3241 		    "Invalid journal mac", "Error setting journal mac key");
3242 	if (r)
3243 		goto bad;
3244 
3245 	if (!ic->tag_size) {
3246 		if (!ic->internal_hash) {
3247 			ti->error = "Unknown tag size";
3248 			r = -EINVAL;
3249 			goto bad;
3250 		}
3251 		ic->tag_size = crypto_shash_digestsize(ic->internal_hash);
3252 	}
3253 	if (ic->tag_size > MAX_TAG_SIZE) {
3254 		ti->error = "Too big tag size";
3255 		r = -EINVAL;
3256 		goto bad;
3257 	}
3258 	if (!(ic->tag_size & (ic->tag_size - 1)))
3259 		ic->log2_tag_size = __ffs(ic->tag_size);
3260 	else
3261 		ic->log2_tag_size = -1;
3262 
3263 	ic->autocommit_jiffies = msecs_to_jiffies(sync_msec);
3264 	ic->autocommit_msec = sync_msec;
3265 	timer_setup(&ic->autocommit_timer, autocommit_fn, 0);
3266 
3267 	ic->io = dm_io_client_create();
3268 	if (IS_ERR(ic->io)) {
3269 		r = PTR_ERR(ic->io);
3270 		ic->io = NULL;
3271 		ti->error = "Cannot allocate dm io";
3272 		goto bad;
3273 	}
3274 
3275 	r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache);
3276 	if (r) {
3277 		ti->error = "Cannot allocate mempool";
3278 		goto bad;
3279 	}
3280 
3281 	ic->metadata_wq = alloc_workqueue("dm-integrity-metadata",
3282 					  WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE);
3283 	if (!ic->metadata_wq) {
3284 		ti->error = "Cannot allocate workqueue";
3285 		r = -ENOMEM;
3286 		goto bad;
3287 	}
3288 
3289 	/*
3290 	 * If this workqueue were percpu, it would cause bio reordering
3291 	 * and reduced performance.
3292 	 */
3293 	ic->wait_wq = alloc_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
3294 	if (!ic->wait_wq) {
3295 		ti->error = "Cannot allocate workqueue";
3296 		r = -ENOMEM;
3297 		goto bad;
3298 	}
3299 
3300 	ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1);
3301 	if (!ic->commit_wq) {
3302 		ti->error = "Cannot allocate workqueue";
3303 		r = -ENOMEM;
3304 		goto bad;
3305 	}
3306 	INIT_WORK(&ic->commit_work, integrity_commit);
3307 
3308 	if (ic->mode == 'J') {
3309 		ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1);
3310 		if (!ic->writer_wq) {
3311 			ti->error = "Cannot allocate workqueue";
3312 			r = -ENOMEM;
3313 			goto bad;
3314 		}
3315 		INIT_WORK(&ic->writer_work, integrity_writer);
3316 	}
3317 
3318 	ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL);
3319 	if (!ic->sb) {
3320 		r = -ENOMEM;
3321 		ti->error = "Cannot allocate superblock area";
3322 		goto bad;
3323 	}
3324 
3325 	r = sync_rw_sb(ic, REQ_OP_READ, 0);
3326 	if (r) {
3327 		ti->error = "Error reading superblock";
3328 		goto bad;
3329 	}
3330 	should_write_sb = false;
3331 	if (memcmp(ic->sb->magic, SB_MAGIC, 8)) {
3332 		if (ic->mode != 'R') {
3333 			if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) {
3334 				r = -EINVAL;
3335 				ti->error = "The device is not initialized";
3336 				goto bad;
3337 			}
3338 		}
3339 
3340 		r = initialize_superblock(ic, journal_sectors, interleave_sectors);
3341 		if (r) {
3342 			ti->error = "Could not initialize superblock";
3343 			goto bad;
3344 		}
3345 		if (ic->mode != 'R')
3346 			should_write_sb = true;
3347 	}
3348 
3349 	if (!ic->sb->version || ic->sb->version > SB_VERSION_2) {
3350 		r = -EINVAL;
3351 		ti->error = "Unknown version";
3352 		goto bad;
3353 	}
3354 	if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) {
3355 		r = -EINVAL;
3356 		ti->error = "Tag size doesn't match the information in superblock";
3357 		goto bad;
3358 	}
3359 	if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) {
3360 		r = -EINVAL;
3361 		ti->error = "Block size doesn't match the information in superblock";
3362 		goto bad;
3363 	}
3364 	if (!le32_to_cpu(ic->sb->journal_sections)) {
3365 		r = -EINVAL;
3366 		ti->error = "Corrupted superblock, journal_sections is 0";
3367 		goto bad;
3368 	}
3369 	/* make sure that ti->max_io_len doesn't overflow */
3370 	if (!ic->meta_dev) {
3371 		if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS ||
3372 		    ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) {
3373 			r = -EINVAL;
3374 			ti->error = "Invalid interleave_sectors in the superblock";
3375 			goto bad;
3376 		}
3377 	} else {
3378 		if (ic->sb->log2_interleave_sectors) {
3379 			r = -EINVAL;
3380 			ti->error = "Invalid interleave_sectors in the superblock";
3381 			goto bad;
3382 		}
3383 	}
3384 	ic->provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors);
3385 	if (ic->provided_data_sectors != le64_to_cpu(ic->sb->provided_data_sectors)) {
3386 		/* test for overflow */
3387 		r = -EINVAL;
3388 		ti->error = "The superblock has 64-bit device size, but the kernel was compiled with 32-bit sectors";
3389 		goto bad;
3390 	}
3391 	if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) {
3392 		r = -EINVAL;
3393 		ti->error = "Journal mac mismatch";
3394 		goto bad;
3395 	}
3396 
3397 try_smaller_buffer:
3398 	r = calculate_device_limits(ic);
3399 	if (r) {
3400 		if (ic->meta_dev) {
3401 			if (ic->log2_buffer_sectors > 3) {
3402 				ic->log2_buffer_sectors--;
3403 				goto try_smaller_buffer;
3404 			}
3405 		}
3406 		ti->error = "The device is too small";
3407 		goto bad;
3408 	}
3409 	if (!ic->meta_dev)
3410 		ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run));
3411 
3412 	if (ti->len > ic->provided_data_sectors) {
3413 		r = -EINVAL;
3414 		ti->error = "Not enough provided sectors for requested mapping size";
3415 		goto bad;
3416 	}
3417 
3418 
3419 	threshold = (__u64)ic->journal_entries * (100 - journal_watermark);
3420 	threshold += 50;
3421 	do_div(threshold, 100);
3422 	ic->free_sectors_threshold = threshold;
3423 
3424 	DEBUG_print("initialized:\n");
3425 	DEBUG_print("	integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size));
3426 	DEBUG_print("	journal_entry_size %u\n", ic->journal_entry_size);
3427 	DEBUG_print("	journal_entries_per_sector %u\n", ic->journal_entries_per_sector);
3428 	DEBUG_print("	journal_section_entries %u\n", ic->journal_section_entries);
3429 	DEBUG_print("	journal_section_sectors %u\n", ic->journal_section_sectors);
3430 	DEBUG_print("	journal_sections %u\n", (unsigned)le32_to_cpu(ic->sb->journal_sections));
3431 	DEBUG_print("	journal_entries %u\n", ic->journal_entries);
3432 	DEBUG_print("	log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors);
3433 	DEBUG_print("	device_sectors 0x%llx\n", (unsigned long long)ic->device_sectors);
3434 	DEBUG_print("	initial_sectors 0x%x\n", ic->initial_sectors);
3435 	DEBUG_print("	metadata_run 0x%x\n", ic->metadata_run);
3436 	DEBUG_print("	log2_metadata_run %d\n", ic->log2_metadata_run);
3437 	DEBUG_print("	provided_data_sectors 0x%llx (%llu)\n", (unsigned long long)ic->provided_data_sectors,
3438 		    (unsigned long long)ic->provided_data_sectors);
3439 	DEBUG_print("	log2_buffer_sectors %u\n", ic->log2_buffer_sectors);
3440 
3441 	if (recalculate && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) {
3442 		ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3443 		ic->sb->recalc_sector = cpu_to_le64(0);
3444 	}
3445 
3446 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
3447 		if (!ic->internal_hash) {
3448 			r = -EINVAL;
3449 			ti->error = "Recalculate is only valid with internal hash";
3450 			goto bad;
3451 		}
3452 		ic->recalc_wq = alloc_workqueue("dm-intergrity-recalc", WQ_MEM_RECLAIM, 1);
3453 		if (!ic->recalc_wq ) {
3454 			ti->error = "Cannot allocate workqueue";
3455 			r = -ENOMEM;
3456 			goto bad;
3457 		}
3458 		INIT_WORK(&ic->recalc_work, integrity_recalc);
3459 		ic->recalc_buffer = vmalloc(RECALC_SECTORS << SECTOR_SHIFT);
3460 		if (!ic->recalc_buffer) {
3461 			ti->error = "Cannot allocate buffer for recalculating";
3462 			r = -ENOMEM;
3463 			goto bad;
3464 		}
3465 		ic->recalc_tags = kvmalloc((RECALC_SECTORS >> ic->sb->log2_sectors_per_block) * ic->tag_size, GFP_KERNEL);
3466 		if (!ic->recalc_tags) {
3467 			ti->error = "Cannot allocate tags for recalculating";
3468 			r = -ENOMEM;
3469 			goto bad;
3470 		}
3471 	}
3472 
3473 	ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev,
3474 			1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL);
3475 	if (IS_ERR(ic->bufio)) {
3476 		r = PTR_ERR(ic->bufio);
3477 		ti->error = "Cannot initialize dm-bufio";
3478 		ic->bufio = NULL;
3479 		goto bad;
3480 	}
3481 	dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors);
3482 
3483 	if (ic->mode != 'R') {
3484 		r = create_journal(ic, &ti->error);
3485 		if (r)
3486 			goto bad;
3487 	}
3488 
3489 	if (should_write_sb) {
3490 		int r;
3491 
3492 		init_journal(ic, 0, ic->journal_sections, 0);
3493 		r = dm_integrity_failed(ic);
3494 		if (unlikely(r)) {
3495 			ti->error = "Error initializing journal";
3496 			goto bad;
3497 		}
3498 		r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA);
3499 		if (r) {
3500 			ti->error = "Error initializing superblock";
3501 			goto bad;
3502 		}
3503 		ic->just_formatted = true;
3504 	}
3505 
3506 	if (!ic->meta_dev) {
3507 		r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors);
3508 		if (r)
3509 			goto bad;
3510 	}
3511 
3512 	if (!ic->internal_hash)
3513 		dm_integrity_set(ti, ic);
3514 
3515 	ti->num_flush_bios = 1;
3516 	ti->flush_supported = true;
3517 
3518 	return 0;
3519 bad:
3520 	dm_integrity_dtr(ti);
3521 	return r;
3522 }
3523 
3524 static void dm_integrity_dtr(struct dm_target *ti)
3525 {
3526 	struct dm_integrity_c *ic = ti->private;
3527 
3528 	BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
3529 	BUG_ON(!list_empty(&ic->wait_list));
3530 
3531 	if (ic->metadata_wq)
3532 		destroy_workqueue(ic->metadata_wq);
3533 	if (ic->wait_wq)
3534 		destroy_workqueue(ic->wait_wq);
3535 	if (ic->commit_wq)
3536 		destroy_workqueue(ic->commit_wq);
3537 	if (ic->writer_wq)
3538 		destroy_workqueue(ic->writer_wq);
3539 	if (ic->recalc_wq)
3540 		destroy_workqueue(ic->recalc_wq);
3541 	if (ic->recalc_buffer)
3542 		vfree(ic->recalc_buffer);
3543 	if (ic->recalc_tags)
3544 		kvfree(ic->recalc_tags);
3545 	if (ic->bufio)
3546 		dm_bufio_client_destroy(ic->bufio);
3547 	mempool_exit(&ic->journal_io_mempool);
3548 	if (ic->io)
3549 		dm_io_client_destroy(ic->io);
3550 	if (ic->dev)
3551 		dm_put_device(ti, ic->dev);
3552 	if (ic->meta_dev)
3553 		dm_put_device(ti, ic->meta_dev);
3554 	dm_integrity_free_page_list(ic, ic->journal);
3555 	dm_integrity_free_page_list(ic, ic->journal_io);
3556 	dm_integrity_free_page_list(ic, ic->journal_xor);
3557 	if (ic->journal_scatterlist)
3558 		dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist);
3559 	if (ic->journal_io_scatterlist)
3560 		dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist);
3561 	if (ic->sk_requests) {
3562 		unsigned i;
3563 
3564 		for (i = 0; i < ic->journal_sections; i++) {
3565 			struct skcipher_request *req = ic->sk_requests[i];
3566 			if (req) {
3567 				kzfree(req->iv);
3568 				skcipher_request_free(req);
3569 			}
3570 		}
3571 		kvfree(ic->sk_requests);
3572 	}
3573 	kvfree(ic->journal_tree);
3574 	if (ic->sb)
3575 		free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT);
3576 
3577 	if (ic->internal_hash)
3578 		crypto_free_shash(ic->internal_hash);
3579 	free_alg(&ic->internal_hash_alg);
3580 
3581 	if (ic->journal_crypt)
3582 		crypto_free_skcipher(ic->journal_crypt);
3583 	free_alg(&ic->journal_crypt_alg);
3584 
3585 	if (ic->journal_mac)
3586 		crypto_free_shash(ic->journal_mac);
3587 	free_alg(&ic->journal_mac_alg);
3588 
3589 	kfree(ic);
3590 }
3591 
3592 static struct target_type integrity_target = {
3593 	.name			= "integrity",
3594 	.version		= {1, 2, 0},
3595 	.module			= THIS_MODULE,
3596 	.features		= DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
3597 	.ctr			= dm_integrity_ctr,
3598 	.dtr			= dm_integrity_dtr,
3599 	.map			= dm_integrity_map,
3600 	.postsuspend		= dm_integrity_postsuspend,
3601 	.resume			= dm_integrity_resume,
3602 	.status			= dm_integrity_status,
3603 	.iterate_devices	= dm_integrity_iterate_devices,
3604 	.io_hints		= dm_integrity_io_hints,
3605 };
3606 
3607 int __init dm_integrity_init(void)
3608 {
3609 	int r;
3610 
3611 	journal_io_cache = kmem_cache_create("integrity_journal_io",
3612 					     sizeof(struct journal_io), 0, 0, NULL);
3613 	if (!journal_io_cache) {
3614 		DMERR("can't allocate journal io cache");
3615 		return -ENOMEM;
3616 	}
3617 
3618 	r = dm_register_target(&integrity_target);
3619 
3620 	if (r < 0)
3621 		DMERR("register failed %d", r);
3622 
3623 	return r;
3624 }
3625 
3626 void dm_integrity_exit(void)
3627 {
3628 	dm_unregister_target(&integrity_target);
3629 	kmem_cache_destroy(journal_io_cache);
3630 }
3631 
3632 module_init(dm_integrity_init);
3633 module_exit(dm_integrity_exit);
3634 
3635 MODULE_AUTHOR("Milan Broz");
3636 MODULE_AUTHOR("Mikulas Patocka");
3637 MODULE_DESCRIPTION(DM_NAME " target for integrity tags extension");
3638 MODULE_LICENSE("GPL");
3639