xref: /openbmc/linux/drivers/md/dm-integrity.c (revision 4da722ca)
1 /*
2  * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved.
3  * Copyright (C) 2016-2017 Milan Broz
4  * Copyright (C) 2016-2017 Mikulas Patocka
5  *
6  * This file is released under the GPL.
7  */
8 
9 #include <linux/module.h>
10 #include <linux/device-mapper.h>
11 #include <linux/dm-io.h>
12 #include <linux/vmalloc.h>
13 #include <linux/sort.h>
14 #include <linux/rbtree.h>
15 #include <linux/delay.h>
16 #include <linux/random.h>
17 #include <crypto/hash.h>
18 #include <crypto/skcipher.h>
19 #include <linux/async_tx.h>
20 #include "dm-bufio.h"
21 
22 #define DM_MSG_PREFIX "integrity"
23 
24 #define DEFAULT_INTERLEAVE_SECTORS	32768
25 #define DEFAULT_JOURNAL_SIZE_FACTOR	7
26 #define DEFAULT_BUFFER_SECTORS		128
27 #define DEFAULT_JOURNAL_WATERMARK	50
28 #define DEFAULT_SYNC_MSEC		10000
29 #define DEFAULT_MAX_JOURNAL_SECTORS	131072
30 #define MIN_LOG2_INTERLEAVE_SECTORS	3
31 #define MAX_LOG2_INTERLEAVE_SECTORS	31
32 #define METADATA_WORKQUEUE_MAX_ACTIVE	16
33 
34 /*
35  * Warning - DEBUG_PRINT prints security-sensitive data to the log,
36  * so it should not be enabled in the official kernel
37  */
38 //#define DEBUG_PRINT
39 //#define INTERNAL_VERIFY
40 
41 /*
42  * On disk structures
43  */
44 
45 #define SB_MAGIC			"integrt"
46 #define SB_VERSION			1
47 #define SB_SECTORS			8
48 #define MAX_SECTORS_PER_BLOCK		8
49 
50 struct superblock {
51 	__u8 magic[8];
52 	__u8 version;
53 	__u8 log2_interleave_sectors;
54 	__u16 integrity_tag_size;
55 	__u32 journal_sections;
56 	__u64 provided_data_sectors;	/* userspace uses this value */
57 	__u32 flags;
58 	__u8 log2_sectors_per_block;
59 };
60 
61 #define SB_FLAG_HAVE_JOURNAL_MAC	0x1
62 
63 #define	JOURNAL_ENTRY_ROUNDUP		8
64 
65 typedef __u64 commit_id_t;
66 #define JOURNAL_MAC_PER_SECTOR		8
67 
68 struct journal_entry {
69 	union {
70 		struct {
71 			__u32 sector_lo;
72 			__u32 sector_hi;
73 		} s;
74 		__u64 sector;
75 	} u;
76 	commit_id_t last_bytes[0];
77 	/* __u8 tag[0]; */
78 };
79 
80 #define journal_entry_tag(ic, je)		((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block])
81 
82 #if BITS_PER_LONG == 64
83 #define journal_entry_set_sector(je, x)		do { smp_wmb(); ACCESS_ONCE((je)->u.sector) = cpu_to_le64(x); } while (0)
84 #define journal_entry_get_sector(je)		le64_to_cpu((je)->u.sector)
85 #elif defined(CONFIG_LBDAF)
86 #define journal_entry_set_sector(je, x)		do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); ACCESS_ONCE((je)->u.s.sector_hi) = cpu_to_le32((x) >> 32); } while (0)
87 #define journal_entry_get_sector(je)		le64_to_cpu((je)->u.sector)
88 #else
89 #define journal_entry_set_sector(je, x)		do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); ACCESS_ONCE((je)->u.s.sector_hi) = cpu_to_le32(0); } while (0)
90 #define journal_entry_get_sector(je)		le32_to_cpu((je)->u.s.sector_lo)
91 #endif
92 #define journal_entry_is_unused(je)		((je)->u.s.sector_hi == cpu_to_le32(-1))
93 #define journal_entry_set_unused(je)		do { ((je)->u.s.sector_hi = cpu_to_le32(-1)); } while (0)
94 #define journal_entry_is_inprogress(je)		((je)->u.s.sector_hi == cpu_to_le32(-2))
95 #define journal_entry_set_inprogress(je)	do { ((je)->u.s.sector_hi = cpu_to_le32(-2)); } while (0)
96 
97 #define JOURNAL_BLOCK_SECTORS		8
98 #define JOURNAL_SECTOR_DATA		((1 << SECTOR_SHIFT) - sizeof(commit_id_t))
99 #define JOURNAL_MAC_SIZE		(JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS)
100 
101 struct journal_sector {
102 	__u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR];
103 	__u8 mac[JOURNAL_MAC_PER_SECTOR];
104 	commit_id_t commit_id;
105 };
106 
107 #define MAX_TAG_SIZE			(JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK]))
108 
109 #define METADATA_PADDING_SECTORS	8
110 
111 #define N_COMMIT_IDS			4
112 
113 static unsigned char prev_commit_seq(unsigned char seq)
114 {
115 	return (seq + N_COMMIT_IDS - 1) % N_COMMIT_IDS;
116 }
117 
118 static unsigned char next_commit_seq(unsigned char seq)
119 {
120 	return (seq + 1) % N_COMMIT_IDS;
121 }
122 
123 /*
124  * In-memory structures
125  */
126 
127 struct journal_node {
128 	struct rb_node node;
129 	sector_t sector;
130 };
131 
132 struct alg_spec {
133 	char *alg_string;
134 	char *key_string;
135 	__u8 *key;
136 	unsigned key_size;
137 };
138 
139 struct dm_integrity_c {
140 	struct dm_dev *dev;
141 	unsigned tag_size;
142 	__s8 log2_tag_size;
143 	sector_t start;
144 	mempool_t *journal_io_mempool;
145 	struct dm_io_client *io;
146 	struct dm_bufio_client *bufio;
147 	struct workqueue_struct *metadata_wq;
148 	struct superblock *sb;
149 	unsigned journal_pages;
150 	struct page_list *journal;
151 	struct page_list *journal_io;
152 	struct page_list *journal_xor;
153 
154 	struct crypto_skcipher *journal_crypt;
155 	struct scatterlist **journal_scatterlist;
156 	struct scatterlist **journal_io_scatterlist;
157 	struct skcipher_request **sk_requests;
158 
159 	struct crypto_shash *journal_mac;
160 
161 	struct journal_node *journal_tree;
162 	struct rb_root journal_tree_root;
163 
164 	sector_t provided_data_sectors;
165 
166 	unsigned short journal_entry_size;
167 	unsigned char journal_entries_per_sector;
168 	unsigned char journal_section_entries;
169 	unsigned short journal_section_sectors;
170 	unsigned journal_sections;
171 	unsigned journal_entries;
172 	sector_t device_sectors;
173 	unsigned initial_sectors;
174 	unsigned metadata_run;
175 	__s8 log2_metadata_run;
176 	__u8 log2_buffer_sectors;
177 	__u8 sectors_per_block;
178 
179 	unsigned char mode;
180 	bool suspending;
181 
182 	int failed;
183 
184 	struct crypto_shash *internal_hash;
185 
186 	/* these variables are locked with endio_wait.lock */
187 	struct rb_root in_progress;
188 	wait_queue_head_t endio_wait;
189 	struct workqueue_struct *wait_wq;
190 
191 	unsigned char commit_seq;
192 	commit_id_t commit_ids[N_COMMIT_IDS];
193 
194 	unsigned committed_section;
195 	unsigned n_committed_sections;
196 
197 	unsigned uncommitted_section;
198 	unsigned n_uncommitted_sections;
199 
200 	unsigned free_section;
201 	unsigned char free_section_entry;
202 	unsigned free_sectors;
203 
204 	unsigned free_sectors_threshold;
205 
206 	struct workqueue_struct *commit_wq;
207 	struct work_struct commit_work;
208 
209 	struct workqueue_struct *writer_wq;
210 	struct work_struct writer_work;
211 
212 	struct bio_list flush_bio_list;
213 
214 	unsigned long autocommit_jiffies;
215 	struct timer_list autocommit_timer;
216 	unsigned autocommit_msec;
217 
218 	wait_queue_head_t copy_to_journal_wait;
219 
220 	struct completion crypto_backoff;
221 
222 	bool journal_uptodate;
223 	bool just_formatted;
224 
225 	struct alg_spec internal_hash_alg;
226 	struct alg_spec journal_crypt_alg;
227 	struct alg_spec journal_mac_alg;
228 };
229 
230 struct dm_integrity_range {
231 	sector_t logical_sector;
232 	unsigned n_sectors;
233 	struct rb_node node;
234 };
235 
236 struct dm_integrity_io {
237 	struct work_struct work;
238 
239 	struct dm_integrity_c *ic;
240 	bool write;
241 	bool fua;
242 
243 	struct dm_integrity_range range;
244 
245 	sector_t metadata_block;
246 	unsigned metadata_offset;
247 
248 	atomic_t in_flight;
249 	blk_status_t bi_status;
250 
251 	struct completion *completion;
252 
253 	struct block_device *orig_bi_bdev;
254 	bio_end_io_t *orig_bi_end_io;
255 	struct bio_integrity_payload *orig_bi_integrity;
256 	struct bvec_iter orig_bi_iter;
257 };
258 
259 struct journal_completion {
260 	struct dm_integrity_c *ic;
261 	atomic_t in_flight;
262 	struct completion comp;
263 };
264 
265 struct journal_io {
266 	struct dm_integrity_range range;
267 	struct journal_completion *comp;
268 };
269 
270 static struct kmem_cache *journal_io_cache;
271 
272 #define JOURNAL_IO_MEMPOOL	32
273 
274 #ifdef DEBUG_PRINT
275 #define DEBUG_print(x, ...)	printk(KERN_DEBUG x, ##__VA_ARGS__)
276 static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...)
277 {
278 	va_list args;
279 	va_start(args, msg);
280 	vprintk(msg, args);
281 	va_end(args);
282 	if (len)
283 		pr_cont(":");
284 	while (len) {
285 		pr_cont(" %02x", *bytes);
286 		bytes++;
287 		len--;
288 	}
289 	pr_cont("\n");
290 }
291 #define DEBUG_bytes(bytes, len, msg, ...)	__DEBUG_bytes(bytes, len, KERN_DEBUG msg, ##__VA_ARGS__)
292 #else
293 #define DEBUG_print(x, ...)			do { } while (0)
294 #define DEBUG_bytes(bytes, len, msg, ...)	do { } while (0)
295 #endif
296 
297 /*
298  * DM Integrity profile, protection is performed layer above (dm-crypt)
299  */
300 static struct blk_integrity_profile dm_integrity_profile = {
301 	.name			= "DM-DIF-EXT-TAG",
302 	.generate_fn		= NULL,
303 	.verify_fn		= NULL,
304 };
305 
306 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
307 static void integrity_bio_wait(struct work_struct *w);
308 static void dm_integrity_dtr(struct dm_target *ti);
309 
310 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err)
311 {
312 	if (!cmpxchg(&ic->failed, 0, err))
313 		DMERR("Error on %s: %d", msg, err);
314 }
315 
316 static int dm_integrity_failed(struct dm_integrity_c *ic)
317 {
318 	return ACCESS_ONCE(ic->failed);
319 }
320 
321 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i,
322 					  unsigned j, unsigned char seq)
323 {
324 	/*
325 	 * Xor the number with section and sector, so that if a piece of
326 	 * journal is written at wrong place, it is detected.
327 	 */
328 	return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j);
329 }
330 
331 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector,
332 				sector_t *area, sector_t *offset)
333 {
334 	__u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors;
335 
336 	*area = data_sector >> log2_interleave_sectors;
337 	*offset = (unsigned)data_sector & ((1U << log2_interleave_sectors) - 1);
338 }
339 
340 #define sector_to_block(ic, n)						\
341 do {									\
342 	BUG_ON((n) & (unsigned)((ic)->sectors_per_block - 1));		\
343 	(n) >>= (ic)->sb->log2_sectors_per_block;			\
344 } while (0)
345 
346 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area,
347 					    sector_t offset, unsigned *metadata_offset)
348 {
349 	__u64 ms;
350 	unsigned mo;
351 
352 	ms = area << ic->sb->log2_interleave_sectors;
353 	if (likely(ic->log2_metadata_run >= 0))
354 		ms += area << ic->log2_metadata_run;
355 	else
356 		ms += area * ic->metadata_run;
357 	ms >>= ic->log2_buffer_sectors;
358 
359 	sector_to_block(ic, offset);
360 
361 	if (likely(ic->log2_tag_size >= 0)) {
362 		ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size);
363 		mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
364 	} else {
365 		ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors);
366 		mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
367 	}
368 	*metadata_offset = mo;
369 	return ms;
370 }
371 
372 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset)
373 {
374 	sector_t result;
375 
376 	result = area << ic->sb->log2_interleave_sectors;
377 	if (likely(ic->log2_metadata_run >= 0))
378 		result += (area + 1) << ic->log2_metadata_run;
379 	else
380 		result += (area + 1) * ic->metadata_run;
381 
382 	result += (sector_t)ic->initial_sectors + offset;
383 	return result;
384 }
385 
386 static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr)
387 {
388 	if (unlikely(*sec_ptr >= ic->journal_sections))
389 		*sec_ptr -= ic->journal_sections;
390 }
391 
392 static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags)
393 {
394 	struct dm_io_request io_req;
395 	struct dm_io_region io_loc;
396 
397 	io_req.bi_op = op;
398 	io_req.bi_op_flags = op_flags;
399 	io_req.mem.type = DM_IO_KMEM;
400 	io_req.mem.ptr.addr = ic->sb;
401 	io_req.notify.fn = NULL;
402 	io_req.client = ic->io;
403 	io_loc.bdev = ic->dev->bdev;
404 	io_loc.sector = ic->start;
405 	io_loc.count = SB_SECTORS;
406 
407 	return dm_io(&io_req, 1, &io_loc, NULL);
408 }
409 
410 static void access_journal_check(struct dm_integrity_c *ic, unsigned section, unsigned offset,
411 				 bool e, const char *function)
412 {
413 #if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY)
414 	unsigned limit = e ? ic->journal_section_entries : ic->journal_section_sectors;
415 
416 	if (unlikely(section >= ic->journal_sections) ||
417 	    unlikely(offset >= limit)) {
418 		printk(KERN_CRIT "%s: invalid access at (%u,%u), limit (%u,%u)\n",
419 			function, section, offset, ic->journal_sections, limit);
420 		BUG();
421 	}
422 #endif
423 }
424 
425 static void page_list_location(struct dm_integrity_c *ic, unsigned section, unsigned offset,
426 			       unsigned *pl_index, unsigned *pl_offset)
427 {
428 	unsigned sector;
429 
430 	access_journal_check(ic, section, offset, false, "page_list_location");
431 
432 	sector = section * ic->journal_section_sectors + offset;
433 
434 	*pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
435 	*pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
436 }
437 
438 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl,
439 					       unsigned section, unsigned offset, unsigned *n_sectors)
440 {
441 	unsigned pl_index, pl_offset;
442 	char *va;
443 
444 	page_list_location(ic, section, offset, &pl_index, &pl_offset);
445 
446 	if (n_sectors)
447 		*n_sectors = (PAGE_SIZE - pl_offset) >> SECTOR_SHIFT;
448 
449 	va = lowmem_page_address(pl[pl_index].page);
450 
451 	return (struct journal_sector *)(va + pl_offset);
452 }
453 
454 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset)
455 {
456 	return access_page_list(ic, ic->journal, section, offset, NULL);
457 }
458 
459 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned section, unsigned n)
460 {
461 	unsigned rel_sector, offset;
462 	struct journal_sector *js;
463 
464 	access_journal_check(ic, section, n, true, "access_journal_entry");
465 
466 	rel_sector = n % JOURNAL_BLOCK_SECTORS;
467 	offset = n / JOURNAL_BLOCK_SECTORS;
468 
469 	js = access_journal(ic, section, rel_sector);
470 	return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size);
471 }
472 
473 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned section, unsigned n)
474 {
475 	n <<= ic->sb->log2_sectors_per_block;
476 
477 	n += JOURNAL_BLOCK_SECTORS;
478 
479 	access_journal_check(ic, section, n, false, "access_journal_data");
480 
481 	return access_journal(ic, section, n);
482 }
483 
484 static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result[JOURNAL_MAC_SIZE])
485 {
486 	SHASH_DESC_ON_STACK(desc, ic->journal_mac);
487 	int r;
488 	unsigned j, size;
489 
490 	desc->tfm = ic->journal_mac;
491 	desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
492 
493 	r = crypto_shash_init(desc);
494 	if (unlikely(r)) {
495 		dm_integrity_io_error(ic, "crypto_shash_init", r);
496 		goto err;
497 	}
498 
499 	for (j = 0; j < ic->journal_section_entries; j++) {
500 		struct journal_entry *je = access_journal_entry(ic, section, j);
501 		r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof je->u.sector);
502 		if (unlikely(r)) {
503 			dm_integrity_io_error(ic, "crypto_shash_update", r);
504 			goto err;
505 		}
506 	}
507 
508 	size = crypto_shash_digestsize(ic->journal_mac);
509 
510 	if (likely(size <= JOURNAL_MAC_SIZE)) {
511 		r = crypto_shash_final(desc, result);
512 		if (unlikely(r)) {
513 			dm_integrity_io_error(ic, "crypto_shash_final", r);
514 			goto err;
515 		}
516 		memset(result + size, 0, JOURNAL_MAC_SIZE - size);
517 	} else {
518 		__u8 digest[size];
519 		r = crypto_shash_final(desc, digest);
520 		if (unlikely(r)) {
521 			dm_integrity_io_error(ic, "crypto_shash_final", r);
522 			goto err;
523 		}
524 		memcpy(result, digest, JOURNAL_MAC_SIZE);
525 	}
526 
527 	return;
528 err:
529 	memset(result, 0, JOURNAL_MAC_SIZE);
530 }
531 
532 static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr)
533 {
534 	__u8 result[JOURNAL_MAC_SIZE];
535 	unsigned j;
536 
537 	if (!ic->journal_mac)
538 		return;
539 
540 	section_mac(ic, section, result);
541 
542 	for (j = 0; j < JOURNAL_BLOCK_SECTORS; j++) {
543 		struct journal_sector *js = access_journal(ic, section, j);
544 
545 		if (likely(wr))
546 			memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR);
547 		else {
548 			if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR))
549 				dm_integrity_io_error(ic, "journal mac", -EILSEQ);
550 		}
551 	}
552 }
553 
554 static void complete_journal_op(void *context)
555 {
556 	struct journal_completion *comp = context;
557 	BUG_ON(!atomic_read(&comp->in_flight));
558 	if (likely(atomic_dec_and_test(&comp->in_flight)))
559 		complete(&comp->comp);
560 }
561 
562 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
563 			unsigned n_sections, struct journal_completion *comp)
564 {
565 	struct async_submit_ctl submit;
566 	size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT;
567 	unsigned pl_index, pl_offset, section_index;
568 	struct page_list *source_pl, *target_pl;
569 
570 	if (likely(encrypt)) {
571 		source_pl = ic->journal;
572 		target_pl = ic->journal_io;
573 	} else {
574 		source_pl = ic->journal_io;
575 		target_pl = ic->journal;
576 	}
577 
578 	page_list_location(ic, section, 0, &pl_index, &pl_offset);
579 
580 	atomic_add(roundup(pl_offset + n_bytes, PAGE_SIZE) >> PAGE_SHIFT, &comp->in_flight);
581 
582 	init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, complete_journal_op, comp, NULL);
583 
584 	section_index = pl_index;
585 
586 	do {
587 		size_t this_step;
588 		struct page *src_pages[2];
589 		struct page *dst_page;
590 
591 		while (unlikely(pl_index == section_index)) {
592 			unsigned dummy;
593 			if (likely(encrypt))
594 				rw_section_mac(ic, section, true);
595 			section++;
596 			n_sections--;
597 			if (!n_sections)
598 				break;
599 			page_list_location(ic, section, 0, &section_index, &dummy);
600 		}
601 
602 		this_step = min(n_bytes, (size_t)PAGE_SIZE - pl_offset);
603 		dst_page = target_pl[pl_index].page;
604 		src_pages[0] = source_pl[pl_index].page;
605 		src_pages[1] = ic->journal_xor[pl_index].page;
606 
607 		async_xor(dst_page, src_pages, pl_offset, 2, this_step, &submit);
608 
609 		pl_index++;
610 		pl_offset = 0;
611 		n_bytes -= this_step;
612 	} while (n_bytes);
613 
614 	BUG_ON(n_sections);
615 
616 	async_tx_issue_pending_all();
617 }
618 
619 static void complete_journal_encrypt(struct crypto_async_request *req, int err)
620 {
621 	struct journal_completion *comp = req->data;
622 	if (unlikely(err)) {
623 		if (likely(err == -EINPROGRESS)) {
624 			complete(&comp->ic->crypto_backoff);
625 			return;
626 		}
627 		dm_integrity_io_error(comp->ic, "asynchronous encrypt", err);
628 	}
629 	complete_journal_op(comp);
630 }
631 
632 static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp)
633 {
634 	int r;
635 	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
636 				      complete_journal_encrypt, comp);
637 	if (likely(encrypt))
638 		r = crypto_skcipher_encrypt(req);
639 	else
640 		r = crypto_skcipher_decrypt(req);
641 	if (likely(!r))
642 		return false;
643 	if (likely(r == -EINPROGRESS))
644 		return true;
645 	if (likely(r == -EBUSY)) {
646 		wait_for_completion(&comp->ic->crypto_backoff);
647 		reinit_completion(&comp->ic->crypto_backoff);
648 		return true;
649 	}
650 	dm_integrity_io_error(comp->ic, "encrypt", r);
651 	return false;
652 }
653 
654 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
655 			  unsigned n_sections, struct journal_completion *comp)
656 {
657 	struct scatterlist **source_sg;
658 	struct scatterlist **target_sg;
659 
660 	atomic_add(2, &comp->in_flight);
661 
662 	if (likely(encrypt)) {
663 		source_sg = ic->journal_scatterlist;
664 		target_sg = ic->journal_io_scatterlist;
665 	} else {
666 		source_sg = ic->journal_io_scatterlist;
667 		target_sg = ic->journal_scatterlist;
668 	}
669 
670 	do {
671 		struct skcipher_request *req;
672 		unsigned ivsize;
673 		char *iv;
674 
675 		if (likely(encrypt))
676 			rw_section_mac(ic, section, true);
677 
678 		req = ic->sk_requests[section];
679 		ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
680 		iv = req->iv;
681 
682 		memcpy(iv, iv + ivsize, ivsize);
683 
684 		req->src = source_sg[section];
685 		req->dst = target_sg[section];
686 
687 		if (unlikely(do_crypt(encrypt, req, comp)))
688 			atomic_inc(&comp->in_flight);
689 
690 		section++;
691 		n_sections--;
692 	} while (n_sections);
693 
694 	atomic_dec(&comp->in_flight);
695 	complete_journal_op(comp);
696 }
697 
698 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section,
699 			    unsigned n_sections, struct journal_completion *comp)
700 {
701 	if (ic->journal_xor)
702 		return xor_journal(ic, encrypt, section, n_sections, comp);
703 	else
704 		return crypt_journal(ic, encrypt, section, n_sections, comp);
705 }
706 
707 static void complete_journal_io(unsigned long error, void *context)
708 {
709 	struct journal_completion *comp = context;
710 	if (unlikely(error != 0))
711 		dm_integrity_io_error(comp->ic, "writing journal", -EIO);
712 	complete_journal_op(comp);
713 }
714 
715 static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned section,
716 		       unsigned n_sections, struct journal_completion *comp)
717 {
718 	struct dm_io_request io_req;
719 	struct dm_io_region io_loc;
720 	unsigned sector, n_sectors, pl_index, pl_offset;
721 	int r;
722 
723 	if (unlikely(dm_integrity_failed(ic))) {
724 		if (comp)
725 			complete_journal_io(-1UL, comp);
726 		return;
727 	}
728 
729 	sector = section * ic->journal_section_sectors;
730 	n_sectors = n_sections * ic->journal_section_sectors;
731 
732 	pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
733 	pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
734 
735 	io_req.bi_op = op;
736 	io_req.bi_op_flags = op_flags;
737 	io_req.mem.type = DM_IO_PAGE_LIST;
738 	if (ic->journal_io)
739 		io_req.mem.ptr.pl = &ic->journal_io[pl_index];
740 	else
741 		io_req.mem.ptr.pl = &ic->journal[pl_index];
742 	io_req.mem.offset = pl_offset;
743 	if (likely(comp != NULL)) {
744 		io_req.notify.fn = complete_journal_io;
745 		io_req.notify.context = comp;
746 	} else {
747 		io_req.notify.fn = NULL;
748 	}
749 	io_req.client = ic->io;
750 	io_loc.bdev = ic->dev->bdev;
751 	io_loc.sector = ic->start + SB_SECTORS + sector;
752 	io_loc.count = n_sectors;
753 
754 	r = dm_io(&io_req, 1, &io_loc, NULL);
755 	if (unlikely(r)) {
756 		dm_integrity_io_error(ic, op == REQ_OP_READ ? "reading journal" : "writing journal", r);
757 		if (comp) {
758 			WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
759 			complete_journal_io(-1UL, comp);
760 		}
761 	}
762 }
763 
764 static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsigned commit_sections)
765 {
766 	struct journal_completion io_comp;
767 	struct journal_completion crypt_comp_1;
768 	struct journal_completion crypt_comp_2;
769 	unsigned i;
770 
771 	io_comp.ic = ic;
772 	io_comp.comp = COMPLETION_INITIALIZER_ONSTACK(io_comp.comp);
773 
774 	if (commit_start + commit_sections <= ic->journal_sections) {
775 		io_comp.in_flight = (atomic_t)ATOMIC_INIT(1);
776 		if (ic->journal_io) {
777 			crypt_comp_1.ic = ic;
778 			crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp);
779 			crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
780 			encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1);
781 			wait_for_completion_io(&crypt_comp_1.comp);
782 		} else {
783 			for (i = 0; i < commit_sections; i++)
784 				rw_section_mac(ic, commit_start + i, true);
785 		}
786 		rw_journal(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, commit_start,
787 			   commit_sections, &io_comp);
788 	} else {
789 		unsigned to_end;
790 		io_comp.in_flight = (atomic_t)ATOMIC_INIT(2);
791 		to_end = ic->journal_sections - commit_start;
792 		if (ic->journal_io) {
793 			crypt_comp_1.ic = ic;
794 			crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp);
795 			crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
796 			encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
797 			if (try_wait_for_completion(&crypt_comp_1.comp)) {
798 				rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
799 				crypt_comp_1.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_1.comp);
800 				crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
801 				encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
802 				wait_for_completion_io(&crypt_comp_1.comp);
803 			} else {
804 				crypt_comp_2.ic = ic;
805 				crypt_comp_2.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp_2.comp);
806 				crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
807 				encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
808 				wait_for_completion_io(&crypt_comp_1.comp);
809 				rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
810 				wait_for_completion_io(&crypt_comp_2.comp);
811 			}
812 		} else {
813 			for (i = 0; i < to_end; i++)
814 				rw_section_mac(ic, commit_start + i, true);
815 			rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp);
816 			for (i = 0; i < commit_sections - to_end; i++)
817 				rw_section_mac(ic, i, true);
818 		}
819 		rw_journal(ic, REQ_OP_WRITE, REQ_FUA, 0, commit_sections - to_end, &io_comp);
820 	}
821 
822 	wait_for_completion_io(&io_comp.comp);
823 }
824 
825 static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset,
826 			      unsigned n_sectors, sector_t target, io_notify_fn fn, void *data)
827 {
828 	struct dm_io_request io_req;
829 	struct dm_io_region io_loc;
830 	int r;
831 	unsigned sector, pl_index, pl_offset;
832 
833 	BUG_ON((target | n_sectors | offset) & (unsigned)(ic->sectors_per_block - 1));
834 
835 	if (unlikely(dm_integrity_failed(ic))) {
836 		fn(-1UL, data);
837 		return;
838 	}
839 
840 	sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset;
841 
842 	pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
843 	pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
844 
845 	io_req.bi_op = REQ_OP_WRITE;
846 	io_req.bi_op_flags = 0;
847 	io_req.mem.type = DM_IO_PAGE_LIST;
848 	io_req.mem.ptr.pl = &ic->journal[pl_index];
849 	io_req.mem.offset = pl_offset;
850 	io_req.notify.fn = fn;
851 	io_req.notify.context = data;
852 	io_req.client = ic->io;
853 	io_loc.bdev = ic->dev->bdev;
854 	io_loc.sector = ic->start + target;
855 	io_loc.count = n_sectors;
856 
857 	r = dm_io(&io_req, 1, &io_loc, NULL);
858 	if (unlikely(r)) {
859 		WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
860 		fn(-1UL, data);
861 	}
862 }
863 
864 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
865 {
866 	struct rb_node **n = &ic->in_progress.rb_node;
867 	struct rb_node *parent;
868 
869 	BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned)(ic->sectors_per_block - 1));
870 
871 	parent = NULL;
872 
873 	while (*n) {
874 		struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node);
875 
876 		parent = *n;
877 		if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector) {
878 			n = &range->node.rb_left;
879 		} else if (new_range->logical_sector >= range->logical_sector + range->n_sectors) {
880 			n = &range->node.rb_right;
881 		} else {
882 			return false;
883 		}
884 	}
885 
886 	rb_link_node(&new_range->node, parent, n);
887 	rb_insert_color(&new_range->node, &ic->in_progress);
888 
889 	return true;
890 }
891 
892 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range)
893 {
894 	rb_erase(&range->node, &ic->in_progress);
895 	wake_up_locked(&ic->endio_wait);
896 }
897 
898 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range)
899 {
900 	unsigned long flags;
901 
902 	spin_lock_irqsave(&ic->endio_wait.lock, flags);
903 	remove_range_unlocked(ic, range);
904 	spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
905 }
906 
907 static void init_journal_node(struct journal_node *node)
908 {
909 	RB_CLEAR_NODE(&node->node);
910 	node->sector = (sector_t)-1;
911 }
912 
913 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector)
914 {
915 	struct rb_node **link;
916 	struct rb_node *parent;
917 
918 	node->sector = sector;
919 	BUG_ON(!RB_EMPTY_NODE(&node->node));
920 
921 	link = &ic->journal_tree_root.rb_node;
922 	parent = NULL;
923 
924 	while (*link) {
925 		struct journal_node *j;
926 		parent = *link;
927 		j = container_of(parent, struct journal_node, node);
928 		if (sector < j->sector)
929 			link = &j->node.rb_left;
930 		else
931 			link = &j->node.rb_right;
932 	}
933 
934 	rb_link_node(&node->node, parent, link);
935 	rb_insert_color(&node->node, &ic->journal_tree_root);
936 }
937 
938 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node)
939 {
940 	BUG_ON(RB_EMPTY_NODE(&node->node));
941 	rb_erase(&node->node, &ic->journal_tree_root);
942 	init_journal_node(node);
943 }
944 
945 #define NOT_FOUND	(-1U)
946 
947 static unsigned find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector)
948 {
949 	struct rb_node *n = ic->journal_tree_root.rb_node;
950 	unsigned found = NOT_FOUND;
951 	*next_sector = (sector_t)-1;
952 	while (n) {
953 		struct journal_node *j = container_of(n, struct journal_node, node);
954 		if (sector == j->sector) {
955 			found = j - ic->journal_tree;
956 		}
957 		if (sector < j->sector) {
958 			*next_sector = j->sector;
959 			n = j->node.rb_left;
960 		} else {
961 			n = j->node.rb_right;
962 		}
963 	}
964 
965 	return found;
966 }
967 
968 static bool test_journal_node(struct dm_integrity_c *ic, unsigned pos, sector_t sector)
969 {
970 	struct journal_node *node, *next_node;
971 	struct rb_node *next;
972 
973 	if (unlikely(pos >= ic->journal_entries))
974 		return false;
975 	node = &ic->journal_tree[pos];
976 	if (unlikely(RB_EMPTY_NODE(&node->node)))
977 		return false;
978 	if (unlikely(node->sector != sector))
979 		return false;
980 
981 	next = rb_next(&node->node);
982 	if (unlikely(!next))
983 		return true;
984 
985 	next_node = container_of(next, struct journal_node, node);
986 	return next_node->sector != sector;
987 }
988 
989 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node)
990 {
991 	struct rb_node *next;
992 	struct journal_node *next_node;
993 	unsigned next_section;
994 
995 	BUG_ON(RB_EMPTY_NODE(&node->node));
996 
997 	next = rb_next(&node->node);
998 	if (unlikely(!next))
999 		return false;
1000 
1001 	next_node = container_of(next, struct journal_node, node);
1002 
1003 	if (next_node->sector != node->sector)
1004 		return false;
1005 
1006 	next_section = (unsigned)(next_node - ic->journal_tree) / ic->journal_section_entries;
1007 	if (next_section >= ic->committed_section &&
1008 	    next_section < ic->committed_section + ic->n_committed_sections)
1009 		return true;
1010 	if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections)
1011 		return true;
1012 
1013 	return false;
1014 }
1015 
1016 #define TAG_READ	0
1017 #define TAG_WRITE	1
1018 #define TAG_CMP		2
1019 
1020 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block,
1021 			       unsigned *metadata_offset, unsigned total_size, int op)
1022 {
1023 	do {
1024 		unsigned char *data, *dp;
1025 		struct dm_buffer *b;
1026 		unsigned to_copy;
1027 		int r;
1028 
1029 		r = dm_integrity_failed(ic);
1030 		if (unlikely(r))
1031 			return r;
1032 
1033 		data = dm_bufio_read(ic->bufio, *metadata_block, &b);
1034 		if (unlikely(IS_ERR(data)))
1035 			return PTR_ERR(data);
1036 
1037 		to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size);
1038 		dp = data + *metadata_offset;
1039 		if (op == TAG_READ) {
1040 			memcpy(tag, dp, to_copy);
1041 		} else if (op == TAG_WRITE) {
1042 			memcpy(dp, tag, to_copy);
1043 			dm_bufio_mark_buffer_dirty(b);
1044 		} else  {
1045 			/* e.g.: op == TAG_CMP */
1046 			if (unlikely(memcmp(dp, tag, to_copy))) {
1047 				unsigned i;
1048 
1049 				for (i = 0; i < to_copy; i++) {
1050 					if (dp[i] != tag[i])
1051 						break;
1052 					total_size--;
1053 				}
1054 				dm_bufio_release(b);
1055 				return total_size;
1056 			}
1057 		}
1058 		dm_bufio_release(b);
1059 
1060 		tag += to_copy;
1061 		*metadata_offset += to_copy;
1062 		if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) {
1063 			(*metadata_block)++;
1064 			*metadata_offset = 0;
1065 		}
1066 		total_size -= to_copy;
1067 	} while (unlikely(total_size));
1068 
1069 	return 0;
1070 }
1071 
1072 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic)
1073 {
1074 	int r;
1075 	r = dm_bufio_write_dirty_buffers(ic->bufio);
1076 	if (unlikely(r))
1077 		dm_integrity_io_error(ic, "writing tags", r);
1078 }
1079 
1080 static void sleep_on_endio_wait(struct dm_integrity_c *ic)
1081 {
1082 	DECLARE_WAITQUEUE(wait, current);
1083 	__add_wait_queue(&ic->endio_wait, &wait);
1084 	__set_current_state(TASK_UNINTERRUPTIBLE);
1085 	spin_unlock_irq(&ic->endio_wait.lock);
1086 	io_schedule();
1087 	spin_lock_irq(&ic->endio_wait.lock);
1088 	__remove_wait_queue(&ic->endio_wait, &wait);
1089 }
1090 
1091 static void autocommit_fn(unsigned long data)
1092 {
1093 	struct dm_integrity_c *ic = (struct dm_integrity_c *)data;
1094 
1095 	if (likely(!dm_integrity_failed(ic)))
1096 		queue_work(ic->commit_wq, &ic->commit_work);
1097 }
1098 
1099 static void schedule_autocommit(struct dm_integrity_c *ic)
1100 {
1101 	if (!timer_pending(&ic->autocommit_timer))
1102 		mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies);
1103 }
1104 
1105 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1106 {
1107 	struct bio *bio;
1108 	unsigned long flags;
1109 
1110 	spin_lock_irqsave(&ic->endio_wait.lock, flags);
1111 	bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1112 	bio_list_add(&ic->flush_bio_list, bio);
1113 	spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1114 
1115 	queue_work(ic->commit_wq, &ic->commit_work);
1116 }
1117 
1118 static void do_endio(struct dm_integrity_c *ic, struct bio *bio)
1119 {
1120 	int r = dm_integrity_failed(ic);
1121 	if (unlikely(r) && !bio->bi_status)
1122 		bio->bi_status = errno_to_blk_status(r);
1123 	bio_endio(bio);
1124 }
1125 
1126 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1127 {
1128 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1129 
1130 	if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))
1131 		submit_flush_bio(ic, dio);
1132 	else
1133 		do_endio(ic, bio);
1134 }
1135 
1136 static void dec_in_flight(struct dm_integrity_io *dio)
1137 {
1138 	if (atomic_dec_and_test(&dio->in_flight)) {
1139 		struct dm_integrity_c *ic = dio->ic;
1140 		struct bio *bio;
1141 
1142 		remove_range(ic, &dio->range);
1143 
1144 		if (unlikely(dio->write))
1145 			schedule_autocommit(ic);
1146 
1147 		bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1148 
1149 		if (unlikely(dio->bi_status) && !bio->bi_status)
1150 			bio->bi_status = dio->bi_status;
1151 		if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {
1152 			dio->range.logical_sector += dio->range.n_sectors;
1153 			bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);
1154 			INIT_WORK(&dio->work, integrity_bio_wait);
1155 			queue_work(ic->wait_wq, &dio->work);
1156 			return;
1157 		}
1158 		do_endio_flush(ic, dio);
1159 	}
1160 }
1161 
1162 static void integrity_end_io(struct bio *bio)
1163 {
1164 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1165 
1166 	bio->bi_iter = dio->orig_bi_iter;
1167 	bio->bi_bdev = dio->orig_bi_bdev;
1168 	if (dio->orig_bi_integrity) {
1169 		bio->bi_integrity = dio->orig_bi_integrity;
1170 		bio->bi_opf |= REQ_INTEGRITY;
1171 	}
1172 	bio->bi_end_io = dio->orig_bi_end_io;
1173 
1174 	if (dio->completion)
1175 		complete(dio->completion);
1176 
1177 	dec_in_flight(dio);
1178 }
1179 
1180 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector,
1181 				      const char *data, char *result)
1182 {
1183 	__u64 sector_le = cpu_to_le64(sector);
1184 	SHASH_DESC_ON_STACK(req, ic->internal_hash);
1185 	int r;
1186 	unsigned digest_size;
1187 
1188 	req->tfm = ic->internal_hash;
1189 	req->flags = 0;
1190 
1191 	r = crypto_shash_init(req);
1192 	if (unlikely(r < 0)) {
1193 		dm_integrity_io_error(ic, "crypto_shash_init", r);
1194 		goto failed;
1195 	}
1196 
1197 	r = crypto_shash_update(req, (const __u8 *)&sector_le, sizeof sector_le);
1198 	if (unlikely(r < 0)) {
1199 		dm_integrity_io_error(ic, "crypto_shash_update", r);
1200 		goto failed;
1201 	}
1202 
1203 	r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT);
1204 	if (unlikely(r < 0)) {
1205 		dm_integrity_io_error(ic, "crypto_shash_update", r);
1206 		goto failed;
1207 	}
1208 
1209 	r = crypto_shash_final(req, result);
1210 	if (unlikely(r < 0)) {
1211 		dm_integrity_io_error(ic, "crypto_shash_final", r);
1212 		goto failed;
1213 	}
1214 
1215 	digest_size = crypto_shash_digestsize(ic->internal_hash);
1216 	if (unlikely(digest_size < ic->tag_size))
1217 		memset(result + digest_size, 0, ic->tag_size - digest_size);
1218 
1219 	return;
1220 
1221 failed:
1222 	/* this shouldn't happen anyway, the hash functions have no reason to fail */
1223 	get_random_bytes(result, ic->tag_size);
1224 }
1225 
1226 static void integrity_metadata(struct work_struct *w)
1227 {
1228 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1229 	struct dm_integrity_c *ic = dio->ic;
1230 
1231 	int r;
1232 
1233 	if (ic->internal_hash) {
1234 		struct bvec_iter iter;
1235 		struct bio_vec bv;
1236 		unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1237 		struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1238 		char *checksums;
1239 		unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
1240 		char checksums_onstack[ic->tag_size + extra_space];
1241 		unsigned sectors_to_process = dio->range.n_sectors;
1242 		sector_t sector = dio->range.logical_sector;
1243 
1244 		if (unlikely(ic->mode == 'R'))
1245 			goto skip_io;
1246 
1247 		checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
1248 				    GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1249 		if (!checksums)
1250 			checksums = checksums_onstack;
1251 
1252 		__bio_for_each_segment(bv, bio, iter, dio->orig_bi_iter) {
1253 			unsigned pos;
1254 			char *mem, *checksums_ptr;
1255 
1256 again:
1257 			mem = (char *)kmap_atomic(bv.bv_page) + bv.bv_offset;
1258 			pos = 0;
1259 			checksums_ptr = checksums;
1260 			do {
1261 				integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr);
1262 				checksums_ptr += ic->tag_size;
1263 				sectors_to_process -= ic->sectors_per_block;
1264 				pos += ic->sectors_per_block << SECTOR_SHIFT;
1265 				sector += ic->sectors_per_block;
1266 			} while (pos < bv.bv_len && sectors_to_process && checksums != checksums_onstack);
1267 			kunmap_atomic(mem);
1268 
1269 			r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1270 						checksums_ptr - checksums, !dio->write ? TAG_CMP : TAG_WRITE);
1271 			if (unlikely(r)) {
1272 				if (r > 0) {
1273 					DMERR("Checksum failed at sector 0x%llx",
1274 					      (unsigned long long)(sector - ((r + ic->tag_size - 1) / ic->tag_size)));
1275 					r = -EILSEQ;
1276 				}
1277 				if (likely(checksums != checksums_onstack))
1278 					kfree(checksums);
1279 				goto error;
1280 			}
1281 
1282 			if (!sectors_to_process)
1283 				break;
1284 
1285 			if (unlikely(pos < bv.bv_len)) {
1286 				bv.bv_offset += pos;
1287 				bv.bv_len -= pos;
1288 				goto again;
1289 			}
1290 		}
1291 
1292 		if (likely(checksums != checksums_onstack))
1293 			kfree(checksums);
1294 	} else {
1295 		struct bio_integrity_payload *bip = dio->orig_bi_integrity;
1296 
1297 		if (bip) {
1298 			struct bio_vec biv;
1299 			struct bvec_iter iter;
1300 			unsigned data_to_process = dio->range.n_sectors;
1301 			sector_to_block(ic, data_to_process);
1302 			data_to_process *= ic->tag_size;
1303 
1304 			bip_for_each_vec(biv, bip, iter) {
1305 				unsigned char *tag;
1306 				unsigned this_len;
1307 
1308 				BUG_ON(PageHighMem(biv.bv_page));
1309 				tag = lowmem_page_address(biv.bv_page) + biv.bv_offset;
1310 				this_len = min(biv.bv_len, data_to_process);
1311 				r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,
1312 							this_len, !dio->write ? TAG_READ : TAG_WRITE);
1313 				if (unlikely(r))
1314 					goto error;
1315 				data_to_process -= this_len;
1316 				if (!data_to_process)
1317 					break;
1318 			}
1319 		}
1320 	}
1321 skip_io:
1322 	dec_in_flight(dio);
1323 	return;
1324 error:
1325 	dio->bi_status = errno_to_blk_status(r);
1326 	dec_in_flight(dio);
1327 }
1328 
1329 static int dm_integrity_map(struct dm_target *ti, struct bio *bio)
1330 {
1331 	struct dm_integrity_c *ic = ti->private;
1332 	struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1333 	struct bio_integrity_payload *bip;
1334 
1335 	sector_t area, offset;
1336 
1337 	dio->ic = ic;
1338 	dio->bi_status = 0;
1339 
1340 	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1341 		submit_flush_bio(ic, dio);
1342 		return DM_MAPIO_SUBMITTED;
1343 	}
1344 
1345 	dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1346 	dio->write = bio_op(bio) == REQ_OP_WRITE;
1347 	dio->fua = dio->write && bio->bi_opf & REQ_FUA;
1348 	if (unlikely(dio->fua)) {
1349 		/*
1350 		 * Don't pass down the FUA flag because we have to flush
1351 		 * disk cache anyway.
1352 		 */
1353 		bio->bi_opf &= ~REQ_FUA;
1354 	}
1355 	if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {
1356 		DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx",
1357 		      (unsigned long long)dio->range.logical_sector, bio_sectors(bio),
1358 		      (unsigned long long)ic->provided_data_sectors);
1359 		return DM_MAPIO_KILL;
1360 	}
1361 	if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1))) {
1362 		DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x",
1363 		      ic->sectors_per_block,
1364 		      (unsigned long long)dio->range.logical_sector, bio_sectors(bio));
1365 		return DM_MAPIO_KILL;
1366 	}
1367 
1368 	if (ic->sectors_per_block > 1) {
1369 		struct bvec_iter iter;
1370 		struct bio_vec bv;
1371 		bio_for_each_segment(bv, bio, iter) {
1372 			if (unlikely((bv.bv_offset | bv.bv_len) & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) {
1373 				DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary",
1374 					bv.bv_offset, bv.bv_len, ic->sectors_per_block);
1375 				return DM_MAPIO_KILL;
1376 			}
1377 		}
1378 	}
1379 
1380 	bip = bio_integrity(bio);
1381 	if (!ic->internal_hash) {
1382 		if (bip) {
1383 			unsigned wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block;
1384 			if (ic->log2_tag_size >= 0)
1385 				wanted_tag_size <<= ic->log2_tag_size;
1386 			else
1387 				wanted_tag_size *= ic->tag_size;
1388 			if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) {
1389 				DMERR("Invalid integrity data size %u, expected %u", bip->bip_iter.bi_size, wanted_tag_size);
1390 				return DM_MAPIO_KILL;
1391 			}
1392 		}
1393 	} else {
1394 		if (unlikely(bip != NULL)) {
1395 			DMERR("Unexpected integrity data when using internal hash");
1396 			return DM_MAPIO_KILL;
1397 		}
1398 	}
1399 
1400 	if (unlikely(ic->mode == 'R') && unlikely(dio->write))
1401 		return DM_MAPIO_KILL;
1402 
1403 	get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1404 	dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1405 	bio->bi_iter.bi_sector = get_data_sector(ic, area, offset);
1406 
1407 	dm_integrity_map_continue(dio, true);
1408 	return DM_MAPIO_SUBMITTED;
1409 }
1410 
1411 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,
1412 				 unsigned journal_section, unsigned journal_entry)
1413 {
1414 	struct dm_integrity_c *ic = dio->ic;
1415 	sector_t logical_sector;
1416 	unsigned n_sectors;
1417 
1418 	logical_sector = dio->range.logical_sector;
1419 	n_sectors = dio->range.n_sectors;
1420 	do {
1421 		struct bio_vec bv = bio_iovec(bio);
1422 		char *mem;
1423 
1424 		if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors))
1425 			bv.bv_len = n_sectors << SECTOR_SHIFT;
1426 		n_sectors -= bv.bv_len >> SECTOR_SHIFT;
1427 		bio_advance_iter(bio, &bio->bi_iter, bv.bv_len);
1428 retry_kmap:
1429 		mem = kmap_atomic(bv.bv_page);
1430 		if (likely(dio->write))
1431 			flush_dcache_page(bv.bv_page);
1432 
1433 		do {
1434 			struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry);
1435 
1436 			if (unlikely(!dio->write)) {
1437 				struct journal_sector *js;
1438 				char *mem_ptr;
1439 				unsigned s;
1440 
1441 				if (unlikely(journal_entry_is_inprogress(je))) {
1442 					flush_dcache_page(bv.bv_page);
1443 					kunmap_atomic(mem);
1444 
1445 					__io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
1446 					goto retry_kmap;
1447 				}
1448 				smp_rmb();
1449 				BUG_ON(journal_entry_get_sector(je) != logical_sector);
1450 				js = access_journal_data(ic, journal_section, journal_entry);
1451 				mem_ptr = mem + bv.bv_offset;
1452 				s = 0;
1453 				do {
1454 					memcpy(mem_ptr, js, JOURNAL_SECTOR_DATA);
1455 					*(commit_id_t *)(mem_ptr + JOURNAL_SECTOR_DATA) = je->last_bytes[s];
1456 					js++;
1457 					mem_ptr += 1 << SECTOR_SHIFT;
1458 				} while (++s < ic->sectors_per_block);
1459 #ifdef INTERNAL_VERIFY
1460 				if (ic->internal_hash) {
1461 					char checksums_onstack[max(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)];
1462 
1463 					integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack);
1464 					if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) {
1465 						DMERR("Checksum failed when reading from journal, at sector 0x%llx",
1466 						      (unsigned long long)logical_sector);
1467 					}
1468 				}
1469 #endif
1470 			}
1471 
1472 			if (!ic->internal_hash) {
1473 				struct bio_integrity_payload *bip = bio_integrity(bio);
1474 				unsigned tag_todo = ic->tag_size;
1475 				char *tag_ptr = journal_entry_tag(ic, je);
1476 
1477 				if (bip) do {
1478 					struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter);
1479 					unsigned tag_now = min(biv.bv_len, tag_todo);
1480 					char *tag_addr;
1481 					BUG_ON(PageHighMem(biv.bv_page));
1482 					tag_addr = lowmem_page_address(biv.bv_page) + biv.bv_offset;
1483 					if (likely(dio->write))
1484 						memcpy(tag_ptr, tag_addr, tag_now);
1485 					else
1486 						memcpy(tag_addr, tag_ptr, tag_now);
1487 					bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now);
1488 					tag_ptr += tag_now;
1489 					tag_todo -= tag_now;
1490 				} while (unlikely(tag_todo)); else {
1491 					if (likely(dio->write))
1492 						memset(tag_ptr, 0, tag_todo);
1493 				}
1494 			}
1495 
1496 			if (likely(dio->write)) {
1497 				struct journal_sector *js;
1498 				unsigned s;
1499 
1500 				js = access_journal_data(ic, journal_section, journal_entry);
1501 				memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT);
1502 
1503 				s = 0;
1504 				do {
1505 					je->last_bytes[s] = js[s].commit_id;
1506 				} while (++s < ic->sectors_per_block);
1507 
1508 				if (ic->internal_hash) {
1509 					unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1510 					if (unlikely(digest_size > ic->tag_size)) {
1511 						char checksums_onstack[digest_size];
1512 						integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack);
1513 						memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size);
1514 					} else
1515 						integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je));
1516 				}
1517 
1518 				journal_entry_set_sector(je, logical_sector);
1519 			}
1520 			logical_sector += ic->sectors_per_block;
1521 
1522 			journal_entry++;
1523 			if (unlikely(journal_entry == ic->journal_section_entries)) {
1524 				journal_entry = 0;
1525 				journal_section++;
1526 				wraparound_section(ic, &journal_section);
1527 			}
1528 
1529 			bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT;
1530 		} while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT);
1531 
1532 		if (unlikely(!dio->write))
1533 			flush_dcache_page(bv.bv_page);
1534 		kunmap_atomic(mem);
1535 	} while (n_sectors);
1536 
1537 	if (likely(dio->write)) {
1538 		smp_mb();
1539 		if (unlikely(waitqueue_active(&ic->copy_to_journal_wait)))
1540 			wake_up(&ic->copy_to_journal_wait);
1541 		if (ACCESS_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) {
1542 			queue_work(ic->commit_wq, &ic->commit_work);
1543 		} else {
1544 			schedule_autocommit(ic);
1545 		}
1546 	} else {
1547 		remove_range(ic, &dio->range);
1548 	}
1549 
1550 	if (unlikely(bio->bi_iter.bi_size)) {
1551 		sector_t area, offset;
1552 
1553 		dio->range.logical_sector = logical_sector;
1554 		get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1555 		dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1556 		return true;
1557 	}
1558 
1559 	return false;
1560 }
1561 
1562 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)
1563 {
1564 	struct dm_integrity_c *ic = dio->ic;
1565 	struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1566 	unsigned journal_section, journal_entry;
1567 	unsigned journal_read_pos;
1568 	struct completion read_comp;
1569 	bool need_sync_io = ic->internal_hash && !dio->write;
1570 
1571 	if (need_sync_io && from_map) {
1572 		INIT_WORK(&dio->work, integrity_bio_wait);
1573 		queue_work(ic->metadata_wq, &dio->work);
1574 		return;
1575 	}
1576 
1577 lock_retry:
1578 	spin_lock_irq(&ic->endio_wait.lock);
1579 retry:
1580 	if (unlikely(dm_integrity_failed(ic))) {
1581 		spin_unlock_irq(&ic->endio_wait.lock);
1582 		do_endio(ic, bio);
1583 		return;
1584 	}
1585 	dio->range.n_sectors = bio_sectors(bio);
1586 	journal_read_pos = NOT_FOUND;
1587 	if (likely(ic->mode == 'J')) {
1588 		if (dio->write) {
1589 			unsigned next_entry, i, pos;
1590 			unsigned ws, we;
1591 
1592 			dio->range.n_sectors = min(dio->range.n_sectors, ic->free_sectors);
1593 			if (unlikely(!dio->range.n_sectors))
1594 				goto sleep;
1595 			ic->free_sectors -= dio->range.n_sectors;
1596 			journal_section = ic->free_section;
1597 			journal_entry = ic->free_section_entry;
1598 
1599 			next_entry = ic->free_section_entry + dio->range.n_sectors;
1600 			ic->free_section_entry = next_entry % ic->journal_section_entries;
1601 			ic->free_section += next_entry / ic->journal_section_entries;
1602 			ic->n_uncommitted_sections += next_entry / ic->journal_section_entries;
1603 			wraparound_section(ic, &ic->free_section);
1604 
1605 			pos = journal_section * ic->journal_section_entries + journal_entry;
1606 			ws = journal_section;
1607 			we = journal_entry;
1608 			i = 0;
1609 			do {
1610 				struct journal_entry *je;
1611 
1612 				add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);
1613 				pos++;
1614 				if (unlikely(pos >= ic->journal_entries))
1615 					pos = 0;
1616 
1617 				je = access_journal_entry(ic, ws, we);
1618 				BUG_ON(!journal_entry_is_unused(je));
1619 				journal_entry_set_inprogress(je);
1620 				we++;
1621 				if (unlikely(we == ic->journal_section_entries)) {
1622 					we = 0;
1623 					ws++;
1624 					wraparound_section(ic, &ws);
1625 				}
1626 			} while ((i += ic->sectors_per_block) < dio->range.n_sectors);
1627 
1628 			spin_unlock_irq(&ic->endio_wait.lock);
1629 			goto journal_read_write;
1630 		} else {
1631 			sector_t next_sector;
1632 			journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
1633 			if (likely(journal_read_pos == NOT_FOUND)) {
1634 				if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))
1635 					dio->range.n_sectors = next_sector - dio->range.logical_sector;
1636 			} else {
1637 				unsigned i;
1638 				unsigned jp = journal_read_pos + 1;
1639 				for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {
1640 					if (!test_journal_node(ic, jp, dio->range.logical_sector + i))
1641 						break;
1642 				}
1643 				dio->range.n_sectors = i;
1644 			}
1645 		}
1646 	}
1647 	if (unlikely(!add_new_range(ic, &dio->range))) {
1648 		/*
1649 		 * We must not sleep in the request routine because it could
1650 		 * stall bios on current->bio_list.
1651 		 * So, we offload the bio to a workqueue if we have to sleep.
1652 		 */
1653 sleep:
1654 		if (from_map) {
1655 			spin_unlock_irq(&ic->endio_wait.lock);
1656 			INIT_WORK(&dio->work, integrity_bio_wait);
1657 			queue_work(ic->wait_wq, &dio->work);
1658 			return;
1659 		} else {
1660 			sleep_on_endio_wait(ic);
1661 			goto retry;
1662 		}
1663 	}
1664 	spin_unlock_irq(&ic->endio_wait.lock);
1665 
1666 	if (unlikely(journal_read_pos != NOT_FOUND)) {
1667 		journal_section = journal_read_pos / ic->journal_section_entries;
1668 		journal_entry = journal_read_pos % ic->journal_section_entries;
1669 		goto journal_read_write;
1670 	}
1671 
1672 	dio->in_flight = (atomic_t)ATOMIC_INIT(2);
1673 
1674 	if (need_sync_io) {
1675 		read_comp = COMPLETION_INITIALIZER_ONSTACK(read_comp);
1676 		dio->completion = &read_comp;
1677 	} else
1678 		dio->completion = NULL;
1679 
1680 	dio->orig_bi_iter = bio->bi_iter;
1681 
1682 	dio->orig_bi_bdev = bio->bi_bdev;
1683 	bio->bi_bdev = ic->dev->bdev;
1684 
1685 	dio->orig_bi_integrity = bio_integrity(bio);
1686 	bio->bi_integrity = NULL;
1687 	bio->bi_opf &= ~REQ_INTEGRITY;
1688 
1689 	dio->orig_bi_end_io = bio->bi_end_io;
1690 	bio->bi_end_io = integrity_end_io;
1691 
1692 	bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
1693 	bio->bi_iter.bi_sector += ic->start;
1694 	generic_make_request(bio);
1695 
1696 	if (need_sync_io) {
1697 		wait_for_completion_io(&read_comp);
1698 		integrity_metadata(&dio->work);
1699 	} else {
1700 		INIT_WORK(&dio->work, integrity_metadata);
1701 		queue_work(ic->metadata_wq, &dio->work);
1702 	}
1703 
1704 	return;
1705 
1706 journal_read_write:
1707 	if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))
1708 		goto lock_retry;
1709 
1710 	do_endio_flush(ic, dio);
1711 }
1712 
1713 
1714 static void integrity_bio_wait(struct work_struct *w)
1715 {
1716 	struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1717 
1718 	dm_integrity_map_continue(dio, false);
1719 }
1720 
1721 static void pad_uncommitted(struct dm_integrity_c *ic)
1722 {
1723 	if (ic->free_section_entry) {
1724 		ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry;
1725 		ic->free_section_entry = 0;
1726 		ic->free_section++;
1727 		wraparound_section(ic, &ic->free_section);
1728 		ic->n_uncommitted_sections++;
1729 	}
1730 }
1731 
1732 static void integrity_commit(struct work_struct *w)
1733 {
1734 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work);
1735 	unsigned commit_start, commit_sections;
1736 	unsigned i, j, n;
1737 	struct bio *flushes;
1738 
1739 	del_timer(&ic->autocommit_timer);
1740 
1741 	spin_lock_irq(&ic->endio_wait.lock);
1742 	flushes = bio_list_get(&ic->flush_bio_list);
1743 	if (unlikely(ic->mode != 'J')) {
1744 		spin_unlock_irq(&ic->endio_wait.lock);
1745 		dm_integrity_flush_buffers(ic);
1746 		goto release_flush_bios;
1747 	}
1748 
1749 	pad_uncommitted(ic);
1750 	commit_start = ic->uncommitted_section;
1751 	commit_sections = ic->n_uncommitted_sections;
1752 	spin_unlock_irq(&ic->endio_wait.lock);
1753 
1754 	if (!commit_sections)
1755 		goto release_flush_bios;
1756 
1757 	i = commit_start;
1758 	for (n = 0; n < commit_sections; n++) {
1759 		for (j = 0; j < ic->journal_section_entries; j++) {
1760 			struct journal_entry *je;
1761 			je = access_journal_entry(ic, i, j);
1762 			io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
1763 		}
1764 		for (j = 0; j < ic->journal_section_sectors; j++) {
1765 			struct journal_sector *js;
1766 			js = access_journal(ic, i, j);
1767 			js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq);
1768 		}
1769 		i++;
1770 		if (unlikely(i >= ic->journal_sections))
1771 			ic->commit_seq = next_commit_seq(ic->commit_seq);
1772 		wraparound_section(ic, &i);
1773 	}
1774 	smp_rmb();
1775 
1776 	write_journal(ic, commit_start, commit_sections);
1777 
1778 	spin_lock_irq(&ic->endio_wait.lock);
1779 	ic->uncommitted_section += commit_sections;
1780 	wraparound_section(ic, &ic->uncommitted_section);
1781 	ic->n_uncommitted_sections -= commit_sections;
1782 	ic->n_committed_sections += commit_sections;
1783 	spin_unlock_irq(&ic->endio_wait.lock);
1784 
1785 	if (ACCESS_ONCE(ic->free_sectors) <= ic->free_sectors_threshold)
1786 		queue_work(ic->writer_wq, &ic->writer_work);
1787 
1788 release_flush_bios:
1789 	while (flushes) {
1790 		struct bio *next = flushes->bi_next;
1791 		flushes->bi_next = NULL;
1792 		do_endio(ic, flushes);
1793 		flushes = next;
1794 	}
1795 }
1796 
1797 static void complete_copy_from_journal(unsigned long error, void *context)
1798 {
1799 	struct journal_io *io = context;
1800 	struct journal_completion *comp = io->comp;
1801 	struct dm_integrity_c *ic = comp->ic;
1802 	remove_range(ic, &io->range);
1803 	mempool_free(io, ic->journal_io_mempool);
1804 	if (unlikely(error != 0))
1805 		dm_integrity_io_error(ic, "copying from journal", -EIO);
1806 	complete_journal_op(comp);
1807 }
1808 
1809 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js,
1810 			       struct journal_entry *je)
1811 {
1812 	unsigned s = 0;
1813 	do {
1814 		js->commit_id = je->last_bytes[s];
1815 		js++;
1816 	} while (++s < ic->sectors_per_block);
1817 }
1818 
1819 static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start,
1820 			     unsigned write_sections, bool from_replay)
1821 {
1822 	unsigned i, j, n;
1823 	struct journal_completion comp;
1824 
1825 	comp.ic = ic;
1826 	comp.in_flight = (atomic_t)ATOMIC_INIT(1);
1827 	comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp);
1828 
1829 	i = write_start;
1830 	for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) {
1831 #ifndef INTERNAL_VERIFY
1832 		if (unlikely(from_replay))
1833 #endif
1834 			rw_section_mac(ic, i, false);
1835 		for (j = 0; j < ic->journal_section_entries; j++) {
1836 			struct journal_entry *je = access_journal_entry(ic, i, j);
1837 			sector_t sec, area, offset;
1838 			unsigned k, l, next_loop;
1839 			sector_t metadata_block;
1840 			unsigned metadata_offset;
1841 			struct journal_io *io;
1842 
1843 			if (journal_entry_is_unused(je))
1844 				continue;
1845 			BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay);
1846 			sec = journal_entry_get_sector(je);
1847 			if (unlikely(from_replay)) {
1848 				if (unlikely(sec & (unsigned)(ic->sectors_per_block - 1))) {
1849 					dm_integrity_io_error(ic, "invalid sector in journal", -EIO);
1850 					sec &= ~(sector_t)(ic->sectors_per_block - 1);
1851 				}
1852 			}
1853 			get_area_and_offset(ic, sec, &area, &offset);
1854 			restore_last_bytes(ic, access_journal_data(ic, i, j), je);
1855 			for (k = j + 1; k < ic->journal_section_entries; k++) {
1856 				struct journal_entry *je2 = access_journal_entry(ic, i, k);
1857 				sector_t sec2, area2, offset2;
1858 				if (journal_entry_is_unused(je2))
1859 					break;
1860 				BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay);
1861 				sec2 = journal_entry_get_sector(je2);
1862 				get_area_and_offset(ic, sec2, &area2, &offset2);
1863 				if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block))
1864 					break;
1865 				restore_last_bytes(ic, access_journal_data(ic, i, k), je2);
1866 			}
1867 			next_loop = k - 1;
1868 
1869 			io = mempool_alloc(ic->journal_io_mempool, GFP_NOIO);
1870 			io->comp = &comp;
1871 			io->range.logical_sector = sec;
1872 			io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block;
1873 
1874 			spin_lock_irq(&ic->endio_wait.lock);
1875 			while (unlikely(!add_new_range(ic, &io->range)))
1876 				sleep_on_endio_wait(ic);
1877 
1878 			if (likely(!from_replay)) {
1879 				struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries];
1880 
1881 				/* don't write if there is newer committed sector */
1882 				while (j < k && find_newer_committed_node(ic, &section_node[j])) {
1883 					struct journal_entry *je2 = access_journal_entry(ic, i, j);
1884 
1885 					journal_entry_set_unused(je2);
1886 					remove_journal_node(ic, &section_node[j]);
1887 					j++;
1888 					sec += ic->sectors_per_block;
1889 					offset += ic->sectors_per_block;
1890 				}
1891 				while (j < k && find_newer_committed_node(ic, &section_node[k - 1])) {
1892 					struct journal_entry *je2 = access_journal_entry(ic, i, k - 1);
1893 
1894 					journal_entry_set_unused(je2);
1895 					remove_journal_node(ic, &section_node[k - 1]);
1896 					k--;
1897 				}
1898 				if (j == k) {
1899 					remove_range_unlocked(ic, &io->range);
1900 					spin_unlock_irq(&ic->endio_wait.lock);
1901 					mempool_free(io, ic->journal_io_mempool);
1902 					goto skip_io;
1903 				}
1904 				for (l = j; l < k; l++) {
1905 					remove_journal_node(ic, &section_node[l]);
1906 				}
1907 			}
1908 			spin_unlock_irq(&ic->endio_wait.lock);
1909 
1910 			metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
1911 			for (l = j; l < k; l++) {
1912 				int r;
1913 				struct journal_entry *je2 = access_journal_entry(ic, i, l);
1914 
1915 				if (
1916 #ifndef INTERNAL_VERIFY
1917 				    unlikely(from_replay) &&
1918 #endif
1919 				    ic->internal_hash) {
1920 					char test_tag[max(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)];
1921 
1922 					integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block),
1923 								  (char *)access_journal_data(ic, i, l), test_tag);
1924 					if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size)))
1925 						dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ);
1926 				}
1927 
1928 				journal_entry_set_unused(je2);
1929 				r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset,
1930 							ic->tag_size, TAG_WRITE);
1931 				if (unlikely(r)) {
1932 					dm_integrity_io_error(ic, "reading tags", r);
1933 				}
1934 			}
1935 
1936 			atomic_inc(&comp.in_flight);
1937 			copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block,
1938 					  (k - j) << ic->sb->log2_sectors_per_block,
1939 					  get_data_sector(ic, area, offset),
1940 					  complete_copy_from_journal, io);
1941 skip_io:
1942 			j = next_loop;
1943 		}
1944 	}
1945 
1946 	dm_bufio_write_dirty_buffers_async(ic->bufio);
1947 
1948 	complete_journal_op(&comp);
1949 	wait_for_completion_io(&comp.comp);
1950 
1951 	dm_integrity_flush_buffers(ic);
1952 }
1953 
1954 static void integrity_writer(struct work_struct *w)
1955 {
1956 	struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work);
1957 	unsigned write_start, write_sections;
1958 
1959 	unsigned prev_free_sectors;
1960 
1961 	/* the following test is not needed, but it tests the replay code */
1962 	if (ACCESS_ONCE(ic->suspending))
1963 		return;
1964 
1965 	spin_lock_irq(&ic->endio_wait.lock);
1966 	write_start = ic->committed_section;
1967 	write_sections = ic->n_committed_sections;
1968 	spin_unlock_irq(&ic->endio_wait.lock);
1969 
1970 	if (!write_sections)
1971 		return;
1972 
1973 	do_journal_write(ic, write_start, write_sections, false);
1974 
1975 	spin_lock_irq(&ic->endio_wait.lock);
1976 
1977 	ic->committed_section += write_sections;
1978 	wraparound_section(ic, &ic->committed_section);
1979 	ic->n_committed_sections -= write_sections;
1980 
1981 	prev_free_sectors = ic->free_sectors;
1982 	ic->free_sectors += write_sections * ic->journal_section_entries;
1983 	if (unlikely(!prev_free_sectors))
1984 		wake_up_locked(&ic->endio_wait);
1985 
1986 	spin_unlock_irq(&ic->endio_wait.lock);
1987 }
1988 
1989 static void init_journal(struct dm_integrity_c *ic, unsigned start_section,
1990 			 unsigned n_sections, unsigned char commit_seq)
1991 {
1992 	unsigned i, j, n;
1993 
1994 	if (!n_sections)
1995 		return;
1996 
1997 	for (n = 0; n < n_sections; n++) {
1998 		i = start_section + n;
1999 		wraparound_section(ic, &i);
2000 		for (j = 0; j < ic->journal_section_sectors; j++) {
2001 			struct journal_sector *js = access_journal(ic, i, j);
2002 			memset(&js->entries, 0, JOURNAL_SECTOR_DATA);
2003 			js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq);
2004 		}
2005 		for (j = 0; j < ic->journal_section_entries; j++) {
2006 			struct journal_entry *je = access_journal_entry(ic, i, j);
2007 			journal_entry_set_unused(je);
2008 		}
2009 	}
2010 
2011 	write_journal(ic, start_section, n_sections);
2012 }
2013 
2014 static int find_commit_seq(struct dm_integrity_c *ic, unsigned i, unsigned j, commit_id_t id)
2015 {
2016 	unsigned char k;
2017 	for (k = 0; k < N_COMMIT_IDS; k++) {
2018 		if (dm_integrity_commit_id(ic, i, j, k) == id)
2019 			return k;
2020 	}
2021 	dm_integrity_io_error(ic, "journal commit id", -EIO);
2022 	return -EIO;
2023 }
2024 
2025 static void replay_journal(struct dm_integrity_c *ic)
2026 {
2027 	unsigned i, j;
2028 	bool used_commit_ids[N_COMMIT_IDS];
2029 	unsigned max_commit_id_sections[N_COMMIT_IDS];
2030 	unsigned write_start, write_sections;
2031 	unsigned continue_section;
2032 	bool journal_empty;
2033 	unsigned char unused, last_used, want_commit_seq;
2034 
2035 	if (ic->mode == 'R')
2036 		return;
2037 
2038 	if (ic->journal_uptodate)
2039 		return;
2040 
2041 	last_used = 0;
2042 	write_start = 0;
2043 
2044 	if (!ic->just_formatted) {
2045 		DEBUG_print("reading journal\n");
2046 		rw_journal(ic, REQ_OP_READ, 0, 0, ic->journal_sections, NULL);
2047 		if (ic->journal_io)
2048 			DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal");
2049 		if (ic->journal_io) {
2050 			struct journal_completion crypt_comp;
2051 			crypt_comp.ic = ic;
2052 			crypt_comp.comp = COMPLETION_INITIALIZER_ONSTACK(crypt_comp.comp);
2053 			crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0);
2054 			encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp);
2055 			wait_for_completion(&crypt_comp.comp);
2056 		}
2057 		DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal");
2058 	}
2059 
2060 	if (dm_integrity_failed(ic))
2061 		goto clear_journal;
2062 
2063 	journal_empty = true;
2064 	memset(used_commit_ids, 0, sizeof used_commit_ids);
2065 	memset(max_commit_id_sections, 0, sizeof max_commit_id_sections);
2066 	for (i = 0; i < ic->journal_sections; i++) {
2067 		for (j = 0; j < ic->journal_section_sectors; j++) {
2068 			int k;
2069 			struct journal_sector *js = access_journal(ic, i, j);
2070 			k = find_commit_seq(ic, i, j, js->commit_id);
2071 			if (k < 0)
2072 				goto clear_journal;
2073 			used_commit_ids[k] = true;
2074 			max_commit_id_sections[k] = i;
2075 		}
2076 		if (journal_empty) {
2077 			for (j = 0; j < ic->journal_section_entries; j++) {
2078 				struct journal_entry *je = access_journal_entry(ic, i, j);
2079 				if (!journal_entry_is_unused(je)) {
2080 					journal_empty = false;
2081 					break;
2082 				}
2083 			}
2084 		}
2085 	}
2086 
2087 	if (!used_commit_ids[N_COMMIT_IDS - 1]) {
2088 		unused = N_COMMIT_IDS - 1;
2089 		while (unused && !used_commit_ids[unused - 1])
2090 			unused--;
2091 	} else {
2092 		for (unused = 0; unused < N_COMMIT_IDS; unused++)
2093 			if (!used_commit_ids[unused])
2094 				break;
2095 		if (unused == N_COMMIT_IDS) {
2096 			dm_integrity_io_error(ic, "journal commit ids", -EIO);
2097 			goto clear_journal;
2098 		}
2099 	}
2100 	DEBUG_print("first unused commit seq %d [%d,%d,%d,%d]\n",
2101 		    unused, used_commit_ids[0], used_commit_ids[1],
2102 		    used_commit_ids[2], used_commit_ids[3]);
2103 
2104 	last_used = prev_commit_seq(unused);
2105 	want_commit_seq = prev_commit_seq(last_used);
2106 
2107 	if (!used_commit_ids[want_commit_seq] && used_commit_ids[prev_commit_seq(want_commit_seq)])
2108 		journal_empty = true;
2109 
2110 	write_start = max_commit_id_sections[last_used] + 1;
2111 	if (unlikely(write_start >= ic->journal_sections))
2112 		want_commit_seq = next_commit_seq(want_commit_seq);
2113 	wraparound_section(ic, &write_start);
2114 
2115 	i = write_start;
2116 	for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) {
2117 		for (j = 0; j < ic->journal_section_sectors; j++) {
2118 			struct journal_sector *js = access_journal(ic, i, j);
2119 
2120 			if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) {
2121 				/*
2122 				 * This could be caused by crash during writing.
2123 				 * We won't replay the inconsistent part of the
2124 				 * journal.
2125 				 */
2126 				DEBUG_print("commit id mismatch at position (%u, %u): %d != %d\n",
2127 					    i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq);
2128 				goto brk;
2129 			}
2130 		}
2131 		i++;
2132 		if (unlikely(i >= ic->journal_sections))
2133 			want_commit_seq = next_commit_seq(want_commit_seq);
2134 		wraparound_section(ic, &i);
2135 	}
2136 brk:
2137 
2138 	if (!journal_empty) {
2139 		DEBUG_print("replaying %u sections, starting at %u, commit seq %d\n",
2140 			    write_sections, write_start, want_commit_seq);
2141 		do_journal_write(ic, write_start, write_sections, true);
2142 	}
2143 
2144 	if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) {
2145 		continue_section = write_start;
2146 		ic->commit_seq = want_commit_seq;
2147 		DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq);
2148 	} else {
2149 		unsigned s;
2150 		unsigned char erase_seq;
2151 clear_journal:
2152 		DEBUG_print("clearing journal\n");
2153 
2154 		erase_seq = prev_commit_seq(prev_commit_seq(last_used));
2155 		s = write_start;
2156 		init_journal(ic, s, 1, erase_seq);
2157 		s++;
2158 		wraparound_section(ic, &s);
2159 		if (ic->journal_sections >= 2) {
2160 			init_journal(ic, s, ic->journal_sections - 2, erase_seq);
2161 			s += ic->journal_sections - 2;
2162 			wraparound_section(ic, &s);
2163 			init_journal(ic, s, 1, erase_seq);
2164 		}
2165 
2166 		continue_section = 0;
2167 		ic->commit_seq = next_commit_seq(erase_seq);
2168 	}
2169 
2170 	ic->committed_section = continue_section;
2171 	ic->n_committed_sections = 0;
2172 
2173 	ic->uncommitted_section = continue_section;
2174 	ic->n_uncommitted_sections = 0;
2175 
2176 	ic->free_section = continue_section;
2177 	ic->free_section_entry = 0;
2178 	ic->free_sectors = ic->journal_entries;
2179 
2180 	ic->journal_tree_root = RB_ROOT;
2181 	for (i = 0; i < ic->journal_entries; i++)
2182 		init_journal_node(&ic->journal_tree[i]);
2183 }
2184 
2185 static void dm_integrity_postsuspend(struct dm_target *ti)
2186 {
2187 	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2188 
2189 	del_timer_sync(&ic->autocommit_timer);
2190 
2191 	ic->suspending = true;
2192 
2193 	queue_work(ic->commit_wq, &ic->commit_work);
2194 	drain_workqueue(ic->commit_wq);
2195 
2196 	if (ic->mode == 'J') {
2197 		drain_workqueue(ic->writer_wq);
2198 		dm_integrity_flush_buffers(ic);
2199 	}
2200 
2201 	ic->suspending = false;
2202 
2203 	BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
2204 
2205 	ic->journal_uptodate = true;
2206 }
2207 
2208 static void dm_integrity_resume(struct dm_target *ti)
2209 {
2210 	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2211 
2212 	replay_journal(ic);
2213 }
2214 
2215 static void dm_integrity_status(struct dm_target *ti, status_type_t type,
2216 				unsigned status_flags, char *result, unsigned maxlen)
2217 {
2218 	struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private;
2219 	unsigned arg_count;
2220 	size_t sz = 0;
2221 
2222 	switch (type) {
2223 	case STATUSTYPE_INFO:
2224 		result[0] = '\0';
2225 		break;
2226 
2227 	case STATUSTYPE_TABLE: {
2228 		__u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100;
2229 		watermark_percentage += ic->journal_entries / 2;
2230 		do_div(watermark_percentage, ic->journal_entries);
2231 		arg_count = 5;
2232 		arg_count += ic->sectors_per_block != 1;
2233 		arg_count += !!ic->internal_hash_alg.alg_string;
2234 		arg_count += !!ic->journal_crypt_alg.alg_string;
2235 		arg_count += !!ic->journal_mac_alg.alg_string;
2236 		DMEMIT("%s %llu %u %c %u", ic->dev->name, (unsigned long long)ic->start,
2237 		       ic->tag_size, ic->mode, arg_count);
2238 		DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS);
2239 		DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors);
2240 		DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
2241 		DMEMIT(" journal_watermark:%u", (unsigned)watermark_percentage);
2242 		DMEMIT(" commit_time:%u", ic->autocommit_msec);
2243 		if (ic->sectors_per_block != 1)
2244 			DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT);
2245 
2246 #define EMIT_ALG(a, n)							\
2247 		do {							\
2248 			if (ic->a.alg_string) {				\
2249 				DMEMIT(" %s:%s", n, ic->a.alg_string);	\
2250 				if (ic->a.key_string)			\
2251 					DMEMIT(":%s", ic->a.key_string);\
2252 			}						\
2253 		} while (0)
2254 		EMIT_ALG(internal_hash_alg, "internal_hash");
2255 		EMIT_ALG(journal_crypt_alg, "journal_crypt");
2256 		EMIT_ALG(journal_mac_alg, "journal_mac");
2257 		break;
2258 	}
2259 	}
2260 }
2261 
2262 static int dm_integrity_iterate_devices(struct dm_target *ti,
2263 					iterate_devices_callout_fn fn, void *data)
2264 {
2265 	struct dm_integrity_c *ic = ti->private;
2266 
2267 	return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data);
2268 }
2269 
2270 static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits)
2271 {
2272 	struct dm_integrity_c *ic = ti->private;
2273 
2274 	if (ic->sectors_per_block > 1) {
2275 		limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
2276 		limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
2277 		blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT);
2278 	}
2279 }
2280 
2281 static void calculate_journal_section_size(struct dm_integrity_c *ic)
2282 {
2283 	unsigned sector_space = JOURNAL_SECTOR_DATA;
2284 
2285 	ic->journal_sections = le32_to_cpu(ic->sb->journal_sections);
2286 	ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size,
2287 					 JOURNAL_ENTRY_ROUNDUP);
2288 
2289 	if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC))
2290 		sector_space -= JOURNAL_MAC_PER_SECTOR;
2291 	ic->journal_entries_per_sector = sector_space / ic->journal_entry_size;
2292 	ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS;
2293 	ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JOURNAL_BLOCK_SECTORS;
2294 	ic->journal_entries = ic->journal_section_entries * ic->journal_sections;
2295 }
2296 
2297 static int calculate_device_limits(struct dm_integrity_c *ic)
2298 {
2299 	__u64 initial_sectors;
2300 	sector_t last_sector, last_area, last_offset;
2301 
2302 	calculate_journal_section_size(ic);
2303 	initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections;
2304 	if (initial_sectors + METADATA_PADDING_SECTORS >= ic->device_sectors || initial_sectors > UINT_MAX)
2305 		return -EINVAL;
2306 	ic->initial_sectors = initial_sectors;
2307 
2308 	ic->metadata_run = roundup((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block),
2309 				   (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS)) >> SECTOR_SHIFT;
2310 	if (!(ic->metadata_run & (ic->metadata_run - 1)))
2311 		ic->log2_metadata_run = __ffs(ic->metadata_run);
2312 	else
2313 		ic->log2_metadata_run = -1;
2314 
2315 	get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset);
2316 	last_sector = get_data_sector(ic, last_area, last_offset);
2317 
2318 	if (ic->start + last_sector < last_sector || ic->start + last_sector >= ic->device_sectors)
2319 		return -EINVAL;
2320 
2321 	return 0;
2322 }
2323 
2324 static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sectors, unsigned interleave_sectors)
2325 {
2326 	unsigned journal_sections;
2327 	int test_bit;
2328 
2329 	memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT);
2330 	memcpy(ic->sb->magic, SB_MAGIC, 8);
2331 	ic->sb->version = SB_VERSION;
2332 	ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size);
2333 	ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block);
2334 	if (ic->journal_mac_alg.alg_string)
2335 		ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC);
2336 
2337 	calculate_journal_section_size(ic);
2338 	journal_sections = journal_sectors / ic->journal_section_sectors;
2339 	if (!journal_sections)
2340 		journal_sections = 1;
2341 	ic->sb->journal_sections = cpu_to_le32(journal_sections);
2342 
2343 	if (!interleave_sectors)
2344 		interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
2345 	ic->sb->log2_interleave_sectors = __fls(interleave_sectors);
2346 	ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
2347 	ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
2348 
2349 	ic->provided_data_sectors = 0;
2350 	for (test_bit = fls64(ic->device_sectors) - 1; test_bit >= 3; test_bit--) {
2351 		__u64 prev_data_sectors = ic->provided_data_sectors;
2352 
2353 		ic->provided_data_sectors |= (sector_t)1 << test_bit;
2354 		if (calculate_device_limits(ic))
2355 			ic->provided_data_sectors = prev_data_sectors;
2356 	}
2357 
2358 	if (!ic->provided_data_sectors)
2359 		return -EINVAL;
2360 
2361 	ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
2362 
2363 	return 0;
2364 }
2365 
2366 static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic)
2367 {
2368 	struct gendisk *disk = dm_disk(dm_table_get_md(ti->table));
2369 	struct blk_integrity bi;
2370 
2371 	memset(&bi, 0, sizeof(bi));
2372 	bi.profile = &dm_integrity_profile;
2373 	bi.tuple_size = ic->tag_size;
2374 	bi.tag_size = bi.tuple_size;
2375 	bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT;
2376 
2377 	blk_integrity_register(disk, &bi);
2378 	blk_queue_max_integrity_segments(disk->queue, UINT_MAX);
2379 }
2380 
2381 static void dm_integrity_free_page_list(struct dm_integrity_c *ic, struct page_list *pl)
2382 {
2383 	unsigned i;
2384 
2385 	if (!pl)
2386 		return;
2387 	for (i = 0; i < ic->journal_pages; i++)
2388 		if (pl[i].page)
2389 			__free_page(pl[i].page);
2390 	kvfree(pl);
2391 }
2392 
2393 static struct page_list *dm_integrity_alloc_page_list(struct dm_integrity_c *ic)
2394 {
2395 	size_t page_list_desc_size = ic->journal_pages * sizeof(struct page_list);
2396 	struct page_list *pl;
2397 	unsigned i;
2398 
2399 	pl = kvmalloc(page_list_desc_size, GFP_KERNEL | __GFP_ZERO);
2400 	if (!pl)
2401 		return NULL;
2402 
2403 	for (i = 0; i < ic->journal_pages; i++) {
2404 		pl[i].page = alloc_page(GFP_KERNEL);
2405 		if (!pl[i].page) {
2406 			dm_integrity_free_page_list(ic, pl);
2407 			return NULL;
2408 		}
2409 		if (i)
2410 			pl[i - 1].next = &pl[i];
2411 	}
2412 
2413 	return pl;
2414 }
2415 
2416 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl)
2417 {
2418 	unsigned i;
2419 	for (i = 0; i < ic->journal_sections; i++)
2420 		kvfree(sl[i]);
2421 	kfree(sl);
2422 }
2423 
2424 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, struct page_list *pl)
2425 {
2426 	struct scatterlist **sl;
2427 	unsigned i;
2428 
2429 	sl = kvmalloc(ic->journal_sections * sizeof(struct scatterlist *), GFP_KERNEL | __GFP_ZERO);
2430 	if (!sl)
2431 		return NULL;
2432 
2433 	for (i = 0; i < ic->journal_sections; i++) {
2434 		struct scatterlist *s;
2435 		unsigned start_index, start_offset;
2436 		unsigned end_index, end_offset;
2437 		unsigned n_pages;
2438 		unsigned idx;
2439 
2440 		page_list_location(ic, i, 0, &start_index, &start_offset);
2441 		page_list_location(ic, i, ic->journal_section_sectors - 1, &end_index, &end_offset);
2442 
2443 		n_pages = (end_index - start_index + 1);
2444 
2445 		s = kvmalloc(n_pages * sizeof(struct scatterlist), GFP_KERNEL);
2446 		if (!s) {
2447 			dm_integrity_free_journal_scatterlist(ic, sl);
2448 			return NULL;
2449 		}
2450 
2451 		sg_init_table(s, n_pages);
2452 		for (idx = start_index; idx <= end_index; idx++) {
2453 			char *va = lowmem_page_address(pl[idx].page);
2454 			unsigned start = 0, end = PAGE_SIZE;
2455 			if (idx == start_index)
2456 				start = start_offset;
2457 			if (idx == end_index)
2458 				end = end_offset + (1 << SECTOR_SHIFT);
2459 			sg_set_buf(&s[idx - start_index], va + start, end - start);
2460 		}
2461 
2462 		sl[i] = s;
2463 	}
2464 
2465 	return sl;
2466 }
2467 
2468 static void free_alg(struct alg_spec *a)
2469 {
2470 	kzfree(a->alg_string);
2471 	kzfree(a->key);
2472 	memset(a, 0, sizeof *a);
2473 }
2474 
2475 static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval)
2476 {
2477 	char *k;
2478 
2479 	free_alg(a);
2480 
2481 	a->alg_string = kstrdup(strchr(arg, ':') + 1, GFP_KERNEL);
2482 	if (!a->alg_string)
2483 		goto nomem;
2484 
2485 	k = strchr(a->alg_string, ':');
2486 	if (k) {
2487 		*k = 0;
2488 		a->key_string = k + 1;
2489 		if (strlen(a->key_string) & 1)
2490 			goto inval;
2491 
2492 		a->key_size = strlen(a->key_string) / 2;
2493 		a->key = kmalloc(a->key_size, GFP_KERNEL);
2494 		if (!a->key)
2495 			goto nomem;
2496 		if (hex2bin(a->key, a->key_string, a->key_size))
2497 			goto inval;
2498 	}
2499 
2500 	return 0;
2501 inval:
2502 	*error = error_inval;
2503 	return -EINVAL;
2504 nomem:
2505 	*error = "Out of memory for an argument";
2506 	return -ENOMEM;
2507 }
2508 
2509 static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error,
2510 		   char *error_alg, char *error_key)
2511 {
2512 	int r;
2513 
2514 	if (a->alg_string) {
2515 		*hash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ASYNC);
2516 		if (IS_ERR(*hash)) {
2517 			*error = error_alg;
2518 			r = PTR_ERR(*hash);
2519 			*hash = NULL;
2520 			return r;
2521 		}
2522 
2523 		if (a->key) {
2524 			r = crypto_shash_setkey(*hash, a->key, a->key_size);
2525 			if (r) {
2526 				*error = error_key;
2527 				return r;
2528 			}
2529 		}
2530 	}
2531 
2532 	return 0;
2533 }
2534 
2535 static int create_journal(struct dm_integrity_c *ic, char **error)
2536 {
2537 	int r = 0;
2538 	unsigned i;
2539 	__u64 journal_pages, journal_desc_size, journal_tree_size;
2540 	unsigned char *crypt_data = NULL;
2541 
2542 	ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL);
2543 	ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL);
2544 	ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL);
2545 	ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL);
2546 
2547 	journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors,
2548 				PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT);
2549 	journal_desc_size = journal_pages * sizeof(struct page_list);
2550 	if (journal_pages >= totalram_pages - totalhigh_pages || journal_desc_size > ULONG_MAX) {
2551 		*error = "Journal doesn't fit into memory";
2552 		r = -ENOMEM;
2553 		goto bad;
2554 	}
2555 	ic->journal_pages = journal_pages;
2556 
2557 	ic->journal = dm_integrity_alloc_page_list(ic);
2558 	if (!ic->journal) {
2559 		*error = "Could not allocate memory for journal";
2560 		r = -ENOMEM;
2561 		goto bad;
2562 	}
2563 	if (ic->journal_crypt_alg.alg_string) {
2564 		unsigned ivsize, blocksize;
2565 		struct journal_completion comp;
2566 
2567 		comp.ic = ic;
2568 		ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, 0);
2569 		if (IS_ERR(ic->journal_crypt)) {
2570 			*error = "Invalid journal cipher";
2571 			r = PTR_ERR(ic->journal_crypt);
2572 			ic->journal_crypt = NULL;
2573 			goto bad;
2574 		}
2575 		ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
2576 		blocksize = crypto_skcipher_blocksize(ic->journal_crypt);
2577 
2578 		if (ic->journal_crypt_alg.key) {
2579 			r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key,
2580 						   ic->journal_crypt_alg.key_size);
2581 			if (r) {
2582 				*error = "Error setting encryption key";
2583 				goto bad;
2584 			}
2585 		}
2586 		DEBUG_print("cipher %s, block size %u iv size %u\n",
2587 			    ic->journal_crypt_alg.alg_string, blocksize, ivsize);
2588 
2589 		ic->journal_io = dm_integrity_alloc_page_list(ic);
2590 		if (!ic->journal_io) {
2591 			*error = "Could not allocate memory for journal io";
2592 			r = -ENOMEM;
2593 			goto bad;
2594 		}
2595 
2596 		if (blocksize == 1) {
2597 			struct scatterlist *sg;
2598 			SKCIPHER_REQUEST_ON_STACK(req, ic->journal_crypt);
2599 			unsigned char iv[ivsize];
2600 			skcipher_request_set_tfm(req, ic->journal_crypt);
2601 
2602 			ic->journal_xor = dm_integrity_alloc_page_list(ic);
2603 			if (!ic->journal_xor) {
2604 				*error = "Could not allocate memory for journal xor";
2605 				r = -ENOMEM;
2606 				goto bad;
2607 			}
2608 
2609 			sg = kvmalloc((ic->journal_pages + 1) * sizeof(struct scatterlist), GFP_KERNEL);
2610 			if (!sg) {
2611 				*error = "Unable to allocate sg list";
2612 				r = -ENOMEM;
2613 				goto bad;
2614 			}
2615 			sg_init_table(sg, ic->journal_pages + 1);
2616 			for (i = 0; i < ic->journal_pages; i++) {
2617 				char *va = lowmem_page_address(ic->journal_xor[i].page);
2618 				clear_page(va);
2619 				sg_set_buf(&sg[i], va, PAGE_SIZE);
2620 			}
2621 			sg_set_buf(&sg[i], &ic->commit_ids, sizeof ic->commit_ids);
2622 			memset(iv, 0x00, ivsize);
2623 
2624 			skcipher_request_set_crypt(req, sg, sg, PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, iv);
2625 			comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp);
2626 			comp.in_flight = (atomic_t)ATOMIC_INIT(1);
2627 			if (do_crypt(true, req, &comp))
2628 				wait_for_completion(&comp.comp);
2629 			kvfree(sg);
2630 			r = dm_integrity_failed(ic);
2631 			if (r) {
2632 				*error = "Unable to encrypt journal";
2633 				goto bad;
2634 			}
2635 			DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data");
2636 
2637 			crypto_free_skcipher(ic->journal_crypt);
2638 			ic->journal_crypt = NULL;
2639 		} else {
2640 			SKCIPHER_REQUEST_ON_STACK(req, ic->journal_crypt);
2641 			unsigned char iv[ivsize];
2642 			unsigned crypt_len = roundup(ivsize, blocksize);
2643 
2644 			crypt_data = kmalloc(crypt_len, GFP_KERNEL);
2645 			if (!crypt_data) {
2646 				*error = "Unable to allocate crypt data";
2647 				r = -ENOMEM;
2648 				goto bad;
2649 			}
2650 
2651 			skcipher_request_set_tfm(req, ic->journal_crypt);
2652 
2653 			ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal);
2654 			if (!ic->journal_scatterlist) {
2655 				*error = "Unable to allocate sg list";
2656 				r = -ENOMEM;
2657 				goto bad;
2658 			}
2659 			ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io);
2660 			if (!ic->journal_io_scatterlist) {
2661 				*error = "Unable to allocate sg list";
2662 				r = -ENOMEM;
2663 				goto bad;
2664 			}
2665 			ic->sk_requests = kvmalloc(ic->journal_sections * sizeof(struct skcipher_request *), GFP_KERNEL | __GFP_ZERO);
2666 			if (!ic->sk_requests) {
2667 				*error = "Unable to allocate sk requests";
2668 				r = -ENOMEM;
2669 				goto bad;
2670 			}
2671 			for (i = 0; i < ic->journal_sections; i++) {
2672 				struct scatterlist sg;
2673 				struct skcipher_request *section_req;
2674 				__u32 section_le = cpu_to_le32(i);
2675 
2676 				memset(iv, 0x00, ivsize);
2677 				memset(crypt_data, 0x00, crypt_len);
2678 				memcpy(crypt_data, &section_le, min((size_t)crypt_len, sizeof(section_le)));
2679 
2680 				sg_init_one(&sg, crypt_data, crypt_len);
2681 				skcipher_request_set_crypt(req, &sg, &sg, crypt_len, iv);
2682 				comp.comp = COMPLETION_INITIALIZER_ONSTACK(comp.comp);
2683 				comp.in_flight = (atomic_t)ATOMIC_INIT(1);
2684 				if (do_crypt(true, req, &comp))
2685 					wait_for_completion(&comp.comp);
2686 
2687 				r = dm_integrity_failed(ic);
2688 				if (r) {
2689 					*error = "Unable to generate iv";
2690 					goto bad;
2691 				}
2692 
2693 				section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
2694 				if (!section_req) {
2695 					*error = "Unable to allocate crypt request";
2696 					r = -ENOMEM;
2697 					goto bad;
2698 				}
2699 				section_req->iv = kmalloc(ivsize * 2, GFP_KERNEL);
2700 				if (!section_req->iv) {
2701 					skcipher_request_free(section_req);
2702 					*error = "Unable to allocate iv";
2703 					r = -ENOMEM;
2704 					goto bad;
2705 				}
2706 				memcpy(section_req->iv + ivsize, crypt_data, ivsize);
2707 				section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT;
2708 				ic->sk_requests[i] = section_req;
2709 				DEBUG_bytes(crypt_data, ivsize, "iv(%u)", i);
2710 			}
2711 		}
2712 	}
2713 
2714 	for (i = 0; i < N_COMMIT_IDS; i++) {
2715 		unsigned j;
2716 retest_commit_id:
2717 		for (j = 0; j < i; j++) {
2718 			if (ic->commit_ids[j] == ic->commit_ids[i]) {
2719 				ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1);
2720 				goto retest_commit_id;
2721 			}
2722 		}
2723 		DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]);
2724 	}
2725 
2726 	journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node);
2727 	if (journal_tree_size > ULONG_MAX) {
2728 		*error = "Journal doesn't fit into memory";
2729 		r = -ENOMEM;
2730 		goto bad;
2731 	}
2732 	ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL);
2733 	if (!ic->journal_tree) {
2734 		*error = "Could not allocate memory for journal tree";
2735 		r = -ENOMEM;
2736 	}
2737 bad:
2738 	kfree(crypt_data);
2739 	return r;
2740 }
2741 
2742 /*
2743  * Construct a integrity mapping
2744  *
2745  * Arguments:
2746  *	device
2747  *	offset from the start of the device
2748  *	tag size
2749  *	D - direct writes, J - journal writes, R - recovery mode
2750  *	number of optional arguments
2751  *	optional arguments:
2752  *		journal_sectors
2753  *		interleave_sectors
2754  *		buffer_sectors
2755  *		journal_watermark
2756  *		commit_time
2757  *		internal_hash
2758  *		journal_crypt
2759  *		journal_mac
2760  *		block_size
2761  */
2762 static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv)
2763 {
2764 	struct dm_integrity_c *ic;
2765 	char dummy;
2766 	int r;
2767 	unsigned extra_args;
2768 	struct dm_arg_set as;
2769 	static struct dm_arg _args[] = {
2770 		{0, 9, "Invalid number of feature args"},
2771 	};
2772 	unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
2773 	bool should_write_sb;
2774 	__u64 threshold;
2775 	unsigned long long start;
2776 
2777 #define DIRECT_ARGUMENTS	4
2778 
2779 	if (argc <= DIRECT_ARGUMENTS) {
2780 		ti->error = "Invalid argument count";
2781 		return -EINVAL;
2782 	}
2783 
2784 	ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL);
2785 	if (!ic) {
2786 		ti->error = "Cannot allocate integrity context";
2787 		return -ENOMEM;
2788 	}
2789 	ti->private = ic;
2790 	ti->per_io_data_size = sizeof(struct dm_integrity_io);
2791 
2792 	ic->in_progress = RB_ROOT;
2793 	init_waitqueue_head(&ic->endio_wait);
2794 	bio_list_init(&ic->flush_bio_list);
2795 	init_waitqueue_head(&ic->copy_to_journal_wait);
2796 	init_completion(&ic->crypto_backoff);
2797 
2798 	r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev);
2799 	if (r) {
2800 		ti->error = "Device lookup failed";
2801 		goto bad;
2802 	}
2803 
2804 	if (sscanf(argv[1], "%llu%c", &start, &dummy) != 1 || start != (sector_t)start) {
2805 		ti->error = "Invalid starting offset";
2806 		r = -EINVAL;
2807 		goto bad;
2808 	}
2809 	ic->start = start;
2810 
2811 	if (strcmp(argv[2], "-")) {
2812 		if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) {
2813 			ti->error = "Invalid tag size";
2814 			r = -EINVAL;
2815 			goto bad;
2816 		}
2817 	}
2818 
2819 	if (!strcmp(argv[3], "J") || !strcmp(argv[3], "D") || !strcmp(argv[3], "R"))
2820 		ic->mode = argv[3][0];
2821 	else {
2822 		ti->error = "Invalid mode (expecting J, D, R)";
2823 		r = -EINVAL;
2824 		goto bad;
2825 	}
2826 
2827 	ic->device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT;
2828 	journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS,
2829 			ic->device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR);
2830 	interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
2831 	buffer_sectors = DEFAULT_BUFFER_SECTORS;
2832 	journal_watermark = DEFAULT_JOURNAL_WATERMARK;
2833 	sync_msec = DEFAULT_SYNC_MSEC;
2834 	ic->sectors_per_block = 1;
2835 
2836 	as.argc = argc - DIRECT_ARGUMENTS;
2837 	as.argv = argv + DIRECT_ARGUMENTS;
2838 	r = dm_read_arg_group(_args, &as, &extra_args, &ti->error);
2839 	if (r)
2840 		goto bad;
2841 
2842 	while (extra_args--) {
2843 		const char *opt_string;
2844 		unsigned val;
2845 		opt_string = dm_shift_arg(&as);
2846 		if (!opt_string) {
2847 			r = -EINVAL;
2848 			ti->error = "Not enough feature arguments";
2849 			goto bad;
2850 		}
2851 		if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1)
2852 			journal_sectors = val;
2853 		else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1)
2854 			interleave_sectors = val;
2855 		else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1)
2856 			buffer_sectors = val;
2857 		else if (sscanf(opt_string, "journal_watermark:%u%c", &val, &dummy) == 1 && val <= 100)
2858 			journal_watermark = val;
2859 		else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1)
2860 			sync_msec = val;
2861 		else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) {
2862 			if (val < 1 << SECTOR_SHIFT ||
2863 			    val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT ||
2864 			    (val & (val -1))) {
2865 				r = -EINVAL;
2866 				ti->error = "Invalid block_size argument";
2867 				goto bad;
2868 			}
2869 			ic->sectors_per_block = val >> SECTOR_SHIFT;
2870 		} else if (!memcmp(opt_string, "internal_hash:", strlen("internal_hash:"))) {
2871 			r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error,
2872 					    "Invalid internal_hash argument");
2873 			if (r)
2874 				goto bad;
2875 		} else if (!memcmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) {
2876 			r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error,
2877 					    "Invalid journal_crypt argument");
2878 			if (r)
2879 				goto bad;
2880 		} else if (!memcmp(opt_string, "journal_mac:", strlen("journal_mac:"))) {
2881 			r = get_alg_and_key(opt_string, &ic->journal_mac_alg,  &ti->error,
2882 					    "Invalid journal_mac argument");
2883 			if (r)
2884 				goto bad;
2885 		} else {
2886 			r = -EINVAL;
2887 			ti->error = "Invalid argument";
2888 			goto bad;
2889 		}
2890 	}
2891 
2892 	r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error,
2893 		    "Invalid internal hash", "Error setting internal hash key");
2894 	if (r)
2895 		goto bad;
2896 
2897 	r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error,
2898 		    "Invalid journal mac", "Error setting journal mac key");
2899 	if (r)
2900 		goto bad;
2901 
2902 	if (!ic->tag_size) {
2903 		if (!ic->internal_hash) {
2904 			ti->error = "Unknown tag size";
2905 			r = -EINVAL;
2906 			goto bad;
2907 		}
2908 		ic->tag_size = crypto_shash_digestsize(ic->internal_hash);
2909 	}
2910 	if (ic->tag_size > MAX_TAG_SIZE) {
2911 		ti->error = "Too big tag size";
2912 		r = -EINVAL;
2913 		goto bad;
2914 	}
2915 	if (!(ic->tag_size & (ic->tag_size - 1)))
2916 		ic->log2_tag_size = __ffs(ic->tag_size);
2917 	else
2918 		ic->log2_tag_size = -1;
2919 
2920 	ic->autocommit_jiffies = msecs_to_jiffies(sync_msec);
2921 	ic->autocommit_msec = sync_msec;
2922 	setup_timer(&ic->autocommit_timer, autocommit_fn, (unsigned long)ic);
2923 
2924 	ic->io = dm_io_client_create();
2925 	if (IS_ERR(ic->io)) {
2926 		r = PTR_ERR(ic->io);
2927 		ic->io = NULL;
2928 		ti->error = "Cannot allocate dm io";
2929 		goto bad;
2930 	}
2931 
2932 	ic->journal_io_mempool = mempool_create_slab_pool(JOURNAL_IO_MEMPOOL, journal_io_cache);
2933 	if (!ic->journal_io_mempool) {
2934 		r = -ENOMEM;
2935 		ti->error = "Cannot allocate mempool";
2936 		goto bad;
2937 	}
2938 
2939 	ic->metadata_wq = alloc_workqueue("dm-integrity-metadata",
2940 					  WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE);
2941 	if (!ic->metadata_wq) {
2942 		ti->error = "Cannot allocate workqueue";
2943 		r = -ENOMEM;
2944 		goto bad;
2945 	}
2946 
2947 	/*
2948 	 * If this workqueue were percpu, it would cause bio reordering
2949 	 * and reduced performance.
2950 	 */
2951 	ic->wait_wq = alloc_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
2952 	if (!ic->wait_wq) {
2953 		ti->error = "Cannot allocate workqueue";
2954 		r = -ENOMEM;
2955 		goto bad;
2956 	}
2957 
2958 	ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1);
2959 	if (!ic->commit_wq) {
2960 		ti->error = "Cannot allocate workqueue";
2961 		r = -ENOMEM;
2962 		goto bad;
2963 	}
2964 	INIT_WORK(&ic->commit_work, integrity_commit);
2965 
2966 	if (ic->mode == 'J') {
2967 		ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1);
2968 		if (!ic->writer_wq) {
2969 			ti->error = "Cannot allocate workqueue";
2970 			r = -ENOMEM;
2971 			goto bad;
2972 		}
2973 		INIT_WORK(&ic->writer_work, integrity_writer);
2974 	}
2975 
2976 	ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL);
2977 	if (!ic->sb) {
2978 		r = -ENOMEM;
2979 		ti->error = "Cannot allocate superblock area";
2980 		goto bad;
2981 	}
2982 
2983 	r = sync_rw_sb(ic, REQ_OP_READ, 0);
2984 	if (r) {
2985 		ti->error = "Error reading superblock";
2986 		goto bad;
2987 	}
2988 	should_write_sb = false;
2989 	if (memcmp(ic->sb->magic, SB_MAGIC, 8)) {
2990 		if (ic->mode != 'R') {
2991 			if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) {
2992 				r = -EINVAL;
2993 				ti->error = "The device is not initialized";
2994 				goto bad;
2995 			}
2996 		}
2997 
2998 		r = initialize_superblock(ic, journal_sectors, interleave_sectors);
2999 		if (r) {
3000 			ti->error = "Could not initialize superblock";
3001 			goto bad;
3002 		}
3003 		if (ic->mode != 'R')
3004 			should_write_sb = true;
3005 	}
3006 
3007 	if (ic->sb->version != SB_VERSION) {
3008 		r = -EINVAL;
3009 		ti->error = "Unknown version";
3010 		goto bad;
3011 	}
3012 	if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) {
3013 		r = -EINVAL;
3014 		ti->error = "Tag size doesn't match the information in superblock";
3015 		goto bad;
3016 	}
3017 	if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) {
3018 		r = -EINVAL;
3019 		ti->error = "Block size doesn't match the information in superblock";
3020 		goto bad;
3021 	}
3022 	/* make sure that ti->max_io_len doesn't overflow */
3023 	if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS ||
3024 	    ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) {
3025 		r = -EINVAL;
3026 		ti->error = "Invalid interleave_sectors in the superblock";
3027 		goto bad;
3028 	}
3029 	ic->provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors);
3030 	if (ic->provided_data_sectors != le64_to_cpu(ic->sb->provided_data_sectors)) {
3031 		/* test for overflow */
3032 		r = -EINVAL;
3033 		ti->error = "The superblock has 64-bit device size, but the kernel was compiled with 32-bit sectors";
3034 		goto bad;
3035 	}
3036 	if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) {
3037 		r = -EINVAL;
3038 		ti->error = "Journal mac mismatch";
3039 		goto bad;
3040 	}
3041 	r = calculate_device_limits(ic);
3042 	if (r) {
3043 		ti->error = "The device is too small";
3044 		goto bad;
3045 	}
3046 	if (ti->len > ic->provided_data_sectors) {
3047 		r = -EINVAL;
3048 		ti->error = "Not enough provided sectors for requested mapping size";
3049 		goto bad;
3050 	}
3051 
3052 	if (!buffer_sectors)
3053 		buffer_sectors = 1;
3054 	ic->log2_buffer_sectors = min3((int)__fls(buffer_sectors), (int)__ffs(ic->metadata_run), 31 - SECTOR_SHIFT);
3055 
3056 	threshold = (__u64)ic->journal_entries * (100 - journal_watermark);
3057 	threshold += 50;
3058 	do_div(threshold, 100);
3059 	ic->free_sectors_threshold = threshold;
3060 
3061 	DEBUG_print("initialized:\n");
3062 	DEBUG_print("	integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size));
3063 	DEBUG_print("	journal_entry_size %u\n", ic->journal_entry_size);
3064 	DEBUG_print("	journal_entries_per_sector %u\n", ic->journal_entries_per_sector);
3065 	DEBUG_print("	journal_section_entries %u\n", ic->journal_section_entries);
3066 	DEBUG_print("	journal_section_sectors %u\n", ic->journal_section_sectors);
3067 	DEBUG_print("	journal_sections %u\n", (unsigned)le32_to_cpu(ic->sb->journal_sections));
3068 	DEBUG_print("	journal_entries %u\n", ic->journal_entries);
3069 	DEBUG_print("	log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors);
3070 	DEBUG_print("	device_sectors 0x%llx\n", (unsigned long long)ic->device_sectors);
3071 	DEBUG_print("	initial_sectors 0x%x\n", ic->initial_sectors);
3072 	DEBUG_print("	metadata_run 0x%x\n", ic->metadata_run);
3073 	DEBUG_print("	log2_metadata_run %d\n", ic->log2_metadata_run);
3074 	DEBUG_print("	provided_data_sectors 0x%llx (%llu)\n", (unsigned long long)ic->provided_data_sectors,
3075 		    (unsigned long long)ic->provided_data_sectors);
3076 	DEBUG_print("	log2_buffer_sectors %u\n", ic->log2_buffer_sectors);
3077 
3078 	ic->bufio = dm_bufio_client_create(ic->dev->bdev, 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors),
3079 					   1, 0, NULL, NULL);
3080 	if (IS_ERR(ic->bufio)) {
3081 		r = PTR_ERR(ic->bufio);
3082 		ti->error = "Cannot initialize dm-bufio";
3083 		ic->bufio = NULL;
3084 		goto bad;
3085 	}
3086 	dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors);
3087 
3088 	if (ic->mode != 'R') {
3089 		r = create_journal(ic, &ti->error);
3090 		if (r)
3091 			goto bad;
3092 	}
3093 
3094 	if (should_write_sb) {
3095 		int r;
3096 
3097 		init_journal(ic, 0, ic->journal_sections, 0);
3098 		r = dm_integrity_failed(ic);
3099 		if (unlikely(r)) {
3100 			ti->error = "Error initializing journal";
3101 			goto bad;
3102 		}
3103 		r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA);
3104 		if (r) {
3105 			ti->error = "Error initializing superblock";
3106 			goto bad;
3107 		}
3108 		ic->just_formatted = true;
3109 	}
3110 
3111 	r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors);
3112 	if (r)
3113 		goto bad;
3114 
3115 	if (!ic->internal_hash)
3116 		dm_integrity_set(ti, ic);
3117 
3118 	ti->num_flush_bios = 1;
3119 	ti->flush_supported = true;
3120 
3121 	return 0;
3122 bad:
3123 	dm_integrity_dtr(ti);
3124 	return r;
3125 }
3126 
3127 static void dm_integrity_dtr(struct dm_target *ti)
3128 {
3129 	struct dm_integrity_c *ic = ti->private;
3130 
3131 	BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
3132 
3133 	if (ic->metadata_wq)
3134 		destroy_workqueue(ic->metadata_wq);
3135 	if (ic->wait_wq)
3136 		destroy_workqueue(ic->wait_wq);
3137 	if (ic->commit_wq)
3138 		destroy_workqueue(ic->commit_wq);
3139 	if (ic->writer_wq)
3140 		destroy_workqueue(ic->writer_wq);
3141 	if (ic->bufio)
3142 		dm_bufio_client_destroy(ic->bufio);
3143 	mempool_destroy(ic->journal_io_mempool);
3144 	if (ic->io)
3145 		dm_io_client_destroy(ic->io);
3146 	if (ic->dev)
3147 		dm_put_device(ti, ic->dev);
3148 	dm_integrity_free_page_list(ic, ic->journal);
3149 	dm_integrity_free_page_list(ic, ic->journal_io);
3150 	dm_integrity_free_page_list(ic, ic->journal_xor);
3151 	if (ic->journal_scatterlist)
3152 		dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist);
3153 	if (ic->journal_io_scatterlist)
3154 		dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist);
3155 	if (ic->sk_requests) {
3156 		unsigned i;
3157 
3158 		for (i = 0; i < ic->journal_sections; i++) {
3159 			struct skcipher_request *req = ic->sk_requests[i];
3160 			if (req) {
3161 				kzfree(req->iv);
3162 				skcipher_request_free(req);
3163 			}
3164 		}
3165 		kvfree(ic->sk_requests);
3166 	}
3167 	kvfree(ic->journal_tree);
3168 	if (ic->sb)
3169 		free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT);
3170 
3171 	if (ic->internal_hash)
3172 		crypto_free_shash(ic->internal_hash);
3173 	free_alg(&ic->internal_hash_alg);
3174 
3175 	if (ic->journal_crypt)
3176 		crypto_free_skcipher(ic->journal_crypt);
3177 	free_alg(&ic->journal_crypt_alg);
3178 
3179 	if (ic->journal_mac)
3180 		crypto_free_shash(ic->journal_mac);
3181 	free_alg(&ic->journal_mac_alg);
3182 
3183 	kfree(ic);
3184 }
3185 
3186 static struct target_type integrity_target = {
3187 	.name			= "integrity",
3188 	.version		= {1, 0, 0},
3189 	.module			= THIS_MODULE,
3190 	.features		= DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
3191 	.ctr			= dm_integrity_ctr,
3192 	.dtr			= dm_integrity_dtr,
3193 	.map			= dm_integrity_map,
3194 	.postsuspend		= dm_integrity_postsuspend,
3195 	.resume			= dm_integrity_resume,
3196 	.status			= dm_integrity_status,
3197 	.iterate_devices	= dm_integrity_iterate_devices,
3198 	.io_hints		= dm_integrity_io_hints,
3199 };
3200 
3201 int __init dm_integrity_init(void)
3202 {
3203 	int r;
3204 
3205 	journal_io_cache = kmem_cache_create("integrity_journal_io",
3206 					     sizeof(struct journal_io), 0, 0, NULL);
3207 	if (!journal_io_cache) {
3208 		DMERR("can't allocate journal io cache");
3209 		return -ENOMEM;
3210 	}
3211 
3212 	r = dm_register_target(&integrity_target);
3213 
3214 	if (r < 0)
3215 		DMERR("register failed %d", r);
3216 
3217 	return r;
3218 }
3219 
3220 void dm_integrity_exit(void)
3221 {
3222 	dm_unregister_target(&integrity_target);
3223 	kmem_cache_destroy(journal_io_cache);
3224 }
3225 
3226 module_init(dm_integrity_init);
3227 module_exit(dm_integrity_exit);
3228 
3229 MODULE_AUTHOR("Milan Broz");
3230 MODULE_AUTHOR("Mikulas Patocka");
3231 MODULE_DESCRIPTION(DM_NAME " target for integrity tags extension");
3232 MODULE_LICENSE("GPL");
3233