1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2017 Western Digital Corporation or its affiliates.
4  *
5  * This file is released under the GPL.
6  */
7 
8 #include "dm-zoned.h"
9 
10 #include <linux/module.h>
11 #include <linux/crc32.h>
12 #include <linux/sched/mm.h>
13 
14 #define	DM_MSG_PREFIX		"zoned metadata"
15 
16 /*
17  * Metadata version.
18  */
19 #define DMZ_META_VER	1
20 
21 /*
22  * On-disk super block magic.
23  */
24 #define DMZ_MAGIC	((((unsigned int)('D')) << 24) | \
25 			 (((unsigned int)('Z')) << 16) | \
26 			 (((unsigned int)('B')) <<  8) | \
27 			 ((unsigned int)('D')))
28 
29 /*
30  * On disk super block.
31  * This uses only 512 B but uses on disk a full 4KB block. This block is
32  * followed on disk by the mapping table of chunks to zones and the bitmap
33  * blocks indicating zone block validity.
34  * The overall resulting metadata format is:
35  *    (1) Super block (1 block)
36  *    (2) Chunk mapping table (nr_map_blocks)
37  *    (3) Bitmap blocks (nr_bitmap_blocks)
38  * All metadata blocks are stored in conventional zones, starting from
39  * the first conventional zone found on disk.
40  */
41 struct dmz_super {
42 	/* Magic number */
43 	__le32		magic;			/*   4 */
44 
45 	/* Metadata version number */
46 	__le32		version;		/*   8 */
47 
48 	/* Generation number */
49 	__le64		gen;			/*  16 */
50 
51 	/* This block number */
52 	__le64		sb_block;		/*  24 */
53 
54 	/* The number of metadata blocks, including this super block */
55 	__le32		nr_meta_blocks;		/*  28 */
56 
57 	/* The number of sequential zones reserved for reclaim */
58 	__le32		nr_reserved_seq;	/*  32 */
59 
60 	/* The number of entries in the mapping table */
61 	__le32		nr_chunks;		/*  36 */
62 
63 	/* The number of blocks used for the chunk mapping table */
64 	__le32		nr_map_blocks;		/*  40 */
65 
66 	/* The number of blocks used for the block bitmaps */
67 	__le32		nr_bitmap_blocks;	/*  44 */
68 
69 	/* Checksum */
70 	__le32		crc;			/*  48 */
71 
72 	/* Padding to full 512B sector */
73 	u8		reserved[464];		/* 512 */
74 };
75 
76 /*
77  * Chunk mapping entry: entries are indexed by chunk number
78  * and give the zone ID (dzone_id) mapping the chunk on disk.
79  * This zone may be sequential or random. If it is a sequential
80  * zone, a second zone (bzone_id) used as a write buffer may
81  * also be specified. This second zone will always be a randomly
82  * writeable zone.
83  */
84 struct dmz_map {
85 	__le32			dzone_id;
86 	__le32			bzone_id;
87 };
88 
89 /*
90  * Chunk mapping table metadata: 512 8-bytes entries per 4KB block.
91  */
92 #define DMZ_MAP_ENTRIES		(DMZ_BLOCK_SIZE / sizeof(struct dmz_map))
93 #define DMZ_MAP_ENTRIES_SHIFT	(ilog2(DMZ_MAP_ENTRIES))
94 #define DMZ_MAP_ENTRIES_MASK	(DMZ_MAP_ENTRIES - 1)
95 #define DMZ_MAP_UNMAPPED	UINT_MAX
96 
97 /*
98  * Meta data block descriptor (for cached metadata blocks).
99  */
100 struct dmz_mblock {
101 	struct rb_node		node;
102 	struct list_head	link;
103 	sector_t		no;
104 	unsigned int		ref;
105 	unsigned long		state;
106 	struct page		*page;
107 	void			*data;
108 };
109 
110 /*
111  * Metadata block state flags.
112  */
113 enum {
114 	DMZ_META_DIRTY,
115 	DMZ_META_READING,
116 	DMZ_META_WRITING,
117 	DMZ_META_ERROR,
118 };
119 
120 /*
121  * Super block information (one per metadata set).
122  */
123 struct dmz_sb {
124 	sector_t		block;
125 	struct dmz_mblock	*mblk;
126 	struct dmz_super	*sb;
127 };
128 
129 /*
130  * In-memory metadata.
131  */
132 struct dmz_metadata {
133 	struct dmz_dev		*dev;
134 
135 	sector_t		zone_bitmap_size;
136 	unsigned int		zone_nr_bitmap_blocks;
137 	unsigned int		zone_bits_per_mblk;
138 
139 	unsigned int		nr_bitmap_blocks;
140 	unsigned int		nr_map_blocks;
141 
142 	unsigned int		nr_useable_zones;
143 	unsigned int		nr_meta_blocks;
144 	unsigned int		nr_meta_zones;
145 	unsigned int		nr_data_zones;
146 	unsigned int		nr_rnd_zones;
147 	unsigned int		nr_reserved_seq;
148 	unsigned int		nr_chunks;
149 
150 	/* Zone information array */
151 	struct dm_zone		*zones;
152 
153 	struct dm_zone		*sb_zone;
154 	struct dmz_sb		sb[2];
155 	unsigned int		mblk_primary;
156 	u64			sb_gen;
157 	unsigned int		min_nr_mblks;
158 	unsigned int		max_nr_mblks;
159 	atomic_t		nr_mblks;
160 	struct rw_semaphore	mblk_sem;
161 	struct mutex		mblk_flush_lock;
162 	spinlock_t		mblk_lock;
163 	struct rb_root		mblk_rbtree;
164 	struct list_head	mblk_lru_list;
165 	struct list_head	mblk_dirty_list;
166 	struct shrinker		mblk_shrinker;
167 
168 	/* Zone allocation management */
169 	struct mutex		map_lock;
170 	struct dmz_mblock	**map_mblk;
171 	unsigned int		nr_rnd;
172 	atomic_t		unmap_nr_rnd;
173 	struct list_head	unmap_rnd_list;
174 	struct list_head	map_rnd_list;
175 
176 	unsigned int		nr_seq;
177 	atomic_t		unmap_nr_seq;
178 	struct list_head	unmap_seq_list;
179 	struct list_head	map_seq_list;
180 
181 	atomic_t		nr_reserved_seq_zones;
182 	struct list_head	reserved_seq_zones_list;
183 
184 	wait_queue_head_t	free_wq;
185 };
186 
187 /*
188  * Various accessors
189  */
190 unsigned int dmz_id(struct dmz_metadata *zmd, struct dm_zone *zone)
191 {
192 	return ((unsigned int)(zone - zmd->zones));
193 }
194 
195 sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)
196 {
197 	return (sector_t)dmz_id(zmd, zone) << zmd->dev->zone_nr_sectors_shift;
198 }
199 
200 sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)
201 {
202 	return (sector_t)dmz_id(zmd, zone) << zmd->dev->zone_nr_blocks_shift;
203 }
204 
205 unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)
206 {
207 	return zmd->nr_chunks;
208 }
209 
210 unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd)
211 {
212 	return zmd->nr_rnd;
213 }
214 
215 unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd)
216 {
217 	return atomic_read(&zmd->unmap_nr_rnd);
218 }
219 
220 /*
221  * Lock/unlock mapping table.
222  * The map lock also protects all the zone lists.
223  */
224 void dmz_lock_map(struct dmz_metadata *zmd)
225 {
226 	mutex_lock(&zmd->map_lock);
227 }
228 
229 void dmz_unlock_map(struct dmz_metadata *zmd)
230 {
231 	mutex_unlock(&zmd->map_lock);
232 }
233 
234 /*
235  * Lock/unlock metadata access. This is a "read" lock on a semaphore
236  * that prevents metadata flush from running while metadata are being
237  * modified. The actual metadata write mutual exclusion is achieved with
238  * the map lock and zone state management (active and reclaim state are
239  * mutually exclusive).
240  */
241 void dmz_lock_metadata(struct dmz_metadata *zmd)
242 {
243 	down_read(&zmd->mblk_sem);
244 }
245 
246 void dmz_unlock_metadata(struct dmz_metadata *zmd)
247 {
248 	up_read(&zmd->mblk_sem);
249 }
250 
251 /*
252  * Lock/unlock flush: prevent concurrent executions
253  * of dmz_flush_metadata as well as metadata modification in reclaim
254  * while flush is being executed.
255  */
256 void dmz_lock_flush(struct dmz_metadata *zmd)
257 {
258 	mutex_lock(&zmd->mblk_flush_lock);
259 }
260 
261 void dmz_unlock_flush(struct dmz_metadata *zmd)
262 {
263 	mutex_unlock(&zmd->mblk_flush_lock);
264 }
265 
266 /*
267  * Allocate a metadata block.
268  */
269 static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,
270 					   sector_t mblk_no)
271 {
272 	struct dmz_mblock *mblk = NULL;
273 
274 	/* See if we can reuse cached blocks */
275 	if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) {
276 		spin_lock(&zmd->mblk_lock);
277 		mblk = list_first_entry_or_null(&zmd->mblk_lru_list,
278 						struct dmz_mblock, link);
279 		if (mblk) {
280 			list_del_init(&mblk->link);
281 			rb_erase(&mblk->node, &zmd->mblk_rbtree);
282 			mblk->no = mblk_no;
283 		}
284 		spin_unlock(&zmd->mblk_lock);
285 		if (mblk)
286 			return mblk;
287 	}
288 
289 	/* Allocate a new block */
290 	mblk = kmalloc(sizeof(struct dmz_mblock), GFP_NOIO);
291 	if (!mblk)
292 		return NULL;
293 
294 	mblk->page = alloc_page(GFP_NOIO);
295 	if (!mblk->page) {
296 		kfree(mblk);
297 		return NULL;
298 	}
299 
300 	RB_CLEAR_NODE(&mblk->node);
301 	INIT_LIST_HEAD(&mblk->link);
302 	mblk->ref = 0;
303 	mblk->state = 0;
304 	mblk->no = mblk_no;
305 	mblk->data = page_address(mblk->page);
306 
307 	atomic_inc(&zmd->nr_mblks);
308 
309 	return mblk;
310 }
311 
312 /*
313  * Free a metadata block.
314  */
315 static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
316 {
317 	__free_pages(mblk->page, 0);
318 	kfree(mblk);
319 
320 	atomic_dec(&zmd->nr_mblks);
321 }
322 
323 /*
324  * Insert a metadata block in the rbtree.
325  */
326 static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
327 {
328 	struct rb_root *root = &zmd->mblk_rbtree;
329 	struct rb_node **new = &(root->rb_node), *parent = NULL;
330 	struct dmz_mblock *b;
331 
332 	/* Figure out where to put the new node */
333 	while (*new) {
334 		b = container_of(*new, struct dmz_mblock, node);
335 		parent = *new;
336 		new = (b->no < mblk->no) ? &((*new)->rb_left) : &((*new)->rb_right);
337 	}
338 
339 	/* Add new node and rebalance tree */
340 	rb_link_node(&mblk->node, parent, new);
341 	rb_insert_color(&mblk->node, root);
342 }
343 
344 /*
345  * Lookup a metadata block in the rbtree. If the block is found, increment
346  * its reference count.
347  */
348 static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,
349 					      sector_t mblk_no)
350 {
351 	struct rb_root *root = &zmd->mblk_rbtree;
352 	struct rb_node *node = root->rb_node;
353 	struct dmz_mblock *mblk;
354 
355 	while (node) {
356 		mblk = container_of(node, struct dmz_mblock, node);
357 		if (mblk->no == mblk_no) {
358 			/*
359 			 * If this is the first reference to the block,
360 			 * remove it from the LRU list.
361 			 */
362 			mblk->ref++;
363 			if (mblk->ref == 1 &&
364 			    !test_bit(DMZ_META_DIRTY, &mblk->state))
365 				list_del_init(&mblk->link);
366 			return mblk;
367 		}
368 		node = (mblk->no < mblk_no) ? node->rb_left : node->rb_right;
369 	}
370 
371 	return NULL;
372 }
373 
374 /*
375  * Metadata block BIO end callback.
376  */
377 static void dmz_mblock_bio_end_io(struct bio *bio)
378 {
379 	struct dmz_mblock *mblk = bio->bi_private;
380 	int flag;
381 
382 	if (bio->bi_status)
383 		set_bit(DMZ_META_ERROR, &mblk->state);
384 
385 	if (bio_op(bio) == REQ_OP_WRITE)
386 		flag = DMZ_META_WRITING;
387 	else
388 		flag = DMZ_META_READING;
389 
390 	clear_bit_unlock(flag, &mblk->state);
391 	smp_mb__after_atomic();
392 	wake_up_bit(&mblk->state, flag);
393 
394 	bio_put(bio);
395 }
396 
397 /*
398  * Read an uncached metadata block from disk and add it to the cache.
399  */
400 static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,
401 					      sector_t mblk_no)
402 {
403 	struct dmz_mblock *mblk, *m;
404 	sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;
405 	struct bio *bio;
406 
407 	if (dmz_bdev_is_dying(zmd->dev))
408 		return ERR_PTR(-EIO);
409 
410 	/* Get a new block and a BIO to read it */
411 	mblk = dmz_alloc_mblock(zmd, mblk_no);
412 	if (!mblk)
413 		return ERR_PTR(-ENOMEM);
414 
415 	bio = bio_alloc(GFP_NOIO, 1);
416 	if (!bio) {
417 		dmz_free_mblock(zmd, mblk);
418 		return ERR_PTR(-ENOMEM);
419 	}
420 
421 	spin_lock(&zmd->mblk_lock);
422 
423 	/*
424 	 * Make sure that another context did not start reading
425 	 * the block already.
426 	 */
427 	m = dmz_get_mblock_fast(zmd, mblk_no);
428 	if (m) {
429 		spin_unlock(&zmd->mblk_lock);
430 		dmz_free_mblock(zmd, mblk);
431 		bio_put(bio);
432 		return m;
433 	}
434 
435 	mblk->ref++;
436 	set_bit(DMZ_META_READING, &mblk->state);
437 	dmz_insert_mblock(zmd, mblk);
438 
439 	spin_unlock(&zmd->mblk_lock);
440 
441 	/* Submit read BIO */
442 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
443 	bio_set_dev(bio, zmd->dev->bdev);
444 	bio->bi_private = mblk;
445 	bio->bi_end_io = dmz_mblock_bio_end_io;
446 	bio_set_op_attrs(bio, REQ_OP_READ, REQ_META | REQ_PRIO);
447 	bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0);
448 	submit_bio(bio);
449 
450 	return mblk;
451 }
452 
453 /*
454  * Free metadata blocks.
455  */
456 static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,
457 					     unsigned long limit)
458 {
459 	struct dmz_mblock *mblk;
460 	unsigned long count = 0;
461 
462 	if (!zmd->max_nr_mblks)
463 		return 0;
464 
465 	while (!list_empty(&zmd->mblk_lru_list) &&
466 	       atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks &&
467 	       count < limit) {
468 		mblk = list_first_entry(&zmd->mblk_lru_list,
469 					struct dmz_mblock, link);
470 		list_del_init(&mblk->link);
471 		rb_erase(&mblk->node, &zmd->mblk_rbtree);
472 		dmz_free_mblock(zmd, mblk);
473 		count++;
474 	}
475 
476 	return count;
477 }
478 
479 /*
480  * For mblock shrinker: get the number of unused metadata blocks in the cache.
481  */
482 static unsigned long dmz_mblock_shrinker_count(struct shrinker *shrink,
483 					       struct shrink_control *sc)
484 {
485 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);
486 
487 	return atomic_read(&zmd->nr_mblks);
488 }
489 
490 /*
491  * For mblock shrinker: scan unused metadata blocks and shrink the cache.
492  */
493 static unsigned long dmz_mblock_shrinker_scan(struct shrinker *shrink,
494 					      struct shrink_control *sc)
495 {
496 	struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);
497 	unsigned long count;
498 
499 	spin_lock(&zmd->mblk_lock);
500 	count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan);
501 	spin_unlock(&zmd->mblk_lock);
502 
503 	return count ? count : SHRINK_STOP;
504 }
505 
506 /*
507  * Release a metadata block.
508  */
509 static void dmz_release_mblock(struct dmz_metadata *zmd,
510 			       struct dmz_mblock *mblk)
511 {
512 
513 	if (!mblk)
514 		return;
515 
516 	spin_lock(&zmd->mblk_lock);
517 
518 	mblk->ref--;
519 	if (mblk->ref == 0) {
520 		if (test_bit(DMZ_META_ERROR, &mblk->state)) {
521 			rb_erase(&mblk->node, &zmd->mblk_rbtree);
522 			dmz_free_mblock(zmd, mblk);
523 		} else if (!test_bit(DMZ_META_DIRTY, &mblk->state)) {
524 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);
525 			dmz_shrink_mblock_cache(zmd, 1);
526 		}
527 	}
528 
529 	spin_unlock(&zmd->mblk_lock);
530 }
531 
532 /*
533  * Get a metadata block from the rbtree. If the block
534  * is not present, read it from disk.
535  */
536 static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,
537 					 sector_t mblk_no)
538 {
539 	struct dmz_mblock *mblk;
540 
541 	/* Check rbtree */
542 	spin_lock(&zmd->mblk_lock);
543 	mblk = dmz_get_mblock_fast(zmd, mblk_no);
544 	spin_unlock(&zmd->mblk_lock);
545 
546 	if (!mblk) {
547 		/* Cache miss: read the block from disk */
548 		mblk = dmz_get_mblock_slow(zmd, mblk_no);
549 		if (IS_ERR(mblk))
550 			return mblk;
551 	}
552 
553 	/* Wait for on-going read I/O and check for error */
554 	wait_on_bit_io(&mblk->state, DMZ_META_READING,
555 		       TASK_UNINTERRUPTIBLE);
556 	if (test_bit(DMZ_META_ERROR, &mblk->state)) {
557 		dmz_release_mblock(zmd, mblk);
558 		dmz_check_bdev(zmd->dev);
559 		return ERR_PTR(-EIO);
560 	}
561 
562 	return mblk;
563 }
564 
565 /*
566  * Mark a metadata block dirty.
567  */
568 static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
569 {
570 	spin_lock(&zmd->mblk_lock);
571 	if (!test_and_set_bit(DMZ_META_DIRTY, &mblk->state))
572 		list_add_tail(&mblk->link, &zmd->mblk_dirty_list);
573 	spin_unlock(&zmd->mblk_lock);
574 }
575 
576 /*
577  * Issue a metadata block write BIO.
578  */
579 static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
580 			    unsigned int set)
581 {
582 	sector_t block = zmd->sb[set].block + mblk->no;
583 	struct bio *bio;
584 
585 	if (dmz_bdev_is_dying(zmd->dev))
586 		return -EIO;
587 
588 	bio = bio_alloc(GFP_NOIO, 1);
589 	if (!bio) {
590 		set_bit(DMZ_META_ERROR, &mblk->state);
591 		return -ENOMEM;
592 	}
593 
594 	set_bit(DMZ_META_WRITING, &mblk->state);
595 
596 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
597 	bio_set_dev(bio, zmd->dev->bdev);
598 	bio->bi_private = mblk;
599 	bio->bi_end_io = dmz_mblock_bio_end_io;
600 	bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_META | REQ_PRIO);
601 	bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0);
602 	submit_bio(bio);
603 
604 	return 0;
605 }
606 
607 /*
608  * Read/write a metadata block.
609  */
610 static int dmz_rdwr_block(struct dmz_metadata *zmd, int op, sector_t block,
611 			  struct page *page)
612 {
613 	struct bio *bio;
614 	int ret;
615 
616 	if (dmz_bdev_is_dying(zmd->dev))
617 		return -EIO;
618 
619 	bio = bio_alloc(GFP_NOIO, 1);
620 	if (!bio)
621 		return -ENOMEM;
622 
623 	bio->bi_iter.bi_sector = dmz_blk2sect(block);
624 	bio_set_dev(bio, zmd->dev->bdev);
625 	bio_set_op_attrs(bio, op, REQ_SYNC | REQ_META | REQ_PRIO);
626 	bio_add_page(bio, page, DMZ_BLOCK_SIZE, 0);
627 	ret = submit_bio_wait(bio);
628 	bio_put(bio);
629 
630 	if (ret)
631 		dmz_check_bdev(zmd->dev);
632 	return ret;
633 }
634 
635 /*
636  * Write super block of the specified metadata set.
637  */
638 static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)
639 {
640 	sector_t block = zmd->sb[set].block;
641 	struct dmz_mblock *mblk = zmd->sb[set].mblk;
642 	struct dmz_super *sb = zmd->sb[set].sb;
643 	u64 sb_gen = zmd->sb_gen + 1;
644 	int ret;
645 
646 	sb->magic = cpu_to_le32(DMZ_MAGIC);
647 	sb->version = cpu_to_le32(DMZ_META_VER);
648 
649 	sb->gen = cpu_to_le64(sb_gen);
650 
651 	sb->sb_block = cpu_to_le64(block);
652 	sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);
653 	sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);
654 	sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);
655 
656 	sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);
657 	sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);
658 
659 	sb->crc = 0;
660 	sb->crc = cpu_to_le32(crc32_le(sb_gen, (unsigned char *)sb, DMZ_BLOCK_SIZE));
661 
662 	ret = dmz_rdwr_block(zmd, REQ_OP_WRITE, block, mblk->page);
663 	if (ret == 0)
664 		ret = blkdev_issue_flush(zmd->dev->bdev, GFP_NOIO, NULL);
665 
666 	return ret;
667 }
668 
669 /*
670  * Write dirty metadata blocks to the specified set.
671  */
672 static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,
673 				   struct list_head *write_list,
674 				   unsigned int set)
675 {
676 	struct dmz_mblock *mblk;
677 	struct blk_plug plug;
678 	int ret = 0, nr_mblks_submitted = 0;
679 
680 	/* Issue writes */
681 	blk_start_plug(&plug);
682 	list_for_each_entry(mblk, write_list, link) {
683 		ret = dmz_write_mblock(zmd, mblk, set);
684 		if (ret)
685 			break;
686 		nr_mblks_submitted++;
687 	}
688 	blk_finish_plug(&plug);
689 
690 	/* Wait for completion */
691 	list_for_each_entry(mblk, write_list, link) {
692 		if (!nr_mblks_submitted)
693 			break;
694 		wait_on_bit_io(&mblk->state, DMZ_META_WRITING,
695 			       TASK_UNINTERRUPTIBLE);
696 		if (test_bit(DMZ_META_ERROR, &mblk->state)) {
697 			clear_bit(DMZ_META_ERROR, &mblk->state);
698 			dmz_check_bdev(zmd->dev);
699 			ret = -EIO;
700 		}
701 		nr_mblks_submitted--;
702 	}
703 
704 	/* Flush drive cache (this will also sync data) */
705 	if (ret == 0)
706 		ret = blkdev_issue_flush(zmd->dev->bdev, GFP_NOIO, NULL);
707 
708 	return ret;
709 }
710 
711 /*
712  * Log dirty metadata blocks.
713  */
714 static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,
715 				 struct list_head *write_list)
716 {
717 	unsigned int log_set = zmd->mblk_primary ^ 0x1;
718 	int ret;
719 
720 	/* Write dirty blocks to the log */
721 	ret = dmz_write_dirty_mblocks(zmd, write_list, log_set);
722 	if (ret)
723 		return ret;
724 
725 	/*
726 	 * No error so far: now validate the log by updating the
727 	 * log index super block generation.
728 	 */
729 	ret = dmz_write_sb(zmd, log_set);
730 	if (ret)
731 		return ret;
732 
733 	return 0;
734 }
735 
736 /*
737  * Flush dirty metadata blocks.
738  */
739 int dmz_flush_metadata(struct dmz_metadata *zmd)
740 {
741 	struct dmz_mblock *mblk;
742 	struct list_head write_list;
743 	int ret;
744 
745 	if (WARN_ON(!zmd))
746 		return 0;
747 
748 	INIT_LIST_HEAD(&write_list);
749 
750 	/*
751 	 * Make sure that metadata blocks are stable before logging: take
752 	 * the write lock on the metadata semaphore to prevent target BIOs
753 	 * from modifying metadata.
754 	 */
755 	down_write(&zmd->mblk_sem);
756 
757 	/*
758 	 * This is called from the target flush work and reclaim work.
759 	 * Concurrent execution is not allowed.
760 	 */
761 	dmz_lock_flush(zmd);
762 
763 	if (dmz_bdev_is_dying(zmd->dev)) {
764 		ret = -EIO;
765 		goto out;
766 	}
767 
768 	/* Get dirty blocks */
769 	spin_lock(&zmd->mblk_lock);
770 	list_splice_init(&zmd->mblk_dirty_list, &write_list);
771 	spin_unlock(&zmd->mblk_lock);
772 
773 	/* If there are no dirty metadata blocks, just flush the device cache */
774 	if (list_empty(&write_list)) {
775 		ret = blkdev_issue_flush(zmd->dev->bdev, GFP_NOIO, NULL);
776 		goto err;
777 	}
778 
779 	/*
780 	 * The primary metadata set is still clean. Keep it this way until
781 	 * all updates are successful in the secondary set. That is, use
782 	 * the secondary set as a log.
783 	 */
784 	ret = dmz_log_dirty_mblocks(zmd, &write_list);
785 	if (ret)
786 		goto err;
787 
788 	/*
789 	 * The log is on disk. It is now safe to update in place
790 	 * in the primary metadata set.
791 	 */
792 	ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary);
793 	if (ret)
794 		goto err;
795 
796 	ret = dmz_write_sb(zmd, zmd->mblk_primary);
797 	if (ret)
798 		goto err;
799 
800 	while (!list_empty(&write_list)) {
801 		mblk = list_first_entry(&write_list, struct dmz_mblock, link);
802 		list_del_init(&mblk->link);
803 
804 		spin_lock(&zmd->mblk_lock);
805 		clear_bit(DMZ_META_DIRTY, &mblk->state);
806 		if (mblk->ref == 0)
807 			list_add_tail(&mblk->link, &zmd->mblk_lru_list);
808 		spin_unlock(&zmd->mblk_lock);
809 	}
810 
811 	zmd->sb_gen++;
812 out:
813 	dmz_unlock_flush(zmd);
814 	up_write(&zmd->mblk_sem);
815 
816 	return ret;
817 
818 err:
819 	if (!list_empty(&write_list)) {
820 		spin_lock(&zmd->mblk_lock);
821 		list_splice(&write_list, &zmd->mblk_dirty_list);
822 		spin_unlock(&zmd->mblk_lock);
823 	}
824 	if (!dmz_check_bdev(zmd->dev))
825 		ret = -EIO;
826 	goto out;
827 }
828 
829 /*
830  * Check super block.
831  */
832 static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_super *sb)
833 {
834 	unsigned int nr_meta_zones, nr_data_zones;
835 	struct dmz_dev *dev = zmd->dev;
836 	u32 crc, stored_crc;
837 	u64 gen;
838 
839 	gen = le64_to_cpu(sb->gen);
840 	stored_crc = le32_to_cpu(sb->crc);
841 	sb->crc = 0;
842 	crc = crc32_le(gen, (unsigned char *)sb, DMZ_BLOCK_SIZE);
843 	if (crc != stored_crc) {
844 		dmz_dev_err(dev, "Invalid checksum (needed 0x%08x, got 0x%08x)",
845 			    crc, stored_crc);
846 		return -ENXIO;
847 	}
848 
849 	if (le32_to_cpu(sb->magic) != DMZ_MAGIC) {
850 		dmz_dev_err(dev, "Invalid meta magic (needed 0x%08x, got 0x%08x)",
851 			    DMZ_MAGIC, le32_to_cpu(sb->magic));
852 		return -ENXIO;
853 	}
854 
855 	if (le32_to_cpu(sb->version) != DMZ_META_VER) {
856 		dmz_dev_err(dev, "Invalid meta version (needed %d, got %d)",
857 			    DMZ_META_VER, le32_to_cpu(sb->version));
858 		return -ENXIO;
859 	}
860 
861 	nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + dev->zone_nr_blocks - 1)
862 		>> dev->zone_nr_blocks_shift;
863 	if (!nr_meta_zones ||
864 	    nr_meta_zones >= zmd->nr_rnd_zones) {
865 		dmz_dev_err(dev, "Invalid number of metadata blocks");
866 		return -ENXIO;
867 	}
868 
869 	if (!le32_to_cpu(sb->nr_reserved_seq) ||
870 	    le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {
871 		dmz_dev_err(dev, "Invalid number of reserved sequential zones");
872 		return -ENXIO;
873 	}
874 
875 	nr_data_zones = zmd->nr_useable_zones -
876 		(nr_meta_zones * 2 + le32_to_cpu(sb->nr_reserved_seq));
877 	if (le32_to_cpu(sb->nr_chunks) > nr_data_zones) {
878 		dmz_dev_err(dev, "Invalid number of chunks %u / %u",
879 			    le32_to_cpu(sb->nr_chunks), nr_data_zones);
880 		return -ENXIO;
881 	}
882 
883 	/* OK */
884 	zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);
885 	zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);
886 	zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);
887 	zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);
888 	zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);
889 	zmd->nr_meta_zones = nr_meta_zones;
890 	zmd->nr_data_zones = nr_data_zones;
891 
892 	return 0;
893 }
894 
895 /*
896  * Read the first or second super block from disk.
897  */
898 static int dmz_read_sb(struct dmz_metadata *zmd, unsigned int set)
899 {
900 	return dmz_rdwr_block(zmd, REQ_OP_READ, zmd->sb[set].block,
901 			      zmd->sb[set].mblk->page);
902 }
903 
904 /*
905  * Determine the position of the secondary super blocks on disk.
906  * This is used only if a corruption of the primary super block
907  * is detected.
908  */
909 static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)
910 {
911 	unsigned int zone_nr_blocks = zmd->dev->zone_nr_blocks;
912 	struct dmz_mblock *mblk;
913 	int i;
914 
915 	/* Allocate a block */
916 	mblk = dmz_alloc_mblock(zmd, 0);
917 	if (!mblk)
918 		return -ENOMEM;
919 
920 	zmd->sb[1].mblk = mblk;
921 	zmd->sb[1].sb = mblk->data;
922 
923 	/* Bad first super block: search for the second one */
924 	zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;
925 	for (i = 0; i < zmd->nr_rnd_zones - 1; i++) {
926 		if (dmz_read_sb(zmd, 1) != 0)
927 			break;
928 		if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)
929 			return 0;
930 		zmd->sb[1].block += zone_nr_blocks;
931 	}
932 
933 	dmz_free_mblock(zmd, mblk);
934 	zmd->sb[1].mblk = NULL;
935 
936 	return -EIO;
937 }
938 
939 /*
940  * Read the first or second super block from disk.
941  */
942 static int dmz_get_sb(struct dmz_metadata *zmd, unsigned int set)
943 {
944 	struct dmz_mblock *mblk;
945 	int ret;
946 
947 	/* Allocate a block */
948 	mblk = dmz_alloc_mblock(zmd, 0);
949 	if (!mblk)
950 		return -ENOMEM;
951 
952 	zmd->sb[set].mblk = mblk;
953 	zmd->sb[set].sb = mblk->data;
954 
955 	/* Read super block */
956 	ret = dmz_read_sb(zmd, set);
957 	if (ret) {
958 		dmz_free_mblock(zmd, mblk);
959 		zmd->sb[set].mblk = NULL;
960 		return ret;
961 	}
962 
963 	return 0;
964 }
965 
966 /*
967  * Recover a metadata set.
968  */
969 static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)
970 {
971 	unsigned int src_set = dst_set ^ 0x1;
972 	struct page *page;
973 	int i, ret;
974 
975 	dmz_dev_warn(zmd->dev, "Metadata set %u invalid: recovering", dst_set);
976 
977 	if (dst_set == 0)
978 		zmd->sb[0].block = dmz_start_block(zmd, zmd->sb_zone);
979 	else {
980 		zmd->sb[1].block = zmd->sb[0].block +
981 			(zmd->nr_meta_zones << zmd->dev->zone_nr_blocks_shift);
982 	}
983 
984 	page = alloc_page(GFP_NOIO);
985 	if (!page)
986 		return -ENOMEM;
987 
988 	/* Copy metadata blocks */
989 	for (i = 1; i < zmd->nr_meta_blocks; i++) {
990 		ret = dmz_rdwr_block(zmd, REQ_OP_READ,
991 				     zmd->sb[src_set].block + i, page);
992 		if (ret)
993 			goto out;
994 		ret = dmz_rdwr_block(zmd, REQ_OP_WRITE,
995 				     zmd->sb[dst_set].block + i, page);
996 		if (ret)
997 			goto out;
998 	}
999 
1000 	/* Finalize with the super block */
1001 	if (!zmd->sb[dst_set].mblk) {
1002 		zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0);
1003 		if (!zmd->sb[dst_set].mblk) {
1004 			ret = -ENOMEM;
1005 			goto out;
1006 		}
1007 		zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;
1008 	}
1009 
1010 	ret = dmz_write_sb(zmd, dst_set);
1011 out:
1012 	__free_pages(page, 0);
1013 
1014 	return ret;
1015 }
1016 
1017 /*
1018  * Get super block from disk.
1019  */
1020 static int dmz_load_sb(struct dmz_metadata *zmd)
1021 {
1022 	bool sb_good[2] = {false, false};
1023 	u64 sb_gen[2] = {0, 0};
1024 	int ret;
1025 
1026 	/* Read and check the primary super block */
1027 	zmd->sb[0].block = dmz_start_block(zmd, zmd->sb_zone);
1028 	ret = dmz_get_sb(zmd, 0);
1029 	if (ret) {
1030 		dmz_dev_err(zmd->dev, "Read primary super block failed");
1031 		return ret;
1032 	}
1033 
1034 	ret = dmz_check_sb(zmd, zmd->sb[0].sb);
1035 
1036 	/* Read and check secondary super block */
1037 	if (ret == 0) {
1038 		sb_good[0] = true;
1039 		zmd->sb[1].block = zmd->sb[0].block +
1040 			(zmd->nr_meta_zones << zmd->dev->zone_nr_blocks_shift);
1041 		ret = dmz_get_sb(zmd, 1);
1042 	} else
1043 		ret = dmz_lookup_secondary_sb(zmd);
1044 
1045 	if (ret) {
1046 		dmz_dev_err(zmd->dev, "Read secondary super block failed");
1047 		return ret;
1048 	}
1049 
1050 	ret = dmz_check_sb(zmd, zmd->sb[1].sb);
1051 	if (ret == 0)
1052 		sb_good[1] = true;
1053 
1054 	/* Use highest generation sb first */
1055 	if (!sb_good[0] && !sb_good[1]) {
1056 		dmz_dev_err(zmd->dev, "No valid super block found");
1057 		return -EIO;
1058 	}
1059 
1060 	if (sb_good[0])
1061 		sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);
1062 	else
1063 		ret = dmz_recover_mblocks(zmd, 0);
1064 
1065 	if (sb_good[1])
1066 		sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);
1067 	else
1068 		ret = dmz_recover_mblocks(zmd, 1);
1069 
1070 	if (ret) {
1071 		dmz_dev_err(zmd->dev, "Recovery failed");
1072 		return -EIO;
1073 	}
1074 
1075 	if (sb_gen[0] >= sb_gen[1]) {
1076 		zmd->sb_gen = sb_gen[0];
1077 		zmd->mblk_primary = 0;
1078 	} else {
1079 		zmd->sb_gen = sb_gen[1];
1080 		zmd->mblk_primary = 1;
1081 	}
1082 
1083 	dmz_dev_debug(zmd->dev, "Using super block %u (gen %llu)",
1084 		      zmd->mblk_primary, zmd->sb_gen);
1085 
1086 	return 0;
1087 }
1088 
1089 /*
1090  * Initialize a zone descriptor.
1091  */
1092 static int dmz_init_zone(struct blk_zone *blkz, unsigned int idx, void *data)
1093 {
1094 	struct dmz_metadata *zmd = data;
1095 	struct dm_zone *zone = &zmd->zones[idx];
1096 	struct dmz_dev *dev = zmd->dev;
1097 
1098 	/* Ignore the eventual last runt (smaller) zone */
1099 	if (blkz->len != dev->zone_nr_sectors) {
1100 		if (blkz->start + blkz->len == dev->capacity)
1101 			return 0;
1102 		return -ENXIO;
1103 	}
1104 
1105 	INIT_LIST_HEAD(&zone->link);
1106 	atomic_set(&zone->refcount, 0);
1107 	zone->chunk = DMZ_MAP_UNMAPPED;
1108 
1109 	switch (blkz->type) {
1110 	case BLK_ZONE_TYPE_CONVENTIONAL:
1111 		set_bit(DMZ_RND, &zone->flags);
1112 		break;
1113 	case BLK_ZONE_TYPE_SEQWRITE_REQ:
1114 	case BLK_ZONE_TYPE_SEQWRITE_PREF:
1115 		set_bit(DMZ_SEQ, &zone->flags);
1116 		break;
1117 	default:
1118 		return -ENXIO;
1119 	}
1120 
1121 	if (dmz_is_rnd(zone))
1122 		zone->wp_block = 0;
1123 	else
1124 		zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
1125 
1126 	if (blkz->cond == BLK_ZONE_COND_OFFLINE)
1127 		set_bit(DMZ_OFFLINE, &zone->flags);
1128 	else if (blkz->cond == BLK_ZONE_COND_READONLY)
1129 		set_bit(DMZ_READ_ONLY, &zone->flags);
1130 	else {
1131 		zmd->nr_useable_zones++;
1132 		if (dmz_is_rnd(zone)) {
1133 			zmd->nr_rnd_zones++;
1134 			if (!zmd->sb_zone) {
1135 				/* Super block zone */
1136 				zmd->sb_zone = zone;
1137 			}
1138 		}
1139 	}
1140 
1141 	return 0;
1142 }
1143 
1144 /*
1145  * Free zones descriptors.
1146  */
1147 static void dmz_drop_zones(struct dmz_metadata *zmd)
1148 {
1149 	kfree(zmd->zones);
1150 	zmd->zones = NULL;
1151 }
1152 
1153 /*
1154  * Allocate and initialize zone descriptors using the zone
1155  * information from disk.
1156  */
1157 static int dmz_init_zones(struct dmz_metadata *zmd)
1158 {
1159 	struct dmz_dev *dev = zmd->dev;
1160 	int ret;
1161 
1162 	/* Init */
1163 	zmd->zone_bitmap_size = dev->zone_nr_blocks >> 3;
1164 	zmd->zone_nr_bitmap_blocks =
1165 		max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);
1166 	zmd->zone_bits_per_mblk = min_t(sector_t, dev->zone_nr_blocks,
1167 					DMZ_BLOCK_SIZE_BITS);
1168 
1169 	/* Allocate zone array */
1170 	zmd->zones = kcalloc(dev->nr_zones, sizeof(struct dm_zone), GFP_KERNEL);
1171 	if (!zmd->zones)
1172 		return -ENOMEM;
1173 
1174 	dmz_dev_info(dev, "Using %zu B for zone information",
1175 		     sizeof(struct dm_zone) * dev->nr_zones);
1176 
1177 	/*
1178 	 * Get zone information and initialize zone descriptors.  At the same
1179 	 * time, determine where the super block should be: first block of the
1180 	 * first randomly writable zone.
1181 	 */
1182 	ret = blkdev_report_zones(dev->bdev, 0, BLK_ALL_ZONES, dmz_init_zone,
1183 				  zmd);
1184 	if (ret < 0) {
1185 		dmz_drop_zones(zmd);
1186 		return ret;
1187 	}
1188 
1189 	return 0;
1190 }
1191 
1192 static int dmz_update_zone_cb(struct blk_zone *blkz, unsigned int idx,
1193 			      void *data)
1194 {
1195 	struct dm_zone *zone = data;
1196 
1197 	clear_bit(DMZ_OFFLINE, &zone->flags);
1198 	clear_bit(DMZ_READ_ONLY, &zone->flags);
1199 	if (blkz->cond == BLK_ZONE_COND_OFFLINE)
1200 		set_bit(DMZ_OFFLINE, &zone->flags);
1201 	else if (blkz->cond == BLK_ZONE_COND_READONLY)
1202 		set_bit(DMZ_READ_ONLY, &zone->flags);
1203 
1204 	if (dmz_is_seq(zone))
1205 		zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
1206 	else
1207 		zone->wp_block = 0;
1208 	return 0;
1209 }
1210 
1211 /*
1212  * Update a zone information.
1213  */
1214 static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1215 {
1216 	unsigned int noio_flag;
1217 	int ret;
1218 
1219 	/*
1220 	 * Get zone information from disk. Since blkdev_report_zones() uses
1221 	 * GFP_KERNEL by default for memory allocations, set the per-task
1222 	 * PF_MEMALLOC_NOIO flag so that all allocations are done as if
1223 	 * GFP_NOIO was specified.
1224 	 */
1225 	noio_flag = memalloc_noio_save();
1226 	ret = blkdev_report_zones(zmd->dev->bdev, dmz_start_sect(zmd, zone), 1,
1227 				  dmz_update_zone_cb, zone);
1228 	memalloc_noio_restore(noio_flag);
1229 
1230 	if (ret == 0)
1231 		ret = -EIO;
1232 	if (ret < 0) {
1233 		dmz_dev_err(zmd->dev, "Get zone %u report failed",
1234 			    dmz_id(zmd, zone));
1235 		dmz_check_bdev(zmd->dev);
1236 		return ret;
1237 	}
1238 
1239 	return 0;
1240 }
1241 
1242 /*
1243  * Check a zone write pointer position when the zone is marked
1244  * with the sequential write error flag.
1245  */
1246 static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,
1247 				    struct dm_zone *zone)
1248 {
1249 	unsigned int wp = 0;
1250 	int ret;
1251 
1252 	wp = zone->wp_block;
1253 	ret = dmz_update_zone(zmd, zone);
1254 	if (ret)
1255 		return ret;
1256 
1257 	dmz_dev_warn(zmd->dev, "Processing zone %u write error (zone wp %u/%u)",
1258 		     dmz_id(zmd, zone), zone->wp_block, wp);
1259 
1260 	if (zone->wp_block < wp) {
1261 		dmz_invalidate_blocks(zmd, zone, zone->wp_block,
1262 				      wp - zone->wp_block);
1263 	}
1264 
1265 	return 0;
1266 }
1267 
1268 static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)
1269 {
1270 	return &zmd->zones[zone_id];
1271 }
1272 
1273 /*
1274  * Reset a zone write pointer.
1275  */
1276 static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1277 {
1278 	int ret;
1279 
1280 	/*
1281 	 * Ignore offline zones, read only zones,
1282 	 * and conventional zones.
1283 	 */
1284 	if (dmz_is_offline(zone) ||
1285 	    dmz_is_readonly(zone) ||
1286 	    dmz_is_rnd(zone))
1287 		return 0;
1288 
1289 	if (!dmz_is_empty(zone) || dmz_seq_write_err(zone)) {
1290 		struct dmz_dev *dev = zmd->dev;
1291 
1292 		ret = blkdev_zone_mgmt(dev->bdev, REQ_OP_ZONE_RESET,
1293 				       dmz_start_sect(zmd, zone),
1294 				       dev->zone_nr_sectors, GFP_NOIO);
1295 		if (ret) {
1296 			dmz_dev_err(dev, "Reset zone %u failed %d",
1297 				    dmz_id(zmd, zone), ret);
1298 			return ret;
1299 		}
1300 	}
1301 
1302 	/* Clear write error bit and rewind write pointer position */
1303 	clear_bit(DMZ_SEQ_WRITE_ERR, &zone->flags);
1304 	zone->wp_block = 0;
1305 
1306 	return 0;
1307 }
1308 
1309 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
1310 
1311 /*
1312  * Initialize chunk mapping.
1313  */
1314 static int dmz_load_mapping(struct dmz_metadata *zmd)
1315 {
1316 	struct dmz_dev *dev = zmd->dev;
1317 	struct dm_zone *dzone, *bzone;
1318 	struct dmz_mblock *dmap_mblk = NULL;
1319 	struct dmz_map *dmap;
1320 	unsigned int i = 0, e = 0, chunk = 0;
1321 	unsigned int dzone_id;
1322 	unsigned int bzone_id;
1323 
1324 	/* Metadata block array for the chunk mapping table */
1325 	zmd->map_mblk = kcalloc(zmd->nr_map_blocks,
1326 				sizeof(struct dmz_mblk *), GFP_KERNEL);
1327 	if (!zmd->map_mblk)
1328 		return -ENOMEM;
1329 
1330 	/* Get chunk mapping table blocks and initialize zone mapping */
1331 	while (chunk < zmd->nr_chunks) {
1332 		if (!dmap_mblk) {
1333 			/* Get mapping block */
1334 			dmap_mblk = dmz_get_mblock(zmd, i + 1);
1335 			if (IS_ERR(dmap_mblk))
1336 				return PTR_ERR(dmap_mblk);
1337 			zmd->map_mblk[i] = dmap_mblk;
1338 			dmap = (struct dmz_map *) dmap_mblk->data;
1339 			i++;
1340 			e = 0;
1341 		}
1342 
1343 		/* Check data zone */
1344 		dzone_id = le32_to_cpu(dmap[e].dzone_id);
1345 		if (dzone_id == DMZ_MAP_UNMAPPED)
1346 			goto next;
1347 
1348 		if (dzone_id >= dev->nr_zones) {
1349 			dmz_dev_err(dev, "Chunk %u mapping: invalid data zone ID %u",
1350 				    chunk, dzone_id);
1351 			return -EIO;
1352 		}
1353 
1354 		dzone = dmz_get(zmd, dzone_id);
1355 		set_bit(DMZ_DATA, &dzone->flags);
1356 		dzone->chunk = chunk;
1357 		dmz_get_zone_weight(zmd, dzone);
1358 
1359 		if (dmz_is_rnd(dzone))
1360 			list_add_tail(&dzone->link, &zmd->map_rnd_list);
1361 		else
1362 			list_add_tail(&dzone->link, &zmd->map_seq_list);
1363 
1364 		/* Check buffer zone */
1365 		bzone_id = le32_to_cpu(dmap[e].bzone_id);
1366 		if (bzone_id == DMZ_MAP_UNMAPPED)
1367 			goto next;
1368 
1369 		if (bzone_id >= dev->nr_zones) {
1370 			dmz_dev_err(dev, "Chunk %u mapping: invalid buffer zone ID %u",
1371 				    chunk, bzone_id);
1372 			return -EIO;
1373 		}
1374 
1375 		bzone = dmz_get(zmd, bzone_id);
1376 		if (!dmz_is_rnd(bzone)) {
1377 			dmz_dev_err(dev, "Chunk %u mapping: invalid buffer zone %u",
1378 				    chunk, bzone_id);
1379 			return -EIO;
1380 		}
1381 
1382 		set_bit(DMZ_DATA, &bzone->flags);
1383 		set_bit(DMZ_BUF, &bzone->flags);
1384 		bzone->chunk = chunk;
1385 		bzone->bzone = dzone;
1386 		dzone->bzone = bzone;
1387 		dmz_get_zone_weight(zmd, bzone);
1388 		list_add_tail(&bzone->link, &zmd->map_rnd_list);
1389 next:
1390 		chunk++;
1391 		e++;
1392 		if (e >= DMZ_MAP_ENTRIES)
1393 			dmap_mblk = NULL;
1394 	}
1395 
1396 	/*
1397 	 * At this point, only meta zones and mapped data zones were
1398 	 * fully initialized. All remaining zones are unmapped data
1399 	 * zones. Finish initializing those here.
1400 	 */
1401 	for (i = 0; i < dev->nr_zones; i++) {
1402 		dzone = dmz_get(zmd, i);
1403 		if (dmz_is_meta(dzone))
1404 			continue;
1405 
1406 		if (dmz_is_rnd(dzone))
1407 			zmd->nr_rnd++;
1408 		else
1409 			zmd->nr_seq++;
1410 
1411 		if (dmz_is_data(dzone)) {
1412 			/* Already initialized */
1413 			continue;
1414 		}
1415 
1416 		/* Unmapped data zone */
1417 		set_bit(DMZ_DATA, &dzone->flags);
1418 		dzone->chunk = DMZ_MAP_UNMAPPED;
1419 		if (dmz_is_rnd(dzone)) {
1420 			list_add_tail(&dzone->link, &zmd->unmap_rnd_list);
1421 			atomic_inc(&zmd->unmap_nr_rnd);
1422 		} else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {
1423 			list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);
1424 			atomic_inc(&zmd->nr_reserved_seq_zones);
1425 			zmd->nr_seq--;
1426 		} else {
1427 			list_add_tail(&dzone->link, &zmd->unmap_seq_list);
1428 			atomic_inc(&zmd->unmap_nr_seq);
1429 		}
1430 	}
1431 
1432 	return 0;
1433 }
1434 
1435 /*
1436  * Set a data chunk mapping.
1437  */
1438 static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,
1439 				  unsigned int dzone_id, unsigned int bzone_id)
1440 {
1441 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
1442 	struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
1443 	int map_idx = chunk & DMZ_MAP_ENTRIES_MASK;
1444 
1445 	dmap[map_idx].dzone_id = cpu_to_le32(dzone_id);
1446 	dmap[map_idx].bzone_id = cpu_to_le32(bzone_id);
1447 	dmz_dirty_mblock(zmd, dmap_mblk);
1448 }
1449 
1450 /*
1451  * The list of mapped zones is maintained in LRU order.
1452  * This rotates a zone at the end of its map list.
1453  */
1454 static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1455 {
1456 	if (list_empty(&zone->link))
1457 		return;
1458 
1459 	list_del_init(&zone->link);
1460 	if (dmz_is_seq(zone)) {
1461 		/* LRU rotate sequential zone */
1462 		list_add_tail(&zone->link, &zmd->map_seq_list);
1463 	} else {
1464 		/* LRU rotate random zone */
1465 		list_add_tail(&zone->link, &zmd->map_rnd_list);
1466 	}
1467 }
1468 
1469 /*
1470  * The list of mapped random zones is maintained
1471  * in LRU order. This rotates a zone at the end of the list.
1472  */
1473 static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1474 {
1475 	__dmz_lru_zone(zmd, zone);
1476 	if (zone->bzone)
1477 		__dmz_lru_zone(zmd, zone->bzone);
1478 }
1479 
1480 /*
1481  * Wait for any zone to be freed.
1482  */
1483 static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)
1484 {
1485 	DEFINE_WAIT(wait);
1486 
1487 	prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE);
1488 	dmz_unlock_map(zmd);
1489 	dmz_unlock_metadata(zmd);
1490 
1491 	io_schedule_timeout(HZ);
1492 
1493 	dmz_lock_metadata(zmd);
1494 	dmz_lock_map(zmd);
1495 	finish_wait(&zmd->free_wq, &wait);
1496 }
1497 
1498 /*
1499  * Lock a zone for reclaim (set the zone RECLAIM bit).
1500  * Returns false if the zone cannot be locked or if it is already locked
1501  * and 1 otherwise.
1502  */
1503 int dmz_lock_zone_reclaim(struct dm_zone *zone)
1504 {
1505 	/* Active zones cannot be reclaimed */
1506 	if (dmz_is_active(zone))
1507 		return 0;
1508 
1509 	return !test_and_set_bit(DMZ_RECLAIM, &zone->flags);
1510 }
1511 
1512 /*
1513  * Clear a zone reclaim flag.
1514  */
1515 void dmz_unlock_zone_reclaim(struct dm_zone *zone)
1516 {
1517 	WARN_ON(dmz_is_active(zone));
1518 	WARN_ON(!dmz_in_reclaim(zone));
1519 
1520 	clear_bit_unlock(DMZ_RECLAIM, &zone->flags);
1521 	smp_mb__after_atomic();
1522 	wake_up_bit(&zone->flags, DMZ_RECLAIM);
1523 }
1524 
1525 /*
1526  * Wait for a zone reclaim to complete.
1527  */
1528 static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)
1529 {
1530 	dmz_unlock_map(zmd);
1531 	dmz_unlock_metadata(zmd);
1532 	wait_on_bit_timeout(&zone->flags, DMZ_RECLAIM, TASK_UNINTERRUPTIBLE, HZ);
1533 	dmz_lock_metadata(zmd);
1534 	dmz_lock_map(zmd);
1535 }
1536 
1537 /*
1538  * Select a random write zone for reclaim.
1539  */
1540 static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd)
1541 {
1542 	struct dm_zone *dzone = NULL;
1543 	struct dm_zone *zone;
1544 
1545 	if (list_empty(&zmd->map_rnd_list))
1546 		return ERR_PTR(-EBUSY);
1547 
1548 	list_for_each_entry(zone, &zmd->map_rnd_list, link) {
1549 		if (dmz_is_buf(zone))
1550 			dzone = zone->bzone;
1551 		else
1552 			dzone = zone;
1553 		if (dmz_lock_zone_reclaim(dzone))
1554 			return dzone;
1555 	}
1556 
1557 	return ERR_PTR(-EBUSY);
1558 }
1559 
1560 /*
1561  * Select a buffered sequential zone for reclaim.
1562  */
1563 static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd)
1564 {
1565 	struct dm_zone *zone;
1566 
1567 	if (list_empty(&zmd->map_seq_list))
1568 		return ERR_PTR(-EBUSY);
1569 
1570 	list_for_each_entry(zone, &zmd->map_seq_list, link) {
1571 		if (!zone->bzone)
1572 			continue;
1573 		if (dmz_lock_zone_reclaim(zone))
1574 			return zone;
1575 	}
1576 
1577 	return ERR_PTR(-EBUSY);
1578 }
1579 
1580 /*
1581  * Select a zone for reclaim.
1582  */
1583 struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd)
1584 {
1585 	struct dm_zone *zone;
1586 
1587 	/*
1588 	 * Search for a zone candidate to reclaim: 2 cases are possible.
1589 	 * (1) There is no free sequential zones. Then a random data zone
1590 	 *     cannot be reclaimed. So choose a sequential zone to reclaim so
1591 	 *     that afterward a random zone can be reclaimed.
1592 	 * (2) At least one free sequential zone is available, then choose
1593 	 *     the oldest random zone (data or buffer) that can be locked.
1594 	 */
1595 	dmz_lock_map(zmd);
1596 	if (list_empty(&zmd->reserved_seq_zones_list))
1597 		zone = dmz_get_seq_zone_for_reclaim(zmd);
1598 	else
1599 		zone = dmz_get_rnd_zone_for_reclaim(zmd);
1600 	dmz_unlock_map(zmd);
1601 
1602 	return zone;
1603 }
1604 
1605 /*
1606  * Get the zone mapping a chunk, if the chunk is mapped already.
1607  * If no mapping exist and the operation is WRITE, a zone is
1608  * allocated and used to map the chunk.
1609  * The zone returned will be set to the active state.
1610  */
1611 struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk, int op)
1612 {
1613 	struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
1614 	struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
1615 	int dmap_idx = chunk & DMZ_MAP_ENTRIES_MASK;
1616 	unsigned int dzone_id;
1617 	struct dm_zone *dzone = NULL;
1618 	int ret = 0;
1619 
1620 	dmz_lock_map(zmd);
1621 again:
1622 	/* Get the chunk mapping */
1623 	dzone_id = le32_to_cpu(dmap[dmap_idx].dzone_id);
1624 	if (dzone_id == DMZ_MAP_UNMAPPED) {
1625 		/*
1626 		 * Read or discard in unmapped chunks are fine. But for
1627 		 * writes, we need a mapping, so get one.
1628 		 */
1629 		if (op != REQ_OP_WRITE)
1630 			goto out;
1631 
1632 		/* Allocate a random zone */
1633 		dzone = dmz_alloc_zone(zmd, DMZ_ALLOC_RND);
1634 		if (!dzone) {
1635 			if (dmz_bdev_is_dying(zmd->dev)) {
1636 				dzone = ERR_PTR(-EIO);
1637 				goto out;
1638 			}
1639 			dmz_wait_for_free_zones(zmd);
1640 			goto again;
1641 		}
1642 
1643 		dmz_map_zone(zmd, dzone, chunk);
1644 
1645 	} else {
1646 		/* The chunk is already mapped: get the mapping zone */
1647 		dzone = dmz_get(zmd, dzone_id);
1648 		if (dzone->chunk != chunk) {
1649 			dzone = ERR_PTR(-EIO);
1650 			goto out;
1651 		}
1652 
1653 		/* Repair write pointer if the sequential dzone has error */
1654 		if (dmz_seq_write_err(dzone)) {
1655 			ret = dmz_handle_seq_write_err(zmd, dzone);
1656 			if (ret) {
1657 				dzone = ERR_PTR(-EIO);
1658 				goto out;
1659 			}
1660 			clear_bit(DMZ_SEQ_WRITE_ERR, &dzone->flags);
1661 		}
1662 	}
1663 
1664 	/*
1665 	 * If the zone is being reclaimed, the chunk mapping may change
1666 	 * to a different zone. So wait for reclaim and retry. Otherwise,
1667 	 * activate the zone (this will prevent reclaim from touching it).
1668 	 */
1669 	if (dmz_in_reclaim(dzone)) {
1670 		dmz_wait_for_reclaim(zmd, dzone);
1671 		goto again;
1672 	}
1673 	dmz_activate_zone(dzone);
1674 	dmz_lru_zone(zmd, dzone);
1675 out:
1676 	dmz_unlock_map(zmd);
1677 
1678 	return dzone;
1679 }
1680 
1681 /*
1682  * Write and discard change the block validity of data zones and their buffer
1683  * zones. Check here that valid blocks are still present. If all blocks are
1684  * invalid, the zones can be unmapped on the fly without waiting for reclaim
1685  * to do it.
1686  */
1687 void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)
1688 {
1689 	struct dm_zone *bzone;
1690 
1691 	dmz_lock_map(zmd);
1692 
1693 	bzone = dzone->bzone;
1694 	if (bzone) {
1695 		if (dmz_weight(bzone))
1696 			dmz_lru_zone(zmd, bzone);
1697 		else {
1698 			/* Empty buffer zone: reclaim it */
1699 			dmz_unmap_zone(zmd, bzone);
1700 			dmz_free_zone(zmd, bzone);
1701 			bzone = NULL;
1702 		}
1703 	}
1704 
1705 	/* Deactivate the data zone */
1706 	dmz_deactivate_zone(dzone);
1707 	if (dmz_is_active(dzone) || bzone || dmz_weight(dzone))
1708 		dmz_lru_zone(zmd, dzone);
1709 	else {
1710 		/* Unbuffered inactive empty data zone: reclaim it */
1711 		dmz_unmap_zone(zmd, dzone);
1712 		dmz_free_zone(zmd, dzone);
1713 	}
1714 
1715 	dmz_unlock_map(zmd);
1716 }
1717 
1718 /*
1719  * Allocate and map a random zone to buffer a chunk
1720  * already mapped to a sequential zone.
1721  */
1722 struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,
1723 				     struct dm_zone *dzone)
1724 {
1725 	struct dm_zone *bzone;
1726 
1727 	dmz_lock_map(zmd);
1728 again:
1729 	bzone = dzone->bzone;
1730 	if (bzone)
1731 		goto out;
1732 
1733 	/* Allocate a random zone */
1734 	bzone = dmz_alloc_zone(zmd, DMZ_ALLOC_RND);
1735 	if (!bzone) {
1736 		if (dmz_bdev_is_dying(zmd->dev)) {
1737 			bzone = ERR_PTR(-EIO);
1738 			goto out;
1739 		}
1740 		dmz_wait_for_free_zones(zmd);
1741 		goto again;
1742 	}
1743 
1744 	/* Update the chunk mapping */
1745 	dmz_set_chunk_mapping(zmd, dzone->chunk, dmz_id(zmd, dzone),
1746 			      dmz_id(zmd, bzone));
1747 
1748 	set_bit(DMZ_BUF, &bzone->flags);
1749 	bzone->chunk = dzone->chunk;
1750 	bzone->bzone = dzone;
1751 	dzone->bzone = bzone;
1752 	list_add_tail(&bzone->link, &zmd->map_rnd_list);
1753 out:
1754 	dmz_unlock_map(zmd);
1755 
1756 	return bzone;
1757 }
1758 
1759 /*
1760  * Get an unmapped (free) zone.
1761  * This must be called with the mapping lock held.
1762  */
1763 struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned long flags)
1764 {
1765 	struct list_head *list;
1766 	struct dm_zone *zone;
1767 
1768 	if (flags & DMZ_ALLOC_RND)
1769 		list = &zmd->unmap_rnd_list;
1770 	else
1771 		list = &zmd->unmap_seq_list;
1772 again:
1773 	if (list_empty(list)) {
1774 		/*
1775 		 * No free zone: if this is for reclaim, allow using the
1776 		 * reserved sequential zones.
1777 		 */
1778 		if (!(flags & DMZ_ALLOC_RECLAIM) ||
1779 		    list_empty(&zmd->reserved_seq_zones_list))
1780 			return NULL;
1781 
1782 		zone = list_first_entry(&zmd->reserved_seq_zones_list,
1783 					struct dm_zone, link);
1784 		list_del_init(&zone->link);
1785 		atomic_dec(&zmd->nr_reserved_seq_zones);
1786 		return zone;
1787 	}
1788 
1789 	zone = list_first_entry(list, struct dm_zone, link);
1790 	list_del_init(&zone->link);
1791 
1792 	if (dmz_is_rnd(zone))
1793 		atomic_dec(&zmd->unmap_nr_rnd);
1794 	else
1795 		atomic_dec(&zmd->unmap_nr_seq);
1796 
1797 	if (dmz_is_offline(zone)) {
1798 		dmz_dev_warn(zmd->dev, "Zone %u is offline", dmz_id(zmd, zone));
1799 		zone = NULL;
1800 		goto again;
1801 	}
1802 
1803 	return zone;
1804 }
1805 
1806 /*
1807  * Free a zone.
1808  * This must be called with the mapping lock held.
1809  */
1810 void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1811 {
1812 	/* If this is a sequential zone, reset it */
1813 	if (dmz_is_seq(zone))
1814 		dmz_reset_zone(zmd, zone);
1815 
1816 	/* Return the zone to its type unmap list */
1817 	if (dmz_is_rnd(zone)) {
1818 		list_add_tail(&zone->link, &zmd->unmap_rnd_list);
1819 		atomic_inc(&zmd->unmap_nr_rnd);
1820 	} else if (atomic_read(&zmd->nr_reserved_seq_zones) <
1821 		   zmd->nr_reserved_seq) {
1822 		list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);
1823 		atomic_inc(&zmd->nr_reserved_seq_zones);
1824 	} else {
1825 		list_add_tail(&zone->link, &zmd->unmap_seq_list);
1826 		atomic_inc(&zmd->unmap_nr_seq);
1827 	}
1828 
1829 	wake_up_all(&zmd->free_wq);
1830 }
1831 
1832 /*
1833  * Map a chunk to a zone.
1834  * This must be called with the mapping lock held.
1835  */
1836 void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,
1837 		  unsigned int chunk)
1838 {
1839 	/* Set the chunk mapping */
1840 	dmz_set_chunk_mapping(zmd, chunk, dmz_id(zmd, dzone),
1841 			      DMZ_MAP_UNMAPPED);
1842 	dzone->chunk = chunk;
1843 	if (dmz_is_rnd(dzone))
1844 		list_add_tail(&dzone->link, &zmd->map_rnd_list);
1845 	else
1846 		list_add_tail(&dzone->link, &zmd->map_seq_list);
1847 }
1848 
1849 /*
1850  * Unmap a zone.
1851  * This must be called with the mapping lock held.
1852  */
1853 void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1854 {
1855 	unsigned int chunk = zone->chunk;
1856 	unsigned int dzone_id;
1857 
1858 	if (chunk == DMZ_MAP_UNMAPPED) {
1859 		/* Already unmapped */
1860 		return;
1861 	}
1862 
1863 	if (test_and_clear_bit(DMZ_BUF, &zone->flags)) {
1864 		/*
1865 		 * Unmapping the chunk buffer zone: clear only
1866 		 * the chunk buffer mapping
1867 		 */
1868 		dzone_id = dmz_id(zmd, zone->bzone);
1869 		zone->bzone->bzone = NULL;
1870 		zone->bzone = NULL;
1871 
1872 	} else {
1873 		/*
1874 		 * Unmapping the chunk data zone: the zone must
1875 		 * not be buffered.
1876 		 */
1877 		if (WARN_ON(zone->bzone)) {
1878 			zone->bzone->bzone = NULL;
1879 			zone->bzone = NULL;
1880 		}
1881 		dzone_id = DMZ_MAP_UNMAPPED;
1882 	}
1883 
1884 	dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);
1885 
1886 	zone->chunk = DMZ_MAP_UNMAPPED;
1887 	list_del_init(&zone->link);
1888 }
1889 
1890 /*
1891  * Set @nr_bits bits in @bitmap starting from @bit.
1892  * Return the number of bits changed from 0 to 1.
1893  */
1894 static unsigned int dmz_set_bits(unsigned long *bitmap,
1895 				 unsigned int bit, unsigned int nr_bits)
1896 {
1897 	unsigned long *addr;
1898 	unsigned int end = bit + nr_bits;
1899 	unsigned int n = 0;
1900 
1901 	while (bit < end) {
1902 		if (((bit & (BITS_PER_LONG - 1)) == 0) &&
1903 		    ((end - bit) >= BITS_PER_LONG)) {
1904 			/* Try to set the whole word at once */
1905 			addr = bitmap + BIT_WORD(bit);
1906 			if (*addr == 0) {
1907 				*addr = ULONG_MAX;
1908 				n += BITS_PER_LONG;
1909 				bit += BITS_PER_LONG;
1910 				continue;
1911 			}
1912 		}
1913 
1914 		if (!test_and_set_bit(bit, bitmap))
1915 			n++;
1916 		bit++;
1917 	}
1918 
1919 	return n;
1920 }
1921 
1922 /*
1923  * Get the bitmap block storing the bit for chunk_block in zone.
1924  */
1925 static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,
1926 					 struct dm_zone *zone,
1927 					 sector_t chunk_block)
1928 {
1929 	sector_t bitmap_block = 1 + zmd->nr_map_blocks +
1930 		(sector_t)(dmz_id(zmd, zone) * zmd->zone_nr_bitmap_blocks) +
1931 		(chunk_block >> DMZ_BLOCK_SHIFT_BITS);
1932 
1933 	return dmz_get_mblock(zmd, bitmap_block);
1934 }
1935 
1936 /*
1937  * Copy the valid blocks bitmap of from_zone to the bitmap of to_zone.
1938  */
1939 int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
1940 			  struct dm_zone *to_zone)
1941 {
1942 	struct dmz_mblock *from_mblk, *to_mblk;
1943 	sector_t chunk_block = 0;
1944 
1945 	/* Get the zones bitmap blocks */
1946 	while (chunk_block < zmd->dev->zone_nr_blocks) {
1947 		from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);
1948 		if (IS_ERR(from_mblk))
1949 			return PTR_ERR(from_mblk);
1950 		to_mblk = dmz_get_bitmap(zmd, to_zone, chunk_block);
1951 		if (IS_ERR(to_mblk)) {
1952 			dmz_release_mblock(zmd, from_mblk);
1953 			return PTR_ERR(to_mblk);
1954 		}
1955 
1956 		memcpy(to_mblk->data, from_mblk->data, DMZ_BLOCK_SIZE);
1957 		dmz_dirty_mblock(zmd, to_mblk);
1958 
1959 		dmz_release_mblock(zmd, to_mblk);
1960 		dmz_release_mblock(zmd, from_mblk);
1961 
1962 		chunk_block += zmd->zone_bits_per_mblk;
1963 	}
1964 
1965 	to_zone->weight = from_zone->weight;
1966 
1967 	return 0;
1968 }
1969 
1970 /*
1971  * Merge the valid blocks bitmap of from_zone into the bitmap of to_zone,
1972  * starting from chunk_block.
1973  */
1974 int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
1975 			   struct dm_zone *to_zone, sector_t chunk_block)
1976 {
1977 	unsigned int nr_blocks;
1978 	int ret;
1979 
1980 	/* Get the zones bitmap blocks */
1981 	while (chunk_block < zmd->dev->zone_nr_blocks) {
1982 		/* Get a valid region from the source zone */
1983 		ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);
1984 		if (ret <= 0)
1985 			return ret;
1986 
1987 		nr_blocks = ret;
1988 		ret = dmz_validate_blocks(zmd, to_zone, chunk_block, nr_blocks);
1989 		if (ret)
1990 			return ret;
1991 
1992 		chunk_block += nr_blocks;
1993 	}
1994 
1995 	return 0;
1996 }
1997 
1998 /*
1999  * Validate all the blocks in the range [block..block+nr_blocks-1].
2000  */
2001 int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
2002 			sector_t chunk_block, unsigned int nr_blocks)
2003 {
2004 	unsigned int count, bit, nr_bits;
2005 	unsigned int zone_nr_blocks = zmd->dev->zone_nr_blocks;
2006 	struct dmz_mblock *mblk;
2007 	unsigned int n = 0;
2008 
2009 	dmz_dev_debug(zmd->dev, "=> VALIDATE zone %u, block %llu, %u blocks",
2010 		      dmz_id(zmd, zone), (unsigned long long)chunk_block,
2011 		      nr_blocks);
2012 
2013 	WARN_ON(chunk_block + nr_blocks > zone_nr_blocks);
2014 
2015 	while (nr_blocks) {
2016 		/* Get bitmap block */
2017 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2018 		if (IS_ERR(mblk))
2019 			return PTR_ERR(mblk);
2020 
2021 		/* Set bits */
2022 		bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2023 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2024 
2025 		count = dmz_set_bits((unsigned long *)mblk->data, bit, nr_bits);
2026 		if (count) {
2027 			dmz_dirty_mblock(zmd, mblk);
2028 			n += count;
2029 		}
2030 		dmz_release_mblock(zmd, mblk);
2031 
2032 		nr_blocks -= nr_bits;
2033 		chunk_block += nr_bits;
2034 	}
2035 
2036 	if (likely(zone->weight + n <= zone_nr_blocks))
2037 		zone->weight += n;
2038 	else {
2039 		dmz_dev_warn(zmd->dev, "Zone %u: weight %u should be <= %u",
2040 			     dmz_id(zmd, zone), zone->weight,
2041 			     zone_nr_blocks - n);
2042 		zone->weight = zone_nr_blocks;
2043 	}
2044 
2045 	return 0;
2046 }
2047 
2048 /*
2049  * Clear nr_bits bits in bitmap starting from bit.
2050  * Return the number of bits cleared.
2051  */
2052 static int dmz_clear_bits(unsigned long *bitmap, int bit, int nr_bits)
2053 {
2054 	unsigned long *addr;
2055 	int end = bit + nr_bits;
2056 	int n = 0;
2057 
2058 	while (bit < end) {
2059 		if (((bit & (BITS_PER_LONG - 1)) == 0) &&
2060 		    ((end - bit) >= BITS_PER_LONG)) {
2061 			/* Try to clear whole word at once */
2062 			addr = bitmap + BIT_WORD(bit);
2063 			if (*addr == ULONG_MAX) {
2064 				*addr = 0;
2065 				n += BITS_PER_LONG;
2066 				bit += BITS_PER_LONG;
2067 				continue;
2068 			}
2069 		}
2070 
2071 		if (test_and_clear_bit(bit, bitmap))
2072 			n++;
2073 		bit++;
2074 	}
2075 
2076 	return n;
2077 }
2078 
2079 /*
2080  * Invalidate all the blocks in the range [block..block+nr_blocks-1].
2081  */
2082 int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
2083 			  sector_t chunk_block, unsigned int nr_blocks)
2084 {
2085 	unsigned int count, bit, nr_bits;
2086 	struct dmz_mblock *mblk;
2087 	unsigned int n = 0;
2088 
2089 	dmz_dev_debug(zmd->dev, "=> INVALIDATE zone %u, block %llu, %u blocks",
2090 		      dmz_id(zmd, zone), (u64)chunk_block, nr_blocks);
2091 
2092 	WARN_ON(chunk_block + nr_blocks > zmd->dev->zone_nr_blocks);
2093 
2094 	while (nr_blocks) {
2095 		/* Get bitmap block */
2096 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2097 		if (IS_ERR(mblk))
2098 			return PTR_ERR(mblk);
2099 
2100 		/* Clear bits */
2101 		bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2102 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2103 
2104 		count = dmz_clear_bits((unsigned long *)mblk->data,
2105 				       bit, nr_bits);
2106 		if (count) {
2107 			dmz_dirty_mblock(zmd, mblk);
2108 			n += count;
2109 		}
2110 		dmz_release_mblock(zmd, mblk);
2111 
2112 		nr_blocks -= nr_bits;
2113 		chunk_block += nr_bits;
2114 	}
2115 
2116 	if (zone->weight >= n)
2117 		zone->weight -= n;
2118 	else {
2119 		dmz_dev_warn(zmd->dev, "Zone %u: weight %u should be >= %u",
2120 			     dmz_id(zmd, zone), zone->weight, n);
2121 		zone->weight = 0;
2122 	}
2123 
2124 	return 0;
2125 }
2126 
2127 /*
2128  * Get a block bit value.
2129  */
2130 static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2131 			  sector_t chunk_block)
2132 {
2133 	struct dmz_mblock *mblk;
2134 	int ret;
2135 
2136 	WARN_ON(chunk_block >= zmd->dev->zone_nr_blocks);
2137 
2138 	/* Get bitmap block */
2139 	mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2140 	if (IS_ERR(mblk))
2141 		return PTR_ERR(mblk);
2142 
2143 	/* Get offset */
2144 	ret = test_bit(chunk_block & DMZ_BLOCK_MASK_BITS,
2145 		       (unsigned long *) mblk->data) != 0;
2146 
2147 	dmz_release_mblock(zmd, mblk);
2148 
2149 	return ret;
2150 }
2151 
2152 /*
2153  * Return the number of blocks from chunk_block to the first block with a bit
2154  * value specified by set. Search at most nr_blocks blocks from chunk_block.
2155  */
2156 static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2157 				 sector_t chunk_block, unsigned int nr_blocks,
2158 				 int set)
2159 {
2160 	struct dmz_mblock *mblk;
2161 	unsigned int bit, set_bit, nr_bits;
2162 	unsigned int zone_bits = zmd->zone_bits_per_mblk;
2163 	unsigned long *bitmap;
2164 	int n = 0;
2165 
2166 	WARN_ON(chunk_block + nr_blocks > zmd->dev->zone_nr_blocks);
2167 
2168 	while (nr_blocks) {
2169 		/* Get bitmap block */
2170 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2171 		if (IS_ERR(mblk))
2172 			return PTR_ERR(mblk);
2173 
2174 		/* Get offset */
2175 		bitmap = (unsigned long *) mblk->data;
2176 		bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2177 		nr_bits = min(nr_blocks, zone_bits - bit);
2178 		if (set)
2179 			set_bit = find_next_bit(bitmap, zone_bits, bit);
2180 		else
2181 			set_bit = find_next_zero_bit(bitmap, zone_bits, bit);
2182 		dmz_release_mblock(zmd, mblk);
2183 
2184 		n += set_bit - bit;
2185 		if (set_bit < zone_bits)
2186 			break;
2187 
2188 		nr_blocks -= nr_bits;
2189 		chunk_block += nr_bits;
2190 	}
2191 
2192 	return n;
2193 }
2194 
2195 /*
2196  * Test if chunk_block is valid. If it is, the number of consecutive
2197  * valid blocks from chunk_block will be returned.
2198  */
2199 int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,
2200 		    sector_t chunk_block)
2201 {
2202 	int valid;
2203 
2204 	valid = dmz_test_block(zmd, zone, chunk_block);
2205 	if (valid <= 0)
2206 		return valid;
2207 
2208 	/* The block is valid: get the number of valid blocks from block */
2209 	return dmz_to_next_set_block(zmd, zone, chunk_block,
2210 				     zmd->dev->zone_nr_blocks - chunk_block, 0);
2211 }
2212 
2213 /*
2214  * Find the first valid block from @chunk_block in @zone.
2215  * If such a block is found, its number is returned using
2216  * @chunk_block and the total number of valid blocks from @chunk_block
2217  * is returned.
2218  */
2219 int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2220 			  sector_t *chunk_block)
2221 {
2222 	sector_t start_block = *chunk_block;
2223 	int ret;
2224 
2225 	ret = dmz_to_next_set_block(zmd, zone, start_block,
2226 				    zmd->dev->zone_nr_blocks - start_block, 1);
2227 	if (ret < 0)
2228 		return ret;
2229 
2230 	start_block += ret;
2231 	*chunk_block = start_block;
2232 
2233 	return dmz_to_next_set_block(zmd, zone, start_block,
2234 				     zmd->dev->zone_nr_blocks - start_block, 0);
2235 }
2236 
2237 /*
2238  * Count the number of bits set starting from bit up to bit + nr_bits - 1.
2239  */
2240 static int dmz_count_bits(void *bitmap, int bit, int nr_bits)
2241 {
2242 	unsigned long *addr;
2243 	int end = bit + nr_bits;
2244 	int n = 0;
2245 
2246 	while (bit < end) {
2247 		if (((bit & (BITS_PER_LONG - 1)) == 0) &&
2248 		    ((end - bit) >= BITS_PER_LONG)) {
2249 			addr = (unsigned long *)bitmap + BIT_WORD(bit);
2250 			if (*addr == ULONG_MAX) {
2251 				n += BITS_PER_LONG;
2252 				bit += BITS_PER_LONG;
2253 				continue;
2254 			}
2255 		}
2256 
2257 		if (test_bit(bit, bitmap))
2258 			n++;
2259 		bit++;
2260 	}
2261 
2262 	return n;
2263 }
2264 
2265 /*
2266  * Get a zone weight.
2267  */
2268 static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)
2269 {
2270 	struct dmz_mblock *mblk;
2271 	sector_t chunk_block = 0;
2272 	unsigned int bit, nr_bits;
2273 	unsigned int nr_blocks = zmd->dev->zone_nr_blocks;
2274 	void *bitmap;
2275 	int n = 0;
2276 
2277 	while (nr_blocks) {
2278 		/* Get bitmap block */
2279 		mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2280 		if (IS_ERR(mblk)) {
2281 			n = 0;
2282 			break;
2283 		}
2284 
2285 		/* Count bits in this block */
2286 		bitmap = mblk->data;
2287 		bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2288 		nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2289 		n += dmz_count_bits(bitmap, bit, nr_bits);
2290 
2291 		dmz_release_mblock(zmd, mblk);
2292 
2293 		nr_blocks -= nr_bits;
2294 		chunk_block += nr_bits;
2295 	}
2296 
2297 	zone->weight = n;
2298 }
2299 
2300 /*
2301  * Cleanup the zoned metadata resources.
2302  */
2303 static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
2304 {
2305 	struct rb_root *root;
2306 	struct dmz_mblock *mblk, *next;
2307 	int i;
2308 
2309 	/* Release zone mapping resources */
2310 	if (zmd->map_mblk) {
2311 		for (i = 0; i < zmd->nr_map_blocks; i++)
2312 			dmz_release_mblock(zmd, zmd->map_mblk[i]);
2313 		kfree(zmd->map_mblk);
2314 		zmd->map_mblk = NULL;
2315 	}
2316 
2317 	/* Release super blocks */
2318 	for (i = 0; i < 2; i++) {
2319 		if (zmd->sb[i].mblk) {
2320 			dmz_free_mblock(zmd, zmd->sb[i].mblk);
2321 			zmd->sb[i].mblk = NULL;
2322 		}
2323 	}
2324 
2325 	/* Free cached blocks */
2326 	while (!list_empty(&zmd->mblk_dirty_list)) {
2327 		mblk = list_first_entry(&zmd->mblk_dirty_list,
2328 					struct dmz_mblock, link);
2329 		dmz_dev_warn(zmd->dev, "mblock %llu still in dirty list (ref %u)",
2330 			     (u64)mblk->no, mblk->ref);
2331 		list_del_init(&mblk->link);
2332 		rb_erase(&mblk->node, &zmd->mblk_rbtree);
2333 		dmz_free_mblock(zmd, mblk);
2334 	}
2335 
2336 	while (!list_empty(&zmd->mblk_lru_list)) {
2337 		mblk = list_first_entry(&zmd->mblk_lru_list,
2338 					struct dmz_mblock, link);
2339 		list_del_init(&mblk->link);
2340 		rb_erase(&mblk->node, &zmd->mblk_rbtree);
2341 		dmz_free_mblock(zmd, mblk);
2342 	}
2343 
2344 	/* Sanity checks: the mblock rbtree should now be empty */
2345 	root = &zmd->mblk_rbtree;
2346 	rbtree_postorder_for_each_entry_safe(mblk, next, root, node) {
2347 		dmz_dev_warn(zmd->dev, "mblock %llu ref %u still in rbtree",
2348 			     (u64)mblk->no, mblk->ref);
2349 		mblk->ref = 0;
2350 		dmz_free_mblock(zmd, mblk);
2351 	}
2352 
2353 	/* Free the zone descriptors */
2354 	dmz_drop_zones(zmd);
2355 
2356 	mutex_destroy(&zmd->mblk_flush_lock);
2357 	mutex_destroy(&zmd->map_lock);
2358 }
2359 
2360 /*
2361  * Initialize the zoned metadata.
2362  */
2363 int dmz_ctr_metadata(struct dmz_dev *dev, struct dmz_metadata **metadata)
2364 {
2365 	struct dmz_metadata *zmd;
2366 	unsigned int i, zid;
2367 	struct dm_zone *zone;
2368 	int ret;
2369 
2370 	zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);
2371 	if (!zmd)
2372 		return -ENOMEM;
2373 
2374 	zmd->dev = dev;
2375 	zmd->mblk_rbtree = RB_ROOT;
2376 	init_rwsem(&zmd->mblk_sem);
2377 	mutex_init(&zmd->mblk_flush_lock);
2378 	spin_lock_init(&zmd->mblk_lock);
2379 	INIT_LIST_HEAD(&zmd->mblk_lru_list);
2380 	INIT_LIST_HEAD(&zmd->mblk_dirty_list);
2381 
2382 	mutex_init(&zmd->map_lock);
2383 	atomic_set(&zmd->unmap_nr_rnd, 0);
2384 	INIT_LIST_HEAD(&zmd->unmap_rnd_list);
2385 	INIT_LIST_HEAD(&zmd->map_rnd_list);
2386 
2387 	atomic_set(&zmd->unmap_nr_seq, 0);
2388 	INIT_LIST_HEAD(&zmd->unmap_seq_list);
2389 	INIT_LIST_HEAD(&zmd->map_seq_list);
2390 
2391 	atomic_set(&zmd->nr_reserved_seq_zones, 0);
2392 	INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);
2393 
2394 	init_waitqueue_head(&zmd->free_wq);
2395 
2396 	/* Initialize zone descriptors */
2397 	ret = dmz_init_zones(zmd);
2398 	if (ret)
2399 		goto err;
2400 
2401 	/* Get super block */
2402 	ret = dmz_load_sb(zmd);
2403 	if (ret)
2404 		goto err;
2405 
2406 	/* Set metadata zones starting from sb_zone */
2407 	zid = dmz_id(zmd, zmd->sb_zone);
2408 	for (i = 0; i < zmd->nr_meta_zones << 1; i++) {
2409 		zone = dmz_get(zmd, zid + i);
2410 		if (!dmz_is_rnd(zone))
2411 			goto err;
2412 		set_bit(DMZ_META, &zone->flags);
2413 	}
2414 
2415 	/* Load mapping table */
2416 	ret = dmz_load_mapping(zmd);
2417 	if (ret)
2418 		goto err;
2419 
2420 	/*
2421 	 * Cache size boundaries: allow at least 2 super blocks, the chunk map
2422 	 * blocks and enough blocks to be able to cache the bitmap blocks of
2423 	 * up to 16 zones when idle (min_nr_mblks). Otherwise, if busy, allow
2424 	 * the cache to add 512 more metadata blocks.
2425 	 */
2426 	zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;
2427 	zmd->max_nr_mblks = zmd->min_nr_mblks + 512;
2428 	zmd->mblk_shrinker.count_objects = dmz_mblock_shrinker_count;
2429 	zmd->mblk_shrinker.scan_objects = dmz_mblock_shrinker_scan;
2430 	zmd->mblk_shrinker.seeks = DEFAULT_SEEKS;
2431 
2432 	/* Metadata cache shrinker */
2433 	ret = register_shrinker(&zmd->mblk_shrinker);
2434 	if (ret) {
2435 		dmz_dev_err(dev, "Register metadata cache shrinker failed");
2436 		goto err;
2437 	}
2438 
2439 	dmz_dev_info(dev, "Host-%s zoned block device",
2440 		     bdev_zoned_model(dev->bdev) == BLK_ZONED_HA ?
2441 		     "aware" : "managed");
2442 	dmz_dev_info(dev, "  %llu 512-byte logical sectors",
2443 		     (u64)dev->capacity);
2444 	dmz_dev_info(dev, "  %u zones of %llu 512-byte logical sectors",
2445 		     dev->nr_zones, (u64)dev->zone_nr_sectors);
2446 	dmz_dev_info(dev, "  %u metadata zones",
2447 		     zmd->nr_meta_zones * 2);
2448 	dmz_dev_info(dev, "  %u data zones for %u chunks",
2449 		     zmd->nr_data_zones, zmd->nr_chunks);
2450 	dmz_dev_info(dev, "    %u random zones (%u unmapped)",
2451 		     zmd->nr_rnd, atomic_read(&zmd->unmap_nr_rnd));
2452 	dmz_dev_info(dev, "    %u sequential zones (%u unmapped)",
2453 		     zmd->nr_seq, atomic_read(&zmd->unmap_nr_seq));
2454 	dmz_dev_info(dev, "  %u reserved sequential data zones",
2455 		     zmd->nr_reserved_seq);
2456 
2457 	dmz_dev_debug(dev, "Format:");
2458 	dmz_dev_debug(dev, "%u metadata blocks per set (%u max cache)",
2459 		      zmd->nr_meta_blocks, zmd->max_nr_mblks);
2460 	dmz_dev_debug(dev, "  %u data zone mapping blocks",
2461 		      zmd->nr_map_blocks);
2462 	dmz_dev_debug(dev, "  %u bitmap blocks",
2463 		      zmd->nr_bitmap_blocks);
2464 
2465 	*metadata = zmd;
2466 
2467 	return 0;
2468 err:
2469 	dmz_cleanup_metadata(zmd);
2470 	kfree(zmd);
2471 	*metadata = NULL;
2472 
2473 	return ret;
2474 }
2475 
2476 /*
2477  * Cleanup the zoned metadata resources.
2478  */
2479 void dmz_dtr_metadata(struct dmz_metadata *zmd)
2480 {
2481 	unregister_shrinker(&zmd->mblk_shrinker);
2482 	dmz_cleanup_metadata(zmd);
2483 	kfree(zmd);
2484 }
2485 
2486 /*
2487  * Check zone information on resume.
2488  */
2489 int dmz_resume_metadata(struct dmz_metadata *zmd)
2490 {
2491 	struct dmz_dev *dev = zmd->dev;
2492 	struct dm_zone *zone;
2493 	sector_t wp_block;
2494 	unsigned int i;
2495 	int ret;
2496 
2497 	/* Check zones */
2498 	for (i = 0; i < dev->nr_zones; i++) {
2499 		zone = dmz_get(zmd, i);
2500 		if (!zone) {
2501 			dmz_dev_err(dev, "Unable to get zone %u", i);
2502 			return -EIO;
2503 		}
2504 
2505 		wp_block = zone->wp_block;
2506 
2507 		ret = dmz_update_zone(zmd, zone);
2508 		if (ret) {
2509 			dmz_dev_err(dev, "Broken zone %u", i);
2510 			return ret;
2511 		}
2512 
2513 		if (dmz_is_offline(zone)) {
2514 			dmz_dev_warn(dev, "Zone %u is offline", i);
2515 			continue;
2516 		}
2517 
2518 		/* Check write pointer */
2519 		if (!dmz_is_seq(zone))
2520 			zone->wp_block = 0;
2521 		else if (zone->wp_block != wp_block) {
2522 			dmz_dev_err(dev, "Zone %u: Invalid wp (%llu / %llu)",
2523 				    i, (u64)zone->wp_block, (u64)wp_block);
2524 			zone->wp_block = wp_block;
2525 			dmz_invalidate_blocks(zmd, zone, zone->wp_block,
2526 					      dev->zone_nr_blocks - zone->wp_block);
2527 		}
2528 	}
2529 
2530 	return 0;
2531 }
2532