138 static inline sector_t bio_to_dev_sectors(struct log_writes_c *lc,  in bio_to_dev_sectors()  argument
141 	return sectors >> (lc->sectorshift - SECTOR_SHIFT);  in bio_to_dev_sectors()
144 static inline sector_t dev_to_bio_sectors(struct log_writes_c *lc,  in dev_to_bio_sectors()  argument
147 	return sectors << (lc->sectorshift - SECTOR_SHIFT);  in dev_to_bio_sectors()
150 static void put_pending_block(struct log_writes_c *lc)  in put_pending_block()  argument
152 	if (atomic_dec_and_test(&lc->pending_blocks)) {  in put_pending_block()
154 		if (waitqueue_active(&lc->wait))  in put_pending_block()
155 			wake_up(&lc->wait);  in put_pending_block()
159 static void put_io_block(struct log_writes_c *lc)  in put_io_block()  argument
161 	if (atomic_dec_and_test(&lc->io_blocks)) {  in put_io_block()
163 		if (waitqueue_active(&lc->wait))  in put_io_block()
164 			wake_up(&lc->wait);  in put_io_block()
170 	struct log_writes_c *lc = bio->bi_private;  in log_end_io()  local
176 		spin_lock_irqsave(&lc->blocks_lock, flags);  in log_end_io()
177 		lc->logging_enabled = false;  in log_end_io()
178 		spin_unlock_irqrestore(&lc->blocks_lock, flags);  in log_end_io()
182 	put_io_block(lc);  in log_end_io()
188 	struct log_writes_c *lc = bio->bi_private;  in log_end_super()  local
190 	complete(&lc->super_done);  in log_end_super()
198 static void free_pending_block(struct log_writes_c *lc,  in free_pending_block()  argument
209 	put_pending_block(lc);  in free_pending_block()
212 static int write_metadata(struct log_writes_c *lc, void *entry,  in write_metadata()  argument
221 	bio = bio_alloc(lc->logdev->bdev, 1, REQ_OP_WRITE, GFP_KERNEL);  in write_metadata()
226 	bio->bi_private = lc;  in write_metadata()
240 	       lc->sectorsize - entrylen - datalen);  in write_metadata()
243 	ret = bio_add_page(bio, page, lc->sectorsize, 0);  in write_metadata()
244 	if (ret != lc->sectorsize) {  in write_metadata()
254 	put_io_block(lc);  in write_metadata()
258 static int write_inline_data(struct log_writes_c *lc, void *entry,  in write_inline_data()  argument
271 		atomic_inc(&lc->io_blocks);  in write_inline_data()
273 		bio = bio_alloc(lc->logdev->bdev, bio_pages, REQ_OP_WRITE,  in write_inline_data()
278 		bio->bi_private = lc;  in write_inline_data()
282 			pg_sectorlen = ALIGN(pg_datalen, lc->sectorsize);  in write_inline_data()
314 	put_io_block(lc);  in write_inline_data()
318 static int log_one_block(struct log_writes_c *lc,  in log_one_block()  argument
332 	if (write_metadata(lc, &entry, sizeof(entry), block->data,  in log_one_block()
334 		free_pending_block(lc, block);  in log_one_block()
338 	sector += dev_to_bio_sectors(lc, 1);  in log_one_block()
341 		if (write_inline_data(lc, &entry, sizeof(entry), block->data,  in log_one_block()
343 			free_pending_block(lc, block);  in log_one_block()
353 	atomic_inc(&lc->io_blocks);  in log_one_block()
354 	bio = bio_alloc(lc->logdev->bdev, bio_max_segs(block->vec_cnt),  in log_one_block()
359 	bio->bi_private = lc;  in log_one_block()
369 			atomic_inc(&lc->io_blocks);  in log_one_block()
371 			bio = bio_alloc(lc->logdev->bdev,  in log_one_block()
377 			bio->bi_private = lc;  in log_one_block()
393 	put_pending_block(lc);  in log_one_block()
396 	free_pending_block(lc, block);  in log_one_block()
397 	put_io_block(lc);  in log_one_block()
401 static int log_super(struct log_writes_c *lc)  in log_super()  argument
407 	super.nr_entries = cpu_to_le64(lc->logged_entries);  in log_super()
408 	super.sectorsize = cpu_to_le32(lc->sectorsize);  in log_super()
410 	if (write_metadata(lc, &super, sizeof(super), NULL, 0,  in log_super()
420 	wait_for_completion_io(&lc->super_done);  in log_super()
425 static inline sector_t logdev_last_sector(struct log_writes_c *lc)  in logdev_last_sector()  argument
427 	return bdev_nr_sectors(lc->logdev->bdev);  in logdev_last_sector()
432 	struct log_writes_c *lc = arg;  in log_writes_kthread()  local
441 		spin_lock_irq(&lc->blocks_lock);  in log_writes_kthread()
442 		if (!list_empty(&lc->logging_blocks)) {  in log_writes_kthread()
443 			block = list_first_entry(&lc->logging_blocks,  in log_writes_kthread()
446 			if (!lc->logging_enabled)  in log_writes_kthread()
449 			sector = lc->next_sector;  in log_writes_kthread()
451 				lc->next_sector += dev_to_bio_sectors(lc, block->nr_sectors);  in log_writes_kthread()
452 			lc->next_sector += dev_to_bio_sectors(lc, 1);  in log_writes_kthread()
458 			if (!lc->end_sector)  in log_writes_kthread()
459 				lc->end_sector = logdev_last_sector(lc);  in log_writes_kthread()
460 			if (lc->end_sector &&  in log_writes_kthread()
461 			    lc->next_sector >= lc->end_sector) {  in log_writes_kthread()
463 				lc->logging_enabled = false;  in log_writes_kthread()
466 			lc->logged_entries++;  in log_writes_kthread()
467 			atomic_inc(&lc->io_blocks);  in log_writes_kthread()
471 				atomic_inc(&lc->io_blocks);  in log_writes_kthread()
474 		logging_enabled = lc->logging_enabled;  in log_writes_kthread()
475 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_kthread()
478 				ret = log_one_block(lc, block, sector);  in log_writes_kthread()
480 					ret = log_super(lc);  in log_writes_kthread()
482 					spin_lock_irq(&lc->blocks_lock);  in log_writes_kthread()
483 					lc->logging_enabled = false;  in log_writes_kthread()
484 					spin_unlock_irq(&lc->blocks_lock);  in log_writes_kthread()
487 				free_pending_block(lc, block);  in log_writes_kthread()
494 			    list_empty(&lc->logging_blocks))  in log_writes_kthread()
508 	struct log_writes_c *lc;  in log_writes_ctr()  local
521 	lc = kzalloc(sizeof(struct log_writes_c), GFP_KERNEL);  in log_writes_ctr()
522 	if (!lc) {  in log_writes_ctr()
526 	spin_lock_init(&lc->blocks_lock);  in log_writes_ctr()
527 	INIT_LIST_HEAD(&lc->unflushed_blocks);  in log_writes_ctr()
528 	INIT_LIST_HEAD(&lc->logging_blocks);  in log_writes_ctr()
529 	init_waitqueue_head(&lc->wait);  in log_writes_ctr()
530 	init_completion(&lc->super_done);  in log_writes_ctr()
531 	atomic_set(&lc->io_blocks, 0);  in log_writes_ctr()
532 	atomic_set(&lc->pending_blocks, 0);  in log_writes_ctr()
535 	ret = dm_get_device(ti, devname, dm_table_get_mode(ti->table), &lc->dev);  in log_writes_ctr()
543 			    &lc->logdev);  in log_writes_ctr()
546 		dm_put_device(ti, lc->dev);  in log_writes_ctr()
550 	lc->sectorsize = bdev_logical_block_size(lc->dev->bdev);  in log_writes_ctr()
551 	lc->sectorshift = ilog2(lc->sectorsize);  in log_writes_ctr()
552 	lc->log_kthread = kthread_run(log_writes_kthread, lc, "log-write");  in log_writes_ctr()
553 	if (IS_ERR(lc->log_kthread)) {  in log_writes_ctr()
554 		ret = PTR_ERR(lc->log_kthread);  in log_writes_ctr()
556 		dm_put_device(ti, lc->dev);  in log_writes_ctr()
557 		dm_put_device(ti, lc->logdev);  in log_writes_ctr()
566 	lc->next_sector = lc->sectorsize >> SECTOR_SHIFT;  in log_writes_ctr()
567 	lc->logging_enabled = true;  in log_writes_ctr()
568 	lc->end_sector = logdev_last_sector(lc);  in log_writes_ctr()
569 	lc->device_supports_discard = true;  in log_writes_ctr()
576 	ti->private = lc;  in log_writes_ctr()
580 	kfree(lc);  in log_writes_ctr()
584 static int log_mark(struct log_writes_c *lc, char *data)  in log_mark()  argument
587 	size_t maxsize = lc->sectorsize - sizeof(struct log_write_entry);  in log_mark()
601 	atomic_inc(&lc->pending_blocks);  in log_mark()
604 	spin_lock_irq(&lc->blocks_lock);  in log_mark()
605 	list_add_tail(&block->list, &lc->logging_blocks);  in log_mark()
606 	spin_unlock_irq(&lc->blocks_lock);  in log_mark()
607 	wake_up_process(lc->log_kthread);  in log_mark()
613 	struct log_writes_c *lc = ti->private;  in log_writes_dtr()  local
615 	spin_lock_irq(&lc->blocks_lock);  in log_writes_dtr()
616 	list_splice_init(&lc->unflushed_blocks, &lc->logging_blocks);  in log_writes_dtr()
617 	spin_unlock_irq(&lc->blocks_lock);  in log_writes_dtr()
623 	log_mark(lc, "dm-log-writes-end");  in log_writes_dtr()
624 	wake_up_process(lc->log_kthread);  in log_writes_dtr()
625 	wait_event(lc->wait, !atomic_read(&lc->io_blocks) &&  in log_writes_dtr()
626 		   !atomic_read(&lc->pending_blocks));  in log_writes_dtr()
627 	kthread_stop(lc->log_kthread);  in log_writes_dtr()
629 	WARN_ON(!list_empty(&lc->logging_blocks));  in log_writes_dtr()
630 	WARN_ON(!list_empty(&lc->unflushed_blocks));  in log_writes_dtr()
631 	dm_put_device(ti, lc->dev);  in log_writes_dtr()
632 	dm_put_device(ti, lc->logdev);  in log_writes_dtr()
633 	kfree(lc);  in log_writes_dtr()
638 	struct log_writes_c *lc = ti->private;  in normal_map_bio()  local
640 	bio_set_dev(bio, lc->dev->bdev);  in normal_map_bio()
645 	struct log_writes_c *lc = ti->private;  in log_writes_map()  local
660 	if (!lc->logging_enabled)  in log_writes_map()
685 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
686 		lc->logging_enabled = false;  in log_writes_map()
687 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
692 	atomic_inc(&lc->pending_blocks);  in log_writes_map()
703 	block->sector = bio_to_dev_sectors(lc, bio->bi_iter.bi_sector);  in log_writes_map()
704 	block->nr_sectors = bio_to_dev_sectors(lc, bio_sectors(bio));  in log_writes_map()
709 		if (lc->device_supports_discard)  in log_writes_map()
717 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
718 		list_splice_init(&lc->unflushed_blocks, &block->list);  in log_writes_map()
719 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
739 			free_pending_block(lc, block);  in log_writes_map()
740 			spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
741 			lc->logging_enabled = false;  in log_writes_map()
742 			spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
757 		spin_lock_irq(&lc->blocks_lock);  in log_writes_map()
758 		list_splice_init(&lc->unflushed_blocks, &block->list);  in log_writes_map()
759 		spin_unlock_irq(&lc->blocks_lock);  in log_writes_map()
769 	struct log_writes_c *lc = ti->private;  in normal_end_io()  local
776 		spin_lock_irqsave(&lc->blocks_lock, flags);  in normal_end_io()
778 			list_splice_tail_init(&block->list, &lc->logging_blocks);  in normal_end_io()
779 			list_add_tail(&block->list, &lc->logging_blocks);  in normal_end_io()
780 			wake_up_process(lc->log_kthread);  in normal_end_io()
782 			list_add_tail(&block->list, &lc->logging_blocks);  in normal_end_io()
783 			wake_up_process(lc->log_kthread);  in normal_end_io()
785 			list_add_tail(&block->list, &lc->unflushed_blocks);  in normal_end_io()
786 		spin_unlock_irqrestore(&lc->blocks_lock, flags);  in normal_end_io()
800 	struct log_writes_c *lc = ti->private;  in log_writes_status()  local
804 		DMEMIT("%llu %llu", lc->logged_entries,  in log_writes_status()
805 		       (unsigned long long)lc->next_sector - 1);  in log_writes_status()
806 		if (!lc->logging_enabled)  in log_writes_status()
811 		DMEMIT("%s %s", lc->dev->name, lc->logdev->name);  in log_writes_status()
823 	struct log_writes_c *lc = ti->private;  in log_writes_prepare_ioctl()  local
824 	struct dm_dev *dev = lc->dev;  in log_writes_prepare_ioctl()
839 	struct log_writes_c *lc = ti->private;  in log_writes_iterate_devices()  local
841 	return fn(ti, lc->dev, 0, ti->len, data);  in log_writes_iterate_devices()
852 	struct log_writes_c *lc = ti->private;  in log_writes_message()  local
860 		r = log_mark(lc, argv[1]);  in log_writes_message()
869 	struct log_writes_c *lc = ti->private;  in log_writes_io_hints()  local
871 	if (!bdev_max_discard_sectors(lc->dev->bdev)) {  in log_writes_io_hints()
872 		lc->device_supports_discard = false;  in log_writes_io_hints()
873 		limits->discard_granularity = lc->sectorsize;  in log_writes_io_hints()
876 	limits->logical_block_size = bdev_logical_block_size(lc->dev->bdev);  in log_writes_io_hints()
877 	limits->physical_block_size = bdev_physical_block_size(lc->dev->bdev);  in log_writes_io_hints()
886 	struct log_writes_c *lc = ti->private;  in log_writes_dax_pgoff()  local
888 	*pgoff += (get_start_sect(lc->dev->bdev) >> PAGE_SECTORS_SHIFT);  in log_writes_dax_pgoff()
889 	return lc->dev->dax_dev;  in log_writes_dax_pgoff()