dm-integrity.c (9b031c86506cef9acae45e61339fcf9deaabb793) dm-integrity.c (5df96f2b9f58a5d2dc1f30fe7de75e197f2c25f2)
1/*
2 * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved.
3 * Copyright (C) 2016-2017 Milan Broz
4 * Copyright (C) 2016-2017 Mikulas Patocka
5 *
6 * This file is released under the GPL.
7 */
8

--- 78 unchanged lines hidden (view full) ---

87struct journal_entry {
88 union {
89 struct {
90 __u32 sector_lo;
91 __u32 sector_hi;
92 } s;
93 __u64 sector;
94 } u;
1/*
2 * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved.
3 * Copyright (C) 2016-2017 Milan Broz
4 * Copyright (C) 2016-2017 Mikulas Patocka
5 *
6 * This file is released under the GPL.
7 */
8

--- 78 unchanged lines hidden (view full) ---

87struct journal_entry {
88 union {
89 struct {
90 __u32 sector_lo;
91 __u32 sector_hi;
92 } s;
93 __u64 sector;
94 } u;
95 commit_id_t last_bytes[0];
95 commit_id_t last_bytes[];
96 /* __u8 tag[0]; */
97};
98
99#define journal_entry_tag(ic, je) ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block])
100
101#if BITS_PER_LONG == 64
102#define journal_entry_set_sector(je, x) do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0)
103#else

--- 1444 unchanged lines hidden (view full) ---

1548 struct bio_vec bv;
1549 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1550 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1551 char *checksums;
1552 unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
1553 char checksums_onstack[max((size_t)HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
1554 sector_t sector;
1555 unsigned sectors_to_process;
96 /* __u8 tag[0]; */
97};
98
99#define journal_entry_tag(ic, je) ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block])
100
101#if BITS_PER_LONG == 64
102#define journal_entry_set_sector(je, x) do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0)
103#else

--- 1444 unchanged lines hidden (view full) ---

1548 struct bio_vec bv;
1549 unsigned digest_size = crypto_shash_digestsize(ic->internal_hash);
1550 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1551 char *checksums;
1552 unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
1553 char checksums_onstack[max((size_t)HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
1554 sector_t sector;
1555 unsigned sectors_to_process;
1556 sector_t save_metadata_block;
1557 unsigned save_metadata_offset;
1558
1559 if (unlikely(ic->mode == 'R'))
1560 goto skip_io;
1561
1562 if (likely(dio->op != REQ_OP_DISCARD))
1563 checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
1564 GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1565 else

--- 34 unchanged lines hidden (view full) ---

1600 bi_sector += this_step_blocks << ic->sb->log2_sectors_per_block;
1601 }
1602
1603 if (likely(checksums != checksums_onstack))
1604 kfree(checksums);
1605 goto skip_io;
1606 }
1607
1556
1557 if (unlikely(ic->mode == 'R'))
1558 goto skip_io;
1559
1560 if (likely(dio->op != REQ_OP_DISCARD))
1561 checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
1562 GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1563 else

--- 34 unchanged lines hidden (view full) ---

1598 bi_sector += this_step_blocks << ic->sb->log2_sectors_per_block;
1599 }
1600
1601 if (likely(checksums != checksums_onstack))
1602 kfree(checksums);
1603 goto skip_io;
1604 }
1605
1608 save_metadata_block = dio->metadata_block;
1609 save_metadata_offset = dio->metadata_offset;
1610 sector = dio->range.logical_sector;
1611 sectors_to_process = dio->range.n_sectors;
1612
1613 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1614 unsigned pos;
1615 char *mem, *checksums_ptr;
1616
1617again:

--- 801 unchanged lines hidden (view full) ---

2419static void integrity_writer(struct work_struct *w)
2420{
2421 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work);
2422 unsigned write_start, write_sections;
2423
2424 unsigned prev_free_sectors;
2425
2426 /* the following test is not needed, but it tests the replay code */
1606 sector = dio->range.logical_sector;
1607 sectors_to_process = dio->range.n_sectors;
1608
1609 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1610 unsigned pos;
1611 char *mem, *checksums_ptr;
1612
1613again:

--- 801 unchanged lines hidden (view full) ---

2415static void integrity_writer(struct work_struct *w)
2416{
2417 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work);
2418 unsigned write_start, write_sections;
2419
2420 unsigned prev_free_sectors;
2421
2422 /* the following test is not needed, but it tests the replay code */
2427 if (unlikely(dm_suspended(ic->ti)) && !ic->meta_dev)
2423 if (unlikely(dm_post_suspending(ic->ti)) && !ic->meta_dev)
2428 return;
2429
2430 spin_lock_irq(&ic->endio_wait.lock);
2431 write_start = ic->committed_section;
2432 write_sections = ic->n_committed_sections;
2433 spin_unlock_irq(&ic->endio_wait.lock);
2434
2435 if (!write_sections)

--- 44 unchanged lines hidden (view full) ---

2480 unsigned super_counter = 0;
2481
2482 DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector));
2483
2484 spin_lock_irq(&ic->endio_wait.lock);
2485
2486next_chunk:
2487
2424 return;
2425
2426 spin_lock_irq(&ic->endio_wait.lock);
2427 write_start = ic->committed_section;
2428 write_sections = ic->n_committed_sections;
2429 spin_unlock_irq(&ic->endio_wait.lock);
2430
2431 if (!write_sections)

--- 44 unchanged lines hidden (view full) ---

2476 unsigned super_counter = 0;
2477
2478 DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector));
2479
2480 spin_lock_irq(&ic->endio_wait.lock);
2481
2482next_chunk:
2483
2488 if (unlikely(dm_suspended(ic->ti)))
2484 if (unlikely(dm_post_suspending(ic->ti)))
2489 goto unlock_ret;
2490
2491 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
2492 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) {
2493 if (ic->mode == 'B') {
2494 DEBUG_print("queue_delayed_work: bitmap_flush_work\n");
2495 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
2496 }

--- 155 unchanged lines hidden (view full) ---

2652 range.n_sectors = ic->provided_data_sectors;
2653
2654 spin_lock_irq(&ic->endio_wait.lock);
2655 add_new_range_and_wait(ic, &range);
2656 spin_unlock_irq(&ic->endio_wait.lock);
2657
2658 dm_integrity_flush_buffers(ic);
2659 if (ic->meta_dev)
2485 goto unlock_ret;
2486
2487 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
2488 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) {
2489 if (ic->mode == 'B') {
2490 DEBUG_print("queue_delayed_work: bitmap_flush_work\n");
2491 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
2492 }

--- 155 unchanged lines hidden (view full) ---

2648 range.n_sectors = ic->provided_data_sectors;
2649
2650 spin_lock_irq(&ic->endio_wait.lock);
2651 add_new_range_and_wait(ic, &range);
2652 spin_unlock_irq(&ic->endio_wait.lock);
2653
2654 dm_integrity_flush_buffers(ic);
2655 if (ic->meta_dev)
2660 blkdev_issue_flush(ic->dev->bdev, GFP_NOIO, NULL);
2656 blkdev_issue_flush(ic->dev->bdev, GFP_NOIO);
2661
2662 limit = ic->provided_data_sectors;
2663 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
2664 limit = le64_to_cpu(ic->sb->recalc_sector)
2665 >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)
2666 << (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
2667 }
2668 /*DEBUG_print("zeroing journal\n");*/

--- 1747 unchanged lines hidden --- 		2657
2658 limit = ic->provided_data_sectors;
2659 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
2660 limit = le64_to_cpu(ic->sb->recalc_sector)
2661 >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)
2662 << (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
2663 }
2664 /*DEBUG_print("zeroing journal\n");*/

--- 1747 unchanged lines hidden ---