Lines Matching +full:error +full:- +full:correcting
1 // SPDX-License-Identifier: GPL-2.0-or-later
8 #include "dm-verity-fec.h"
11 #define DM_MSG_PREFIX "verity-fec"
14 * If error correction has been configured, returns true.
18 return v->fec && v->fec->dev; in verity_fec_is_enabled()
28 ((char *)io + io->v->ti->per_io_data_size - sizeof(struct dm_verity_fec_io)); in fec_io()
38 mod = do_div(offset, v->fec->rsn); in fec_interleave()
39 return offset + mod * (v->fec->rounds << v->data_dev_block_bits); in fec_interleave()
43 * Decode an RS block using Reed-Solomon.
49 uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; in fec_decode_rs8()
51 for (i = 0; i < v->fec->roots; i++) in fec_decode_rs8()
54 return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, in fec_decode_rs8()
55 fio->erasures, 0, NULL); in fec_decode_rs8()
59 * Read error-correcting codes for the requested RS block. Returns a pointer
73 position = (index + rsb) * v->fec->roots; in fec_read_parity()
74 block = div64_u64_rem(position, v->fec->io_size, &rem); in fec_read_parity()
77 res = dm_bufio_read(v->fec->bufio, block, buf); in fec_read_parity()
80 v->data_dev->name, (unsigned long long)rsb, in fec_read_parity()
98 for (__i = 0; __i < (io)->nbufs; __i++)
113 return &fio->bufs[i][j * v->fec->rsn]; in fec_buffer_rs_block()
126 * Decode all RS blocks from buffers and copy corrected bytes into fio->output
136 u8 *par, *block, par_buf[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; in fec_decode_bufs()
145 * one corrected target byte and consumes fec->roots parity bytes. in fec_decode_bufs()
149 memcpy(&par_buf[par_buf_offset], &par[offset], v->fec->roots - par_buf_offset); in fec_decode_bufs()
153 goto error; in fec_decode_bufs()
157 fio->output[block_offset] = block[byte_index]; in fec_decode_bufs()
160 if (block_offset >= 1 << v->data_dev_block_bits) in fec_decode_bufs()
164 offset += (v->fec->roots - par_buf_offset); in fec_decode_bufs()
166 if (offset < v->fec->io_size && (offset + v->fec->roots) > v->fec->io_size) { in fec_decode_bufs()
167 par_buf_offset = v->fec->io_size - offset; in fec_decode_bufs()
173 if (offset >= v->fec->io_size) { in fec_decode_bufs()
184 error: in fec_decode_bufs()
189 v->data_dev->name, (unsigned long long)rsb, r); in fec_decode_bufs()
192 v->data_dev->name, (unsigned long long)rsb, r); in fec_decode_bufs()
204 data, 1 << v->data_dev_block_bits, in fec_is_erasure()
209 v->digest_size) != 0; in fec_is_erasure()
214 * fits into buffers. Check for erasure locations if @neras is non-NULL.
221 int i, j, target_index = -1; in fec_read_bufs()
233 if (WARN_ON(v->digest_size > sizeof(want_digest))) in fec_read_bufs()
234 return -EINVAL; in fec_read_bufs()
240 for (i = 0; i < v->fec->rsn; i++) { in fec_read_bufs()
241 ileaved = fec_interleave(v, rsb * v->fec->rsn + i); in fec_read_bufs()
250 block = ileaved >> v->data_dev_block_bits; in fec_read_bufs()
251 bufio = v->fec->data_bufio; in fec_read_bufs()
253 if (block >= v->data_blocks) { in fec_read_bufs()
254 block -= v->data_blocks; in fec_read_bufs()
260 if (unlikely(block >= v->fec->hash_blocks)) in fec_read_bufs()
263 block += v->hash_start; in fec_read_bufs()
264 bufio = v->bufio; in fec_read_bufs()
270 v->data_dev->name, in fec_read_bufs()
275 if (neras && *neras <= v->fec->roots) in fec_read_bufs()
276 fio->erasures[(*neras)++] = i; in fec_read_bufs()
282 if (bufio == v->fec->data_bufio && in fec_read_bufs()
291 * maximum number (i.e. fec->roots) of erasures in fec_read_bufs()
293 if (neras && *neras <= v->fec->roots && in fec_read_bufs()
295 fio->erasures[(*neras)++] = i; in fec_read_bufs()
305 if (k >= 1 << v->data_dev_block_bits) in fec_read_bufs()
326 if (!fio->rs) in fec_alloc_bufs()
327 fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); in fec_alloc_bufs()
330 if (fio->bufs[n]) in fec_alloc_bufs()
333 fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT); in fec_alloc_bufs()
334 if (unlikely(!fio->bufs[n])) { in fec_alloc_bufs()
336 return -ENOMEM; in fec_alloc_bufs()
342 if (fio->bufs[n]) in fec_alloc_bufs()
345 fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT); in fec_alloc_bufs()
347 if (unlikely(!fio->bufs[n])) in fec_alloc_bufs()
350 fio->nbufs = n; in fec_alloc_bufs()
352 if (!fio->output) in fec_alloc_bufs()
353 fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO); in fec_alloc_bufs()
367 memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); in fec_init_bufs()
369 memset(fio->erasures, 0, sizeof(fio->erasures)); in fec_init_bufs()
374 * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses
388 for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { in fec_decode_rsb()
400 pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; in fec_decode_rsb()
403 /* Always re-validate the corrected block against the expected hash */ in fec_decode_rsb()
404 r = verity_hash(v, verity_io_hash_req(v, io), fio->output, in fec_decode_rsb()
405 1 << v->data_dev_block_bits, in fec_decode_rsb()
411 v->digest_size)) { in fec_decode_rsb()
413 v->data_dev->name, (unsigned long long)rsb, neras); in fec_decode_rsb()
414 return -EILSEQ; in fec_decode_rsb()
425 memcpy(data, &fio->output[fio->output_pos], len); in fec_bv_copy()
426 fio->output_pos += len; in fec_bv_copy()
432 * Correct errors in a block. Copies corrected block to dest if non-NULL,
444 return -EOPNOTSUPP; in verity_fec_decode()
446 if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { in verity_fec_decode()
447 DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); in verity_fec_decode()
448 return -EIO; in verity_fec_decode()
451 fio->level++; in verity_fec_decode()
454 block = block - v->hash_start + v->data_blocks; in verity_fec_decode()
466 offset = block << v->data_dev_block_bits; in verity_fec_decode()
467 res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); in verity_fec_decode()
473 rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); in verity_fec_decode()
488 memcpy(dest, fio->output, 1 << v->data_dev_block_bits); in verity_fec_decode()
490 fio->output_pos = 0; in verity_fec_decode()
495 fio->level--; in verity_fec_decode()
500 * Clean up per-bio data.
505 struct dm_verity_fec *f = io->v->fec; in verity_fec_finish_io()
508 if (!verity_fec_is_enabled(io->v)) in verity_fec_finish_io()
511 mempool_free(fio->rs, &f->rs_pool); in verity_fec_finish_io()
514 mempool_free(fio->bufs[n], &f->prealloc_pool); in verity_fec_finish_io()
517 mempool_free(fio->bufs[n], &f->extra_pool); in verity_fec_finish_io()
519 mempool_free(fio->output, &f->output_pool); in verity_fec_finish_io()
523 * Initialize per-bio data.
529 if (!verity_fec_is_enabled(io->v)) in verity_fec_init_io()
532 fio->rs = NULL; in verity_fec_init_io()
533 memset(fio->bufs, 0, sizeof(fio->bufs)); in verity_fec_init_io()
534 fio->nbufs = 0; in verity_fec_init_io()
535 fio->output = NULL; in verity_fec_init_io()
536 fio->level = 0; in verity_fec_init_io()
552 v->fec->dev->name, in verity_fec_status_table()
553 (unsigned long long)v->fec->blocks, in verity_fec_status_table()
554 (unsigned long long)v->fec->start, in verity_fec_status_table()
555 v->fec->roots); in verity_fec_status_table()
562 struct dm_verity_fec *f = v->fec; in verity_fec_dtr()
567 mempool_exit(&f->rs_pool); in verity_fec_dtr()
568 mempool_exit(&f->prealloc_pool); in verity_fec_dtr()
569 mempool_exit(&f->extra_pool); in verity_fec_dtr()
570 mempool_exit(&f->output_pool); in verity_fec_dtr()
571 kmem_cache_destroy(f->cache); in verity_fec_dtr()
573 if (f->data_bufio) in verity_fec_dtr()
574 dm_bufio_client_destroy(f->data_bufio); in verity_fec_dtr()
575 if (f->bufio) in verity_fec_dtr()
576 dm_bufio_client_destroy(f->bufio); in verity_fec_dtr()
578 if (f->dev) in verity_fec_dtr()
579 dm_put_device(v->ti, f->dev); in verity_fec_dtr()
582 v->fec = NULL; in verity_fec_dtr()
589 return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask); in fec_rs_alloc()
612 struct dm_target *ti = v->ti; in verity_fec_parse_opt_args()
619 ti->error = "FEC feature arguments require a value"; in verity_fec_parse_opt_args()
620 return -EINVAL; in verity_fec_parse_opt_args()
624 (*argc)--; in verity_fec_parse_opt_args()
627 r = dm_get_device(ti, arg_value, BLK_OPEN_READ, &v->fec->dev); in verity_fec_parse_opt_args()
629 ti->error = "FEC device lookup failed"; in verity_fec_parse_opt_args()
635 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) in verity_fec_parse_opt_args()
636 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { in verity_fec_parse_opt_args()
637 ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; in verity_fec_parse_opt_args()
638 return -EINVAL; in verity_fec_parse_opt_args()
640 v->fec->blocks = num_ll; in verity_fec_parse_opt_args()
644 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> in verity_fec_parse_opt_args()
645 (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { in verity_fec_parse_opt_args()
646 ti->error = "Invalid " DM_VERITY_OPT_FEC_START; in verity_fec_parse_opt_args()
647 return -EINVAL; in verity_fec_parse_opt_args()
649 v->fec->start = num_ll; in verity_fec_parse_opt_args()
653 num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || in verity_fec_parse_opt_args()
654 num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { in verity_fec_parse_opt_args()
655 ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; in verity_fec_parse_opt_args()
656 return -EINVAL; in verity_fec_parse_opt_args()
658 v->fec->roots = num_c; in verity_fec_parse_opt_args()
661 ti->error = "Unrecognized verity FEC feature request"; in verity_fec_parse_opt_args()
662 return -EINVAL; in verity_fec_parse_opt_args()
669 * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
677 v->ti->error = "Cannot allocate FEC structure"; in verity_fec_ctr_alloc()
678 return -ENOMEM; in verity_fec_ctr_alloc()
680 v->fec = f; in verity_fec_ctr_alloc()
691 struct dm_verity_fec *f = v->fec; in verity_fec_ctr()
692 struct dm_target *ti = v->ti; in verity_fec_ctr()
716 hash_blocks = v->hash_blocks - v->hash_start; in verity_fec_ctr()
722 if (v->data_dev_block_bits != v->hash_dev_block_bits) { in verity_fec_ctr()
723 ti->error = "Block sizes must match to use FEC"; in verity_fec_ctr()
724 return -EINVAL; in verity_fec_ctr()
727 if (!f->roots) { in verity_fec_ctr()
728 ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; in verity_fec_ctr()
729 return -EINVAL; in verity_fec_ctr()
731 f->rsn = DM_VERITY_FEC_RSM - f->roots; in verity_fec_ctr()
733 if (!f->blocks) { in verity_fec_ctr()
734 ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; in verity_fec_ctr()
735 return -EINVAL; in verity_fec_ctr()
738 f->rounds = f->blocks; in verity_fec_ctr()
739 if (sector_div(f->rounds, f->rsn)) in verity_fec_ctr()
740 f->rounds++; in verity_fec_ctr()
743 * Due to optional metadata, f->blocks can be larger than in verity_fec_ctr()
746 if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { in verity_fec_ctr()
747 ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; in verity_fec_ctr()
748 return -EINVAL; in verity_fec_ctr()
755 f->hash_blocks = f->blocks - v->data_blocks; in verity_fec_ctr()
756 if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { in verity_fec_ctr()
757 ti->error = "Hash device is too small for " in verity_fec_ctr()
759 return -E2BIG; in verity_fec_ctr()
762 f->io_size = 1 << v->data_dev_block_bits; in verity_fec_ctr()
764 f->bufio = dm_bufio_client_create(f->dev->bdev, in verity_fec_ctr()
765 f->io_size, in verity_fec_ctr()
767 if (IS_ERR(f->bufio)) { in verity_fec_ctr()
768 ti->error = "Cannot initialize FEC bufio client"; in verity_fec_ctr()
769 return PTR_ERR(f->bufio); in verity_fec_ctr()
772 dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); in verity_fec_ctr()
774 fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT); in verity_fec_ctr()
775 if (dm_bufio_get_device_size(f->bufio) < fec_blocks) { in verity_fec_ctr()
776 ti->error = "FEC device is too small"; in verity_fec_ctr()
777 return -E2BIG; in verity_fec_ctr()
780 f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, in verity_fec_ctr()
781 1 << v->data_dev_block_bits, in verity_fec_ctr()
783 if (IS_ERR(f->data_bufio)) { in verity_fec_ctr()
784 ti->error = "Cannot initialize FEC data bufio client"; in verity_fec_ctr()
785 return PTR_ERR(f->data_bufio); in verity_fec_ctr()
788 if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { in verity_fec_ctr()
789 ti->error = "Data device is too small"; in verity_fec_ctr()
790 return -E2BIG; in verity_fec_ctr()
794 ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc, in verity_fec_ctr()
797 ti->error = "Cannot allocate RS pool"; in verity_fec_ctr()
801 f->cache = kmem_cache_create("dm_verity_fec_buffers", in verity_fec_ctr()
802 f->rsn << DM_VERITY_FEC_BUF_RS_BITS, in verity_fec_ctr()
804 if (!f->cache) { in verity_fec_ctr()
805 ti->error = "Cannot create FEC buffer cache"; in verity_fec_ctr()
806 return -ENOMEM; in verity_fec_ctr()
810 ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() * in verity_fec_ctr()
812 f->cache); in verity_fec_ctr()
814 ti->error = "Cannot allocate FEC buffer prealloc pool"; in verity_fec_ctr()
818 ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache); in verity_fec_ctr()
820 ti->error = "Cannot allocate FEC buffer extra pool"; in verity_fec_ctr()
825 ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(), in verity_fec_ctr()
826 1 << v->data_dev_block_bits); in verity_fec_ctr()
828 ti->error = "Cannot allocate FEC output pool"; in verity_fec_ctr()
832 /* Reserve space for our per-bio data */ in verity_fec_ctr()
833 ti->per_io_data_size += sizeof(struct dm_verity_fec_io); in verity_fec_ctr()