45 static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio,  in fec_decode_rs8()  argument
54 	return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras,  in fec_decode_rs8()
55 			  fio->erasures, 0, NULL);  in fec_decode_rs8()
110 				      struct dm_verity_fec_io *fio,  in fec_buffer_rs_block()  argument
113 	return &fio->bufs[i][j * v->fec->rsn];  in fec_buffer_rs_block()
129 static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio,  in fec_decode_bufs()  argument
147 	fec_for_each_buffer_rs_block(fio, n, i) {  in fec_decode_bufs()
148 		block = fec_buffer_rs_block(v, fio, n, i);  in fec_decode_bufs()
150 		res = fec_decode_rs8(v, fio, block, par_buf, neras);  in fec_decode_bufs()
157 		fio->output[block_offset] = block[byte_index];  in fec_decode_bufs()
224 	struct dm_verity_fec_io *fio = fec_io(io);  in fec_read_bufs()  local
276 				fio->erasures[(*neras)++] = i;  in fec_read_bufs()
295 				fio->erasures[(*neras)++] = i;  in fec_read_bufs()
302 		fec_for_each_buffer_rs_block(fio, n, j) {  in fec_read_bufs()
308 			rs_block = fec_buffer_rs_block(v, fio, n, j);  in fec_read_bufs()
322 static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)  in fec_alloc_bufs()  argument
326 	if (!fio->rs)  in fec_alloc_bufs()
327 		fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);  in fec_alloc_bufs()
330 		if (fio->bufs[n])  in fec_alloc_bufs()
333 		fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);  in fec_alloc_bufs()
334 		if (unlikely(!fio->bufs[n])) {  in fec_alloc_bufs()
341 	fec_for_each_extra_buffer(fio, n) {  in fec_alloc_bufs()
342 		if (fio->bufs[n])  in fec_alloc_bufs()
345 		fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);  in fec_alloc_bufs()
347 		if (unlikely(!fio->bufs[n]))  in fec_alloc_bufs()
350 	fio->nbufs = n;  in fec_alloc_bufs()
352 	if (!fio->output)  in fec_alloc_bufs()
353 		fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);  in fec_alloc_bufs()
362 static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)  in fec_init_bufs()  argument
366 	fec_for_each_buffer(fio, n)  in fec_init_bufs()
367 		memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);  in fec_init_bufs()
369 	memset(fio->erasures, 0, sizeof(fio->erasures));  in fec_init_bufs()
378 			  struct dm_verity_fec_io *fio, u64 rsb, u64 offset,  in fec_decode_rsb()  argument
384 	r = fec_alloc_bufs(v, fio);  in fec_decode_rsb()
389 		fec_init_bufs(v, fio);  in fec_decode_rsb()
396 		r = fec_decode_bufs(v, fio, rsb, r, pos, neras);  in fec_decode_rsb()
400 		pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS;  in fec_decode_rsb()
404 	r = verity_hash(v, verity_io_hash_req(v, io), fio->output,  in fec_decode_rsb()
423 	struct dm_verity_fec_io *fio = fec_io(io);  in fec_bv_copy()  local
425 	memcpy(data, &fio->output[fio->output_pos], len);  in fec_bv_copy()
426 	fio->output_pos += len;  in fec_bv_copy()
440 	struct dm_verity_fec_io *fio = fec_io(io);  in verity_fec_decode()  local
446 	if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) {  in verity_fec_decode()
451 	fio->level++;  in verity_fec_decode()
480 	r = fec_decode_rsb(v, io, fio, rsb, offset, false);  in verity_fec_decode()
482 		r = fec_decode_rsb(v, io, fio, rsb, offset, true);  in verity_fec_decode()
488 		memcpy(dest, fio->output, 1 << v->data_dev_block_bits);  in verity_fec_decode()
490 		fio->output_pos = 0;  in verity_fec_decode()
495 	fio->level--;  in verity_fec_decode()
506 	struct dm_verity_fec_io *fio = fec_io(io);  in verity_fec_finish_io()  local
511 	mempool_free(fio->rs, &f->rs_pool);  in verity_fec_finish_io()
514 		mempool_free(fio->bufs[n], &f->prealloc_pool);  in verity_fec_finish_io()
516 	fec_for_each_extra_buffer(fio, n)  in verity_fec_finish_io()
517 		mempool_free(fio->bufs[n], &f->extra_pool);  in verity_fec_finish_io()
519 	mempool_free(fio->output, &f->output_pool);  in verity_fec_finish_io()
527 	struct dm_verity_fec_io *fio = fec_io(io);  in verity_fec_init_io()  local
532 	fio->rs = NULL;  in verity_fec_init_io()
533 	memset(fio->bufs, 0, sizeof(fio->bufs));  in verity_fec_init_io()
534 	fio->nbufs = 0;  in verity_fec_init_io()
535 	fio->output = NULL;  in verity_fec_init_io()
536 	fio->level = 0;  in verity_fec_init_io()