12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
2a739ff3fSSami Tolvanen /*
3a739ff3fSSami Tolvanen * Copyright (C) 2015 Google, Inc.
4a739ff3fSSami Tolvanen *
5a739ff3fSSami Tolvanen * Author: Sami Tolvanen <samitolvanen@google.com>
6a739ff3fSSami Tolvanen */
7a739ff3fSSami Tolvanen
8a739ff3fSSami Tolvanen #include "dm-verity-fec.h"
9a739ff3fSSami Tolvanen #include <linux/math64.h>
10a739ff3fSSami Tolvanen
11a739ff3fSSami Tolvanen #define DM_MSG_PREFIX "verity-fec"
12a739ff3fSSami Tolvanen
13a739ff3fSSami Tolvanen /*
14a739ff3fSSami Tolvanen * If error correction has been configured, returns true.
15a739ff3fSSami Tolvanen */
verity_fec_is_enabled(struct dm_verity * v)16a739ff3fSSami Tolvanen bool verity_fec_is_enabled(struct dm_verity *v)
17a739ff3fSSami Tolvanen {
18a739ff3fSSami Tolvanen return v->fec && v->fec->dev;
19a739ff3fSSami Tolvanen }
20a739ff3fSSami Tolvanen
21a739ff3fSSami Tolvanen /*
22a739ff3fSSami Tolvanen * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable
23a739ff3fSSami Tolvanen * length fields.
24a739ff3fSSami Tolvanen */
fec_io(struct dm_verity_io * io)25a739ff3fSSami Tolvanen static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io)
26a739ff3fSSami Tolvanen {
27*99277dd2SMikulas Patocka return (struct dm_verity_fec_io *)
28*99277dd2SMikulas Patocka ((char *)io + io->v->ti->per_io_data_size - sizeof(struct dm_verity_fec_io));
29a739ff3fSSami Tolvanen }
30a739ff3fSSami Tolvanen
31a739ff3fSSami Tolvanen /*
32a739ff3fSSami Tolvanen * Return an interleaved offset for a byte in RS block.
33a739ff3fSSami Tolvanen */
fec_interleave(struct dm_verity * v,u64 offset)34a739ff3fSSami Tolvanen static inline u64 fec_interleave(struct dm_verity *v, u64 offset)
35a739ff3fSSami Tolvanen {
36a739ff3fSSami Tolvanen u32 mod;
37a739ff3fSSami Tolvanen
38a739ff3fSSami Tolvanen mod = do_div(offset, v->fec->rsn);
39a739ff3fSSami Tolvanen return offset + mod * (v->fec->rounds << v->data_dev_block_bits);
40a739ff3fSSami Tolvanen }
41a739ff3fSSami Tolvanen
42a739ff3fSSami Tolvanen /*
43a739ff3fSSami Tolvanen * Decode an RS block using Reed-Solomon.
44a739ff3fSSami Tolvanen */
fec_decode_rs8(struct dm_verity * v,struct dm_verity_fec_io * fio,u8 * data,u8 * fec,int neras)45a739ff3fSSami Tolvanen static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio,
46a739ff3fSSami Tolvanen u8 *data, u8 *fec, int neras)
47a739ff3fSSami Tolvanen {
48a739ff3fSSami Tolvanen int i;
49a739ff3fSSami Tolvanen uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN];
50a739ff3fSSami Tolvanen
51a739ff3fSSami Tolvanen for (i = 0; i < v->fec->roots; i++)
52a739ff3fSSami Tolvanen par[i] = fec[i];
53a739ff3fSSami Tolvanen
54a739ff3fSSami Tolvanen return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras,
55a739ff3fSSami Tolvanen fio->erasures, 0, NULL);
56a739ff3fSSami Tolvanen }
57a739ff3fSSami Tolvanen
58a739ff3fSSami Tolvanen /*
59a739ff3fSSami Tolvanen * Read error-correcting codes for the requested RS block. Returns a pointer
60a739ff3fSSami Tolvanen * to the data block. Caller is responsible for releasing buf.
61a739ff3fSSami Tolvanen */
fec_read_parity(struct dm_verity * v,u64 rsb,int index,unsigned int * offset,struct dm_buffer ** buf)62a739ff3fSSami Tolvanen static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index,
6386a3238cSHeinz Mauelshagen unsigned int *offset, struct dm_buffer **buf)
64a739ff3fSSami Tolvanen {
65df7b59baSMilan Broz u64 position, block, rem;
66a739ff3fSSami Tolvanen u8 *res;
67a739ff3fSSami Tolvanen
68a739ff3fSSami Tolvanen position = (index + rsb) * v->fec->roots;
698ca7cab8SJaegeuk Kim block = div64_u64_rem(position, v->fec->io_size, &rem);
7086a3238cSHeinz Mauelshagen *offset = (unsigned int)rem;
71a739ff3fSSami Tolvanen
72df7b59baSMilan Broz res = dm_bufio_read(v->fec->bufio, block, buf);
73821b40daSChengguang Xu if (IS_ERR(res)) {
74a739ff3fSSami Tolvanen DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",
75a739ff3fSSami Tolvanen v->data_dev->name, (unsigned long long)rsb,
76df7b59baSMilan Broz (unsigned long long)block, PTR_ERR(res));
77a739ff3fSSami Tolvanen *buf = NULL;
78a739ff3fSSami Tolvanen }
79a739ff3fSSami Tolvanen
80a739ff3fSSami Tolvanen return res;
81a739ff3fSSami Tolvanen }
82a739ff3fSSami Tolvanen
83a739ff3fSSami Tolvanen /* Loop over each preallocated buffer slot. */
84a739ff3fSSami Tolvanen #define fec_for_each_prealloc_buffer(__i) \
85a739ff3fSSami Tolvanen for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++)
86a739ff3fSSami Tolvanen
87a739ff3fSSami Tolvanen /* Loop over each extra buffer slot. */
88a739ff3fSSami Tolvanen #define fec_for_each_extra_buffer(io, __i) \
89a739ff3fSSami Tolvanen for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++)
90a739ff3fSSami Tolvanen
91a739ff3fSSami Tolvanen /* Loop over each allocated buffer. */
92a739ff3fSSami Tolvanen #define fec_for_each_buffer(io, __i) \
93a739ff3fSSami Tolvanen for (__i = 0; __i < (io)->nbufs; __i++)
94a739ff3fSSami Tolvanen
95a739ff3fSSami Tolvanen /* Loop over each RS block in each allocated buffer. */
96a739ff3fSSami Tolvanen #define fec_for_each_buffer_rs_block(io, __i, __j) \
97a739ff3fSSami Tolvanen fec_for_each_buffer(io, __i) \
98a739ff3fSSami Tolvanen for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++)
99a739ff3fSSami Tolvanen
100a739ff3fSSami Tolvanen /*
101a739ff3fSSami Tolvanen * Return a pointer to the current RS block when called inside
102a739ff3fSSami Tolvanen * fec_for_each_buffer_rs_block.
103a739ff3fSSami Tolvanen */
fec_buffer_rs_block(struct dm_verity * v,struct dm_verity_fec_io * fio,unsigned int i,unsigned int j)104a739ff3fSSami Tolvanen static inline u8 *fec_buffer_rs_block(struct dm_verity *v,
105a739ff3fSSami Tolvanen struct dm_verity_fec_io *fio,
10686a3238cSHeinz Mauelshagen unsigned int i, unsigned int j)
107a739ff3fSSami Tolvanen {
108a739ff3fSSami Tolvanen return &fio->bufs[i][j * v->fec->rsn];
109a739ff3fSSami Tolvanen }
110a739ff3fSSami Tolvanen
111a739ff3fSSami Tolvanen /*
112a739ff3fSSami Tolvanen * Return an index to the current RS block when called inside
113a739ff3fSSami Tolvanen * fec_for_each_buffer_rs_block.
114a739ff3fSSami Tolvanen */
fec_buffer_rs_index(unsigned int i,unsigned int j)11586a3238cSHeinz Mauelshagen static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j)
116a739ff3fSSami Tolvanen {
117a739ff3fSSami Tolvanen return (i << DM_VERITY_FEC_BUF_RS_BITS) + j;
118a739ff3fSSami Tolvanen }
119a739ff3fSSami Tolvanen
120a739ff3fSSami Tolvanen /*
121a739ff3fSSami Tolvanen * Decode all RS blocks from buffers and copy corrected bytes into fio->output
122a739ff3fSSami Tolvanen * starting from block_offset.
123a739ff3fSSami Tolvanen */
fec_decode_bufs(struct dm_verity * v,struct dm_verity_fec_io * fio,u64 rsb,int byte_index,unsigned int block_offset,int neras)124a739ff3fSSami Tolvanen static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio,
12586a3238cSHeinz Mauelshagen u64 rsb, int byte_index, unsigned int block_offset,
126a739ff3fSSami Tolvanen int neras)
127a739ff3fSSami Tolvanen {
128a739ff3fSSami Tolvanen int r, corrected = 0, res;
129a739ff3fSSami Tolvanen struct dm_buffer *buf;
13086a3238cSHeinz Mauelshagen unsigned int n, i, offset;
131a739ff3fSSami Tolvanen u8 *par, *block;
132a739ff3fSSami Tolvanen
133a739ff3fSSami Tolvanen par = fec_read_parity(v, rsb, block_offset, &offset, &buf);
134a739ff3fSSami Tolvanen if (IS_ERR(par))
135a739ff3fSSami Tolvanen return PTR_ERR(par);
136a739ff3fSSami Tolvanen
137a739ff3fSSami Tolvanen /*
138a739ff3fSSami Tolvanen * Decode the RS blocks we have in bufs. Each RS block results in
139a739ff3fSSami Tolvanen * one corrected target byte and consumes fec->roots parity bytes.
140a739ff3fSSami Tolvanen */
141a739ff3fSSami Tolvanen fec_for_each_buffer_rs_block(fio, n, i) {
142a739ff3fSSami Tolvanen block = fec_buffer_rs_block(v, fio, n, i);
143a739ff3fSSami Tolvanen res = fec_decode_rs8(v, fio, block, &par[offset], neras);
144a739ff3fSSami Tolvanen if (res < 0) {
145a739ff3fSSami Tolvanen r = res;
146a739ff3fSSami Tolvanen goto error;
147a739ff3fSSami Tolvanen }
148a739ff3fSSami Tolvanen
149a739ff3fSSami Tolvanen corrected += res;
150a739ff3fSSami Tolvanen fio->output[block_offset] = block[byte_index];
151a739ff3fSSami Tolvanen
152a739ff3fSSami Tolvanen block_offset++;
153a739ff3fSSami Tolvanen if (block_offset >= 1 << v->data_dev_block_bits)
154a739ff3fSSami Tolvanen goto done;
155a739ff3fSSami Tolvanen
156a739ff3fSSami Tolvanen /* read the next block when we run out of parity bytes */
157a739ff3fSSami Tolvanen offset += v->fec->roots;
1588ca7cab8SJaegeuk Kim if (offset >= v->fec->io_size) {
159a739ff3fSSami Tolvanen dm_bufio_release(buf);
160a739ff3fSSami Tolvanen
161a739ff3fSSami Tolvanen par = fec_read_parity(v, rsb, block_offset, &offset, &buf);
162821b40daSChengguang Xu if (IS_ERR(par))
163a739ff3fSSami Tolvanen return PTR_ERR(par);
164a739ff3fSSami Tolvanen }
165a739ff3fSSami Tolvanen }
166a739ff3fSSami Tolvanen done:
167a739ff3fSSami Tolvanen r = corrected;
168a739ff3fSSami Tolvanen error:
16986e3e83bSSami Tolvanen dm_bufio_release(buf);
17086e3e83bSSami Tolvanen
171a739ff3fSSami Tolvanen if (r < 0 && neras)
172a739ff3fSSami Tolvanen DMERR_LIMIT("%s: FEC %llu: failed to correct: %d",
173a739ff3fSSami Tolvanen v->data_dev->name, (unsigned long long)rsb, r);
174a739ff3fSSami Tolvanen else if (r > 0)
175a739ff3fSSami Tolvanen DMWARN_LIMIT("%s: FEC %llu: corrected %d errors",
176a739ff3fSSami Tolvanen v->data_dev->name, (unsigned long long)rsb, r);
177a739ff3fSSami Tolvanen
178a739ff3fSSami Tolvanen return r;
179a739ff3fSSami Tolvanen }
180a739ff3fSSami Tolvanen
181a739ff3fSSami Tolvanen /*
182a739ff3fSSami Tolvanen * Locate data block erasures using verity hashes.
183a739ff3fSSami Tolvanen */
fec_is_erasure(struct dm_verity * v,struct dm_verity_io * io,u8 * want_digest,u8 * data)184a739ff3fSSami Tolvanen static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io,
185a739ff3fSSami Tolvanen u8 *want_digest, u8 *data)
186a739ff3fSSami Tolvanen {
187d1ac3ff0SGilad Ben-Yossef if (unlikely(verity_hash(v, verity_io_hash_req(v, io),
188a739ff3fSSami Tolvanen data, 1 << v->data_dev_block_bits,
18957985252SMikulas Patocka verity_io_real_digest(v, io), true)))
190a739ff3fSSami Tolvanen return 0;
191a739ff3fSSami Tolvanen
192a739ff3fSSami Tolvanen return memcmp(verity_io_real_digest(v, io), want_digest,
193a739ff3fSSami Tolvanen v->digest_size) != 0;
194a739ff3fSSami Tolvanen }
195a739ff3fSSami Tolvanen
196a739ff3fSSami Tolvanen /*
197a739ff3fSSami Tolvanen * Read data blocks that are part of the RS block and deinterleave as much as
198a739ff3fSSami Tolvanen * fits into buffers. Check for erasure locations if @neras is non-NULL.
199a739ff3fSSami Tolvanen */
fec_read_bufs(struct dm_verity * v,struct dm_verity_io * io,u64 rsb,u64 target,unsigned int block_offset,int * neras)200a739ff3fSSami Tolvanen static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
20186a3238cSHeinz Mauelshagen u64 rsb, u64 target, unsigned int block_offset,
202a739ff3fSSami Tolvanen int *neras)
203a739ff3fSSami Tolvanen {
2040cc37c2dSSami Tolvanen bool is_zero;
205a739ff3fSSami Tolvanen int i, j, target_index = -1;
206a739ff3fSSami Tolvanen struct dm_buffer *buf;
207a739ff3fSSami Tolvanen struct dm_bufio_client *bufio;
208a739ff3fSSami Tolvanen struct dm_verity_fec_io *fio = fec_io(io);
209a739ff3fSSami Tolvanen u64 block, ileaved;
210a739ff3fSSami Tolvanen u8 *bbuf, *rs_block;
2116d39a124SKees Cook u8 want_digest[HASH_MAX_DIGESTSIZE];
21286a3238cSHeinz Mauelshagen unsigned int n, k;
213a739ff3fSSami Tolvanen
214a739ff3fSSami Tolvanen if (neras)
215a739ff3fSSami Tolvanen *neras = 0;
216a739ff3fSSami Tolvanen
2176d39a124SKees Cook if (WARN_ON(v->digest_size > sizeof(want_digest)))
2186d39a124SKees Cook return -EINVAL;
2196d39a124SKees Cook
220a739ff3fSSami Tolvanen /*
221a739ff3fSSami Tolvanen * read each of the rsn data blocks that are part of the RS block, and
222a739ff3fSSami Tolvanen * interleave contents to available bufs
223a739ff3fSSami Tolvanen */
224a739ff3fSSami Tolvanen for (i = 0; i < v->fec->rsn; i++) {
225a739ff3fSSami Tolvanen ileaved = fec_interleave(v, rsb * v->fec->rsn + i);
226a739ff3fSSami Tolvanen
227a739ff3fSSami Tolvanen /*
228a739ff3fSSami Tolvanen * target is the data block we want to correct, target_index is
229a739ff3fSSami Tolvanen * the index of this block within the rsn RS blocks
230a739ff3fSSami Tolvanen */
231a739ff3fSSami Tolvanen if (ileaved == target)
232a739ff3fSSami Tolvanen target_index = i;
233a739ff3fSSami Tolvanen
234a739ff3fSSami Tolvanen block = ileaved >> v->data_dev_block_bits;
235a739ff3fSSami Tolvanen bufio = v->fec->data_bufio;
236a739ff3fSSami Tolvanen
237a739ff3fSSami Tolvanen if (block >= v->data_blocks) {
238a739ff3fSSami Tolvanen block -= v->data_blocks;
239a739ff3fSSami Tolvanen
240a739ff3fSSami Tolvanen /*
241a739ff3fSSami Tolvanen * blocks outside the area were assumed to contain
242a739ff3fSSami Tolvanen * zeros when encoding data was generated
243a739ff3fSSami Tolvanen */
244a739ff3fSSami Tolvanen if (unlikely(block >= v->fec->hash_blocks))
245a739ff3fSSami Tolvanen continue;
246a739ff3fSSami Tolvanen
247a739ff3fSSami Tolvanen block += v->hash_start;
248a739ff3fSSami Tolvanen bufio = v->bufio;
249a739ff3fSSami Tolvanen }
250a739ff3fSSami Tolvanen
251a739ff3fSSami Tolvanen bbuf = dm_bufio_read(bufio, block, &buf);
252821b40daSChengguang Xu if (IS_ERR(bbuf)) {
253a739ff3fSSami Tolvanen DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld",
254a739ff3fSSami Tolvanen v->data_dev->name,
255a739ff3fSSami Tolvanen (unsigned long long)rsb,
256a739ff3fSSami Tolvanen (unsigned long long)block, PTR_ERR(bbuf));
257a739ff3fSSami Tolvanen
258a739ff3fSSami Tolvanen /* assume the block is corrupted */
259a739ff3fSSami Tolvanen if (neras && *neras <= v->fec->roots)
260a739ff3fSSami Tolvanen fio->erasures[(*neras)++] = i;
261a739ff3fSSami Tolvanen
262a739ff3fSSami Tolvanen continue;
263a739ff3fSSami Tolvanen }
264a739ff3fSSami Tolvanen
265a739ff3fSSami Tolvanen /* locate erasures if the block is on the data device */
266a739ff3fSSami Tolvanen if (bufio == v->fec->data_bufio &&
2670cc37c2dSSami Tolvanen verity_hash_for_block(v, io, block, want_digest,
2680cc37c2dSSami Tolvanen &is_zero) == 0) {
2690cc37c2dSSami Tolvanen /* skip known zero blocks entirely */
2700cc37c2dSSami Tolvanen if (is_zero)
27186e3e83bSSami Tolvanen goto done;
2720cc37c2dSSami Tolvanen
273a739ff3fSSami Tolvanen /*
274a739ff3fSSami Tolvanen * skip if we have already found the theoretical
275a739ff3fSSami Tolvanen * maximum number (i.e. fec->roots) of erasures
276a739ff3fSSami Tolvanen */
277a739ff3fSSami Tolvanen if (neras && *neras <= v->fec->roots &&
278a739ff3fSSami Tolvanen fec_is_erasure(v, io, want_digest, bbuf))
279a739ff3fSSami Tolvanen fio->erasures[(*neras)++] = i;
280a739ff3fSSami Tolvanen }
281a739ff3fSSami Tolvanen
282a739ff3fSSami Tolvanen /*
283a739ff3fSSami Tolvanen * deinterleave and copy the bytes that fit into bufs,
284a739ff3fSSami Tolvanen * starting from block_offset
285a739ff3fSSami Tolvanen */
286a739ff3fSSami Tolvanen fec_for_each_buffer_rs_block(fio, n, j) {
287a739ff3fSSami Tolvanen k = fec_buffer_rs_index(n, j) + block_offset;
288a739ff3fSSami Tolvanen
289a739ff3fSSami Tolvanen if (k >= 1 << v->data_dev_block_bits)
290a739ff3fSSami Tolvanen goto done;
291a739ff3fSSami Tolvanen
292a739ff3fSSami Tolvanen rs_block = fec_buffer_rs_block(v, fio, n, j);
293a739ff3fSSami Tolvanen rs_block[i] = bbuf[k];
294a739ff3fSSami Tolvanen }
295a739ff3fSSami Tolvanen done:
296a739ff3fSSami Tolvanen dm_bufio_release(buf);
297a739ff3fSSami Tolvanen }
298a739ff3fSSami Tolvanen
299a739ff3fSSami Tolvanen return target_index;
300a739ff3fSSami Tolvanen }
301a739ff3fSSami Tolvanen
302a739ff3fSSami Tolvanen /*
303a739ff3fSSami Tolvanen * Allocate RS control structure and FEC buffers from preallocated mempools,
304a739ff3fSSami Tolvanen * and attempt to allocate as many extra buffers as available.
305a739ff3fSSami Tolvanen */
fec_alloc_bufs(struct dm_verity * v,struct dm_verity_fec_io * fio)306a739ff3fSSami Tolvanen static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
307a739ff3fSSami Tolvanen {
30886a3238cSHeinz Mauelshagen unsigned int n;
309a739ff3fSSami Tolvanen
31034c96507SNeilBrown if (!fio->rs)
3116f1c819cSKent Overstreet fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);
312a739ff3fSSami Tolvanen
313a739ff3fSSami Tolvanen fec_for_each_prealloc_buffer(n) {
314a739ff3fSSami Tolvanen if (fio->bufs[n])
315a739ff3fSSami Tolvanen continue;
316a739ff3fSSami Tolvanen
3176f1c819cSKent Overstreet fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);
318a739ff3fSSami Tolvanen if (unlikely(!fio->bufs[n])) {
319a739ff3fSSami Tolvanen DMERR("failed to allocate FEC buffer");
320a739ff3fSSami Tolvanen return -ENOMEM;
321a739ff3fSSami Tolvanen }
322a739ff3fSSami Tolvanen }
323a739ff3fSSami Tolvanen
324a739ff3fSSami Tolvanen /* try to allocate the maximum number of buffers */
325a739ff3fSSami Tolvanen fec_for_each_extra_buffer(fio, n) {
326a739ff3fSSami Tolvanen if (fio->bufs[n])
327a739ff3fSSami Tolvanen continue;
328a739ff3fSSami Tolvanen
3296f1c819cSKent Overstreet fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);
330a739ff3fSSami Tolvanen /* we can manage with even one buffer if necessary */
331a739ff3fSSami Tolvanen if (unlikely(!fio->bufs[n]))
332a739ff3fSSami Tolvanen break;
333a739ff3fSSami Tolvanen }
334a739ff3fSSami Tolvanen fio->nbufs = n;
335a739ff3fSSami Tolvanen
33634c96507SNeilBrown if (!fio->output)
3376f1c819cSKent Overstreet fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);
338a739ff3fSSami Tolvanen
339a739ff3fSSami Tolvanen return 0;
340a739ff3fSSami Tolvanen }
341a739ff3fSSami Tolvanen
342a739ff3fSSami Tolvanen /*
343a739ff3fSSami Tolvanen * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are
344a739ff3fSSami Tolvanen * zeroed before deinterleaving.
345a739ff3fSSami Tolvanen */
fec_init_bufs(struct dm_verity * v,struct dm_verity_fec_io * fio)346a739ff3fSSami Tolvanen static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
347a739ff3fSSami Tolvanen {
34886a3238cSHeinz Mauelshagen unsigned int n;
349a739ff3fSSami Tolvanen
350a739ff3fSSami Tolvanen fec_for_each_buffer(fio, n)
351a739ff3fSSami Tolvanen memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);
352a739ff3fSSami Tolvanen
353a739ff3fSSami Tolvanen memset(fio->erasures, 0, sizeof(fio->erasures));
354a739ff3fSSami Tolvanen }
355a739ff3fSSami Tolvanen
356a739ff3fSSami Tolvanen /*
357a739ff3fSSami Tolvanen * Decode all RS blocks in a single data block and return the target block
358a739ff3fSSami Tolvanen * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses
359a739ff3fSSami Tolvanen * hashes to locate erasures.
360a739ff3fSSami Tolvanen */
fec_decode_rsb(struct dm_verity * v,struct dm_verity_io * io,struct dm_verity_fec_io * fio,u64 rsb,u64 offset,bool use_erasures)361a739ff3fSSami Tolvanen static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io,
362a739ff3fSSami Tolvanen struct dm_verity_fec_io *fio, u64 rsb, u64 offset,
363a739ff3fSSami Tolvanen bool use_erasures)
364a739ff3fSSami Tolvanen {
365a739ff3fSSami Tolvanen int r, neras = 0;
36686a3238cSHeinz Mauelshagen unsigned int pos;
367a739ff3fSSami Tolvanen
368a739ff3fSSami Tolvanen r = fec_alloc_bufs(v, fio);
369a739ff3fSSami Tolvanen if (unlikely(r < 0))
370a739ff3fSSami Tolvanen return r;
371a739ff3fSSami Tolvanen
372a739ff3fSSami Tolvanen for (pos = 0; pos < 1 << v->data_dev_block_bits; ) {
373a739ff3fSSami Tolvanen fec_init_bufs(v, fio);
374a739ff3fSSami Tolvanen
375a739ff3fSSami Tolvanen r = fec_read_bufs(v, io, rsb, offset, pos,
376a739ff3fSSami Tolvanen use_erasures ? &neras : NULL);
377a739ff3fSSami Tolvanen if (unlikely(r < 0))
378a739ff3fSSami Tolvanen return r;
379a739ff3fSSami Tolvanen
380a739ff3fSSami Tolvanen r = fec_decode_bufs(v, fio, rsb, r, pos, neras);
381a739ff3fSSami Tolvanen if (r < 0)
382a739ff3fSSami Tolvanen return r;
383a739ff3fSSami Tolvanen
384a739ff3fSSami Tolvanen pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS;
385a739ff3fSSami Tolvanen }
386a739ff3fSSami Tolvanen
387a739ff3fSSami Tolvanen /* Always re-validate the corrected block against the expected hash */
388d1ac3ff0SGilad Ben-Yossef r = verity_hash(v, verity_io_hash_req(v, io), fio->output,
389a739ff3fSSami Tolvanen 1 << v->data_dev_block_bits,
39057985252SMikulas Patocka verity_io_real_digest(v, io), true);
391a739ff3fSSami Tolvanen if (unlikely(r < 0))
392a739ff3fSSami Tolvanen return r;
393a739ff3fSSami Tolvanen
394a739ff3fSSami Tolvanen if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io),
395a739ff3fSSami Tolvanen v->digest_size)) {
396a739ff3fSSami Tolvanen DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)",
397a739ff3fSSami Tolvanen v->data_dev->name, (unsigned long long)rsb, neras);
398a739ff3fSSami Tolvanen return -EILSEQ;
399a739ff3fSSami Tolvanen }
400a739ff3fSSami Tolvanen
401a739ff3fSSami Tolvanen return 0;
402a739ff3fSSami Tolvanen }
403a739ff3fSSami Tolvanen
fec_bv_copy(struct dm_verity * v,struct dm_verity_io * io,u8 * data,size_t len)404a739ff3fSSami Tolvanen static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data,
405a739ff3fSSami Tolvanen size_t len)
406a739ff3fSSami Tolvanen {
407a739ff3fSSami Tolvanen struct dm_verity_fec_io *fio = fec_io(io);
408a739ff3fSSami Tolvanen
409a739ff3fSSami Tolvanen memcpy(data, &fio->output[fio->output_pos], len);
410a739ff3fSSami Tolvanen fio->output_pos += len;
411a739ff3fSSami Tolvanen
412a739ff3fSSami Tolvanen return 0;
413a739ff3fSSami Tolvanen }
414a739ff3fSSami Tolvanen
415a739ff3fSSami Tolvanen /*
416a739ff3fSSami Tolvanen * Correct errors in a block. Copies corrected block to dest if non-NULL,
417a739ff3fSSami Tolvanen * otherwise to a bio_vec starting from iter.
418a739ff3fSSami Tolvanen */
verity_fec_decode(struct dm_verity * v,struct dm_verity_io * io,enum verity_block_type type,sector_t block,u8 * dest,struct bvec_iter * iter)419a739ff3fSSami Tolvanen int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io,
420a739ff3fSSami Tolvanen enum verity_block_type type, sector_t block, u8 *dest,
421a739ff3fSSami Tolvanen struct bvec_iter *iter)
422a739ff3fSSami Tolvanen {
423a739ff3fSSami Tolvanen int r;
424a739ff3fSSami Tolvanen struct dm_verity_fec_io *fio = fec_io(io);
425a739ff3fSSami Tolvanen u64 offset, res, rsb;
426a739ff3fSSami Tolvanen
427a739ff3fSSami Tolvanen if (!verity_fec_is_enabled(v))
428a739ff3fSSami Tolvanen return -EOPNOTSUPP;
429a739ff3fSSami Tolvanen
430f1a880a9SSami Tolvanen if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) {
431f1a880a9SSami Tolvanen DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name);
432f1a880a9SSami Tolvanen return -EIO;
433f1a880a9SSami Tolvanen }
434f1a880a9SSami Tolvanen
435f1a880a9SSami Tolvanen fio->level++;
436f1a880a9SSami Tolvanen
437a739ff3fSSami Tolvanen if (type == DM_VERITY_BLOCK_TYPE_METADATA)
438ad4e80a6SSunwook Eom block = block - v->hash_start + v->data_blocks;
439a739ff3fSSami Tolvanen
440a739ff3fSSami Tolvanen /*
441a739ff3fSSami Tolvanen * For RS(M, N), the continuous FEC data is divided into blocks of N
442a739ff3fSSami Tolvanen * bytes. Since block size may not be divisible by N, the last block
443a739ff3fSSami Tolvanen * is zero padded when decoding.
444a739ff3fSSami Tolvanen *
445a739ff3fSSami Tolvanen * Each byte of the block is covered by a different RS(M, N) code,
446a739ff3fSSami Tolvanen * and each code is interleaved over N blocks to make it less likely
447a739ff3fSSami Tolvanen * that bursty corruption will leave us in unrecoverable state.
448a739ff3fSSami Tolvanen */
449a739ff3fSSami Tolvanen
450a739ff3fSSami Tolvanen offset = block << v->data_dev_block_bits;
451602d1657SSami Tolvanen res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits);
452a739ff3fSSami Tolvanen
453a739ff3fSSami Tolvanen /*
454a739ff3fSSami Tolvanen * The base RS block we can feed to the interleaver to find out all
455a739ff3fSSami Tolvanen * blocks required for decoding.
456a739ff3fSSami Tolvanen */
457a739ff3fSSami Tolvanen rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits);
458a739ff3fSSami Tolvanen
459a739ff3fSSami Tolvanen /*
460a739ff3fSSami Tolvanen * Locating erasures is slow, so attempt to recover the block without
461a739ff3fSSami Tolvanen * them first. Do a second attempt with erasures if the corruption is
462a739ff3fSSami Tolvanen * bad enough.
463a739ff3fSSami Tolvanen */
464a739ff3fSSami Tolvanen r = fec_decode_rsb(v, io, fio, rsb, offset, false);
465a739ff3fSSami Tolvanen if (r < 0) {
466a739ff3fSSami Tolvanen r = fec_decode_rsb(v, io, fio, rsb, offset, true);
467a739ff3fSSami Tolvanen if (r < 0)
468f1a880a9SSami Tolvanen goto done;
469a739ff3fSSami Tolvanen }
470a739ff3fSSami Tolvanen
471a739ff3fSSami Tolvanen if (dest)
472a739ff3fSSami Tolvanen memcpy(dest, fio->output, 1 << v->data_dev_block_bits);
473a739ff3fSSami Tolvanen else if (iter) {
474a739ff3fSSami Tolvanen fio->output_pos = 0;
475a739ff3fSSami Tolvanen r = verity_for_bv_block(v, io, iter, fec_bv_copy);
476a739ff3fSSami Tolvanen }
477a739ff3fSSami Tolvanen
478f1a880a9SSami Tolvanen done:
479f1a880a9SSami Tolvanen fio->level--;
480a739ff3fSSami Tolvanen return r;
481a739ff3fSSami Tolvanen }
482a739ff3fSSami Tolvanen
483a739ff3fSSami Tolvanen /*
484a739ff3fSSami Tolvanen * Clean up per-bio data.
485a739ff3fSSami Tolvanen */
verity_fec_finish_io(struct dm_verity_io * io)486a739ff3fSSami Tolvanen void verity_fec_finish_io(struct dm_verity_io *io)
487a739ff3fSSami Tolvanen {
48886a3238cSHeinz Mauelshagen unsigned int n;
489a739ff3fSSami Tolvanen struct dm_verity_fec *f = io->v->fec;
490a739ff3fSSami Tolvanen struct dm_verity_fec_io *fio = fec_io(io);
491a739ff3fSSami Tolvanen
492a739ff3fSSami Tolvanen if (!verity_fec_is_enabled(io->v))
493a739ff3fSSami Tolvanen return;
494a739ff3fSSami Tolvanen
4956f1c819cSKent Overstreet mempool_free(fio->rs, &f->rs_pool);
496a739ff3fSSami Tolvanen
497a739ff3fSSami Tolvanen fec_for_each_prealloc_buffer(n)
4986f1c819cSKent Overstreet mempool_free(fio->bufs[n], &f->prealloc_pool);
499a739ff3fSSami Tolvanen
500a739ff3fSSami Tolvanen fec_for_each_extra_buffer(fio, n)
5016f1c819cSKent Overstreet mempool_free(fio->bufs[n], &f->extra_pool);
502a739ff3fSSami Tolvanen
5036f1c819cSKent Overstreet mempool_free(fio->output, &f->output_pool);
504a739ff3fSSami Tolvanen }
505a739ff3fSSami Tolvanen
506a739ff3fSSami Tolvanen /*
507a739ff3fSSami Tolvanen * Initialize per-bio data.
508a739ff3fSSami Tolvanen */
verity_fec_init_io(struct dm_verity_io * io)509a739ff3fSSami Tolvanen void verity_fec_init_io(struct dm_verity_io *io)
510a739ff3fSSami Tolvanen {
511a739ff3fSSami Tolvanen struct dm_verity_fec_io *fio = fec_io(io);
512a739ff3fSSami Tolvanen
513a739ff3fSSami Tolvanen if (!verity_fec_is_enabled(io->v))
514a739ff3fSSami Tolvanen return;
515a739ff3fSSami Tolvanen
516a739ff3fSSami Tolvanen fio->rs = NULL;
517a739ff3fSSami Tolvanen memset(fio->bufs, 0, sizeof(fio->bufs));
518a739ff3fSSami Tolvanen fio->nbufs = 0;
519a739ff3fSSami Tolvanen fio->output = NULL;
520f1a880a9SSami Tolvanen fio->level = 0;
521a739ff3fSSami Tolvanen }
522a739ff3fSSami Tolvanen
523a739ff3fSSami Tolvanen /*
524a739ff3fSSami Tolvanen * Append feature arguments and values to the status table.
525a739ff3fSSami Tolvanen */
verity_fec_status_table(struct dm_verity * v,unsigned int sz,char * result,unsigned int maxlen)52686a3238cSHeinz Mauelshagen unsigned int verity_fec_status_table(struct dm_verity *v, unsigned int sz,
52786a3238cSHeinz Mauelshagen char *result, unsigned int maxlen)
528a739ff3fSSami Tolvanen {
529a739ff3fSSami Tolvanen if (!verity_fec_is_enabled(v))
530a739ff3fSSami Tolvanen return sz;
531a739ff3fSSami Tolvanen
532a739ff3fSSami Tolvanen DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s "
533a739ff3fSSami Tolvanen DM_VERITY_OPT_FEC_BLOCKS " %llu "
534a739ff3fSSami Tolvanen DM_VERITY_OPT_FEC_START " %llu "
535a739ff3fSSami Tolvanen DM_VERITY_OPT_FEC_ROOTS " %d",
536a739ff3fSSami Tolvanen v->fec->dev->name,
537a739ff3fSSami Tolvanen (unsigned long long)v->fec->blocks,
538a739ff3fSSami Tolvanen (unsigned long long)v->fec->start,
539a739ff3fSSami Tolvanen v->fec->roots);
540a739ff3fSSami Tolvanen
541a739ff3fSSami Tolvanen return sz;
542a739ff3fSSami Tolvanen }
543a739ff3fSSami Tolvanen
verity_fec_dtr(struct dm_verity * v)544a739ff3fSSami Tolvanen void verity_fec_dtr(struct dm_verity *v)
545a739ff3fSSami Tolvanen {
546a739ff3fSSami Tolvanen struct dm_verity_fec *f = v->fec;
547a739ff3fSSami Tolvanen
548a739ff3fSSami Tolvanen if (!verity_fec_is_enabled(v))
549a739ff3fSSami Tolvanen goto out;
550a739ff3fSSami Tolvanen
5516f1c819cSKent Overstreet mempool_exit(&f->rs_pool);
5526f1c819cSKent Overstreet mempool_exit(&f->prealloc_pool);
5536f1c819cSKent Overstreet mempool_exit(&f->extra_pool);
55475fa6019SShetty, Harshini X (EXT-Sony Mobile) mempool_exit(&f->output_pool);
555a739ff3fSSami Tolvanen kmem_cache_destroy(f->cache);
556a739ff3fSSami Tolvanen
557a739ff3fSSami Tolvanen if (f->data_bufio)
558a739ff3fSSami Tolvanen dm_bufio_client_destroy(f->data_bufio);
559a739ff3fSSami Tolvanen if (f->bufio)
560a739ff3fSSami Tolvanen dm_bufio_client_destroy(f->bufio);
561a739ff3fSSami Tolvanen
562a739ff3fSSami Tolvanen if (f->dev)
563a739ff3fSSami Tolvanen dm_put_device(v->ti, f->dev);
564a739ff3fSSami Tolvanen out:
565a739ff3fSSami Tolvanen kfree(f);
566a739ff3fSSami Tolvanen v->fec = NULL;
567a739ff3fSSami Tolvanen }
568a739ff3fSSami Tolvanen
fec_rs_alloc(gfp_t gfp_mask,void * pool_data)569a739ff3fSSami Tolvanen static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data)
570a739ff3fSSami Tolvanen {
57126cb62a2SYu Zhe struct dm_verity *v = pool_data;
572a739ff3fSSami Tolvanen
573eb366989SThomas Gleixner return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask);
574a739ff3fSSami Tolvanen }
575a739ff3fSSami Tolvanen
fec_rs_free(void * element,void * pool_data)576a739ff3fSSami Tolvanen static void fec_rs_free(void *element, void *pool_data)
577a739ff3fSSami Tolvanen {
57826cb62a2SYu Zhe struct rs_control *rs = element;
579a739ff3fSSami Tolvanen
580a739ff3fSSami Tolvanen if (rs)
581a739ff3fSSami Tolvanen free_rs(rs);
582a739ff3fSSami Tolvanen }
583a739ff3fSSami Tolvanen
verity_is_fec_opt_arg(const char * arg_name)584a739ff3fSSami Tolvanen bool verity_is_fec_opt_arg(const char *arg_name)
585a739ff3fSSami Tolvanen {
586a739ff3fSSami Tolvanen return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) ||
587a739ff3fSSami Tolvanen !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) ||
588a739ff3fSSami Tolvanen !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) ||
589a739ff3fSSami Tolvanen !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS));
590a739ff3fSSami Tolvanen }
591a739ff3fSSami Tolvanen
verity_fec_parse_opt_args(struct dm_arg_set * as,struct dm_verity * v,unsigned int * argc,const char * arg_name)592a739ff3fSSami Tolvanen int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
59386a3238cSHeinz Mauelshagen unsigned int *argc, const char *arg_name)
594a739ff3fSSami Tolvanen {
595a739ff3fSSami Tolvanen int r;
596a739ff3fSSami Tolvanen struct dm_target *ti = v->ti;
597a739ff3fSSami Tolvanen const char *arg_value;
598a739ff3fSSami Tolvanen unsigned long long num_ll;
599a739ff3fSSami Tolvanen unsigned char num_c;
600a739ff3fSSami Tolvanen char dummy;
601a739ff3fSSami Tolvanen
602a739ff3fSSami Tolvanen if (!*argc) {
603a739ff3fSSami Tolvanen ti->error = "FEC feature arguments require a value";
604a739ff3fSSami Tolvanen return -EINVAL;
605a739ff3fSSami Tolvanen }
606a739ff3fSSami Tolvanen
607a739ff3fSSami Tolvanen arg_value = dm_shift_arg(as);
608a739ff3fSSami Tolvanen (*argc)--;
609a739ff3fSSami Tolvanen
610a739ff3fSSami Tolvanen if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) {
61105bdb996SChristoph Hellwig r = dm_get_device(ti, arg_value, BLK_OPEN_READ, &v->fec->dev);
612a739ff3fSSami Tolvanen if (r) {
613a739ff3fSSami Tolvanen ti->error = "FEC device lookup failed";
614a739ff3fSSami Tolvanen return r;
615a739ff3fSSami Tolvanen }
616a739ff3fSSami Tolvanen
617a739ff3fSSami Tolvanen } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) {
618a739ff3fSSami Tolvanen if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 ||
619a739ff3fSSami Tolvanen ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
620a739ff3fSSami Tolvanen >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) {
621a739ff3fSSami Tolvanen ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS;
622a739ff3fSSami Tolvanen return -EINVAL;
623a739ff3fSSami Tolvanen }
624a739ff3fSSami Tolvanen v->fec->blocks = num_ll;
625a739ff3fSSami Tolvanen
626a739ff3fSSami Tolvanen } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) {
627a739ff3fSSami Tolvanen if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 ||
628a739ff3fSSami Tolvanen ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >>
629a739ff3fSSami Tolvanen (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) {
630a739ff3fSSami Tolvanen ti->error = "Invalid " DM_VERITY_OPT_FEC_START;
631a739ff3fSSami Tolvanen return -EINVAL;
632a739ff3fSSami Tolvanen }
633a739ff3fSSami Tolvanen v->fec->start = num_ll;
634a739ff3fSSami Tolvanen
635a739ff3fSSami Tolvanen } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) {
636a739ff3fSSami Tolvanen if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c ||
637a739ff3fSSami Tolvanen num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) ||
638a739ff3fSSami Tolvanen num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) {
639a739ff3fSSami Tolvanen ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS;
640a739ff3fSSami Tolvanen return -EINVAL;
641a739ff3fSSami Tolvanen }
642a739ff3fSSami Tolvanen v->fec->roots = num_c;
643a739ff3fSSami Tolvanen
644a739ff3fSSami Tolvanen } else {
645a739ff3fSSami Tolvanen ti->error = "Unrecognized verity FEC feature request";
646a739ff3fSSami Tolvanen return -EINVAL;
647a739ff3fSSami Tolvanen }
648a739ff3fSSami Tolvanen
649a739ff3fSSami Tolvanen return 0;
650a739ff3fSSami Tolvanen }
651a739ff3fSSami Tolvanen
652a739ff3fSSami Tolvanen /*
653a739ff3fSSami Tolvanen * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
654a739ff3fSSami Tolvanen */
verity_fec_ctr_alloc(struct dm_verity * v)655a739ff3fSSami Tolvanen int verity_fec_ctr_alloc(struct dm_verity *v)
656a739ff3fSSami Tolvanen {
657a739ff3fSSami Tolvanen struct dm_verity_fec *f;
658a739ff3fSSami Tolvanen
659a739ff3fSSami Tolvanen f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL);
660a739ff3fSSami Tolvanen if (!f) {
661a739ff3fSSami Tolvanen v->ti->error = "Cannot allocate FEC structure";
662a739ff3fSSami Tolvanen return -ENOMEM;
663a739ff3fSSami Tolvanen }
664a739ff3fSSami Tolvanen v->fec = f;
665a739ff3fSSami Tolvanen
666a739ff3fSSami Tolvanen return 0;
667a739ff3fSSami Tolvanen }
668a739ff3fSSami Tolvanen
669a739ff3fSSami Tolvanen /*
670a739ff3fSSami Tolvanen * Validate arguments and preallocate memory. Must be called after arguments
671a739ff3fSSami Tolvanen * have been parsed using verity_fec_parse_opt_args.
672a739ff3fSSami Tolvanen */
verity_fec_ctr(struct dm_verity * v)673a739ff3fSSami Tolvanen int verity_fec_ctr(struct dm_verity *v)
674a739ff3fSSami Tolvanen {
675a739ff3fSSami Tolvanen struct dm_verity_fec *f = v->fec;
676a739ff3fSSami Tolvanen struct dm_target *ti = v->ti;
677df7b59baSMilan Broz u64 hash_blocks, fec_blocks;
6786f1c819cSKent Overstreet int ret;
679a739ff3fSSami Tolvanen
680a739ff3fSSami Tolvanen if (!verity_fec_is_enabled(v)) {
681a739ff3fSSami Tolvanen verity_fec_dtr(v);
682a739ff3fSSami Tolvanen return 0;
683a739ff3fSSami Tolvanen }
684a739ff3fSSami Tolvanen
685a739ff3fSSami Tolvanen /*
686a739ff3fSSami Tolvanen * FEC is computed over data blocks, possible metadata, and
687a739ff3fSSami Tolvanen * hash blocks. In other words, FEC covers total of fec_blocks
688a739ff3fSSami Tolvanen * blocks consisting of the following:
689a739ff3fSSami Tolvanen *
690a739ff3fSSami Tolvanen * data blocks | hash blocks | metadata (optional)
691a739ff3fSSami Tolvanen *
692a739ff3fSSami Tolvanen * We allow metadata after hash blocks to support a use case
693a739ff3fSSami Tolvanen * where all data is stored on the same device and FEC covers
694a739ff3fSSami Tolvanen * the entire area.
695a739ff3fSSami Tolvanen *
696a739ff3fSSami Tolvanen * If metadata is included, we require it to be available on the
697a739ff3fSSami Tolvanen * hash device after the hash blocks.
698a739ff3fSSami Tolvanen */
699a739ff3fSSami Tolvanen
700a739ff3fSSami Tolvanen hash_blocks = v->hash_blocks - v->hash_start;
701a739ff3fSSami Tolvanen
702a739ff3fSSami Tolvanen /*
703a739ff3fSSami Tolvanen * Require matching block sizes for data and hash devices for
704a739ff3fSSami Tolvanen * simplicity.
705a739ff3fSSami Tolvanen */
706a739ff3fSSami Tolvanen if (v->data_dev_block_bits != v->hash_dev_block_bits) {
707a739ff3fSSami Tolvanen ti->error = "Block sizes must match to use FEC";
708a739ff3fSSami Tolvanen return -EINVAL;
709a739ff3fSSami Tolvanen }
710a739ff3fSSami Tolvanen
711a739ff3fSSami Tolvanen if (!f->roots) {
712a739ff3fSSami Tolvanen ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS;
713a739ff3fSSami Tolvanen return -EINVAL;
714a739ff3fSSami Tolvanen }
715a739ff3fSSami Tolvanen f->rsn = DM_VERITY_FEC_RSM - f->roots;
716a739ff3fSSami Tolvanen
717a739ff3fSSami Tolvanen if (!f->blocks) {
718a739ff3fSSami Tolvanen ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS;
719a739ff3fSSami Tolvanen return -EINVAL;
720a739ff3fSSami Tolvanen }
721a739ff3fSSami Tolvanen
722a739ff3fSSami Tolvanen f->rounds = f->blocks;
723a739ff3fSSami Tolvanen if (sector_div(f->rounds, f->rsn))
724a739ff3fSSami Tolvanen f->rounds++;
725a739ff3fSSami Tolvanen
726a739ff3fSSami Tolvanen /*
727a739ff3fSSami Tolvanen * Due to optional metadata, f->blocks can be larger than
728a739ff3fSSami Tolvanen * data_blocks and hash_blocks combined.
729a739ff3fSSami Tolvanen */
730a739ff3fSSami Tolvanen if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) {
731a739ff3fSSami Tolvanen ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS;
732a739ff3fSSami Tolvanen return -EINVAL;
733a739ff3fSSami Tolvanen }
734a739ff3fSSami Tolvanen
735a739ff3fSSami Tolvanen /*
736a739ff3fSSami Tolvanen * Metadata is accessed through the hash device, so we require
737a739ff3fSSami Tolvanen * it to be large enough.
738a739ff3fSSami Tolvanen */
739a739ff3fSSami Tolvanen f->hash_blocks = f->blocks - v->data_blocks;
740a739ff3fSSami Tolvanen if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) {
741a739ff3fSSami Tolvanen ti->error = "Hash device is too small for "
742a739ff3fSSami Tolvanen DM_VERITY_OPT_FEC_BLOCKS;
743a739ff3fSSami Tolvanen return -E2BIG;
744a739ff3fSSami Tolvanen }
745a739ff3fSSami Tolvanen
7468ca7cab8SJaegeuk Kim if ((f->roots << SECTOR_SHIFT) & ((1 << v->data_dev_block_bits) - 1))
7478ca7cab8SJaegeuk Kim f->io_size = 1 << v->data_dev_block_bits;
7488ca7cab8SJaegeuk Kim else
7498ca7cab8SJaegeuk Kim f->io_size = v->fec->roots << SECTOR_SHIFT;
7508ca7cab8SJaegeuk Kim
751a739ff3fSSami Tolvanen f->bufio = dm_bufio_client_create(f->dev->bdev,
7528ca7cab8SJaegeuk Kim f->io_size,
7530fcb100dSNathan Huckleberry 1, 0, NULL, NULL, 0);
754a739ff3fSSami Tolvanen if (IS_ERR(f->bufio)) {
755a739ff3fSSami Tolvanen ti->error = "Cannot initialize FEC bufio client";
756a739ff3fSSami Tolvanen return PTR_ERR(f->bufio);
757a739ff3fSSami Tolvanen }
758a739ff3fSSami Tolvanen
759df7b59baSMilan Broz dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT));
760df7b59baSMilan Broz
761df7b59baSMilan Broz fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT);
762df7b59baSMilan Broz if (dm_bufio_get_device_size(f->bufio) < fec_blocks) {
763a739ff3fSSami Tolvanen ti->error = "FEC device is too small";
764a739ff3fSSami Tolvanen return -E2BIG;
765a739ff3fSSami Tolvanen }
766a739ff3fSSami Tolvanen
767a739ff3fSSami Tolvanen f->data_bufio = dm_bufio_client_create(v->data_dev->bdev,
768a739ff3fSSami Tolvanen 1 << v->data_dev_block_bits,
7690fcb100dSNathan Huckleberry 1, 0, NULL, NULL, 0);
770a739ff3fSSami Tolvanen if (IS_ERR(f->data_bufio)) {
771a739ff3fSSami Tolvanen ti->error = "Cannot initialize FEC data bufio client";
772a739ff3fSSami Tolvanen return PTR_ERR(f->data_bufio);
773a739ff3fSSami Tolvanen }
774a739ff3fSSami Tolvanen
775a739ff3fSSami Tolvanen if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) {
776a739ff3fSSami Tolvanen ti->error = "Data device is too small";
777a739ff3fSSami Tolvanen return -E2BIG;
778a739ff3fSSami Tolvanen }
779a739ff3fSSami Tolvanen
780a739ff3fSSami Tolvanen /* Preallocate an rs_control structure for each worker thread */
7816f1c819cSKent Overstreet ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc,
782a739ff3fSSami Tolvanen fec_rs_free, (void *) v);
7836f1c819cSKent Overstreet if (ret) {
784a739ff3fSSami Tolvanen ti->error = "Cannot allocate RS pool";
7856f1c819cSKent Overstreet return ret;
786a739ff3fSSami Tolvanen }
787a739ff3fSSami Tolvanen
788a739ff3fSSami Tolvanen f->cache = kmem_cache_create("dm_verity_fec_buffers",
789a739ff3fSSami Tolvanen f->rsn << DM_VERITY_FEC_BUF_RS_BITS,
790a739ff3fSSami Tolvanen 0, 0, NULL);
791a739ff3fSSami Tolvanen if (!f->cache) {
792a739ff3fSSami Tolvanen ti->error = "Cannot create FEC buffer cache";
793a739ff3fSSami Tolvanen return -ENOMEM;
794a739ff3fSSami Tolvanen }
795a739ff3fSSami Tolvanen
796a739ff3fSSami Tolvanen /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */
7976f1c819cSKent Overstreet ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() *
798a739ff3fSSami Tolvanen DM_VERITY_FEC_BUF_PREALLOC,
799a739ff3fSSami Tolvanen f->cache);
8006f1c819cSKent Overstreet if (ret) {
801a739ff3fSSami Tolvanen ti->error = "Cannot allocate FEC buffer prealloc pool";
8026f1c819cSKent Overstreet return ret;
803a739ff3fSSami Tolvanen }
804a739ff3fSSami Tolvanen
8056f1c819cSKent Overstreet ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache);
8066f1c819cSKent Overstreet if (ret) {
807a739ff3fSSami Tolvanen ti->error = "Cannot allocate FEC buffer extra pool";
8086f1c819cSKent Overstreet return ret;
809a739ff3fSSami Tolvanen }
810a739ff3fSSami Tolvanen
811a739ff3fSSami Tolvanen /* Preallocate an output buffer for each thread */
8126f1c819cSKent Overstreet ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(),
813a739ff3fSSami Tolvanen 1 << v->data_dev_block_bits);
8146f1c819cSKent Overstreet if (ret) {
815a739ff3fSSami Tolvanen ti->error = "Cannot allocate FEC output pool";
8166f1c819cSKent Overstreet return ret;
817a739ff3fSSami Tolvanen }
818a739ff3fSSami Tolvanen
819a739ff3fSSami Tolvanen /* Reserve space for our per-bio data */
82030187e1dSMike Snitzer ti->per_io_data_size += sizeof(struct dm_verity_fec_io);
821a739ff3fSSami Tolvanen
822a739ff3fSSami Tolvanen return 0;
823a739ff3fSSami Tolvanen }
824