xref: /openbmc/linux/drivers/mtd/nand/ecc-sw-bch.c (revision 2a598d0b)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * This file provides ECC correction for more than 1 bit per block of data,
4  * using binary BCH codes. It relies on the generic BCH library lib/bch.c.
5  *
6  * Copyright © 2011 Ivan Djelic <ivan.djelic@parrot.com>
7  */
8 
9 #include <linux/types.h>
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/slab.h>
13 #include <linux/bitops.h>
14 #include <linux/mtd/nand.h>
15 #include <linux/mtd/nand-ecc-sw-bch.h>
16 
17 /**
18  * nand_ecc_sw_bch_calculate - Calculate the ECC corresponding to a data block
19  * @nand: NAND device
20  * @buf: Input buffer with raw data
21  * @code: Output buffer with ECC
22  */
23 int nand_ecc_sw_bch_calculate(struct nand_device *nand,
24 			      const unsigned char *buf, unsigned char *code)
25 {
26 	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
27 	unsigned int i;
28 
29 	memset(code, 0, engine_conf->code_size);
30 	bch_encode(engine_conf->bch, buf, nand->ecc.ctx.conf.step_size, code);
31 
32 	/* apply mask so that an erased page is a valid codeword */
33 	for (i = 0; i < engine_conf->code_size; i++)
34 		code[i] ^= engine_conf->eccmask[i];
35 
36 	return 0;
37 }
38 EXPORT_SYMBOL(nand_ecc_sw_bch_calculate);
39 
40 /**
41  * nand_ecc_sw_bch_correct - Detect, correct and report bit error(s)
42  * @nand: NAND device
43  * @buf: Raw data read from the chip
44  * @read_ecc: ECC bytes from the chip
45  * @calc_ecc: ECC calculated from the raw data
46  *
47  * Detect and correct bit errors for a data block.
48  */
49 int nand_ecc_sw_bch_correct(struct nand_device *nand, unsigned char *buf,
50 			    unsigned char *read_ecc, unsigned char *calc_ecc)
51 {
52 	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
53 	unsigned int step_size = nand->ecc.ctx.conf.step_size;
54 	unsigned int *errloc = engine_conf->errloc;
55 	int i, count;
56 
57 	count = bch_decode(engine_conf->bch, NULL, step_size, read_ecc,
58 			   calc_ecc, NULL, errloc);
59 	if (count > 0) {
60 		for (i = 0; i < count; i++) {
61 			if (errloc[i] < (step_size * 8))
62 				/* The error is in the data area: correct it */
63 				buf[errloc[i] >> 3] ^= (1 << (errloc[i] & 7));
64 
65 			/* Otherwise the error is in the ECC area: nothing to do */
66 			pr_debug("%s: corrected bitflip %u\n", __func__,
67 				 errloc[i]);
68 		}
69 	} else if (count < 0) {
70 		pr_err("ECC unrecoverable error\n");
71 		count = -EBADMSG;
72 	}
73 
74 	return count;
75 }
76 EXPORT_SYMBOL(nand_ecc_sw_bch_correct);
77 
78 /**
79  * nand_ecc_sw_bch_cleanup - Cleanup software BCH ECC resources
80  * @nand: NAND device
81  */
82 static void nand_ecc_sw_bch_cleanup(struct nand_device *nand)
83 {
84 	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
85 
86 	bch_free(engine_conf->bch);
87 	kfree(engine_conf->errloc);
88 	kfree(engine_conf->eccmask);
89 }
90 
91 /**
92  * nand_ecc_sw_bch_init - Initialize software BCH ECC engine
93  * @nand: NAND device
94  *
95  * Returns: a pointer to a new NAND BCH control structure, or NULL upon failure
96  *
97  * Initialize NAND BCH error correction. @nand.ecc parameters 'step_size' and
98  * 'bytes' are used to compute the following BCH parameters:
99  *     m, the Galois field order
100  *     t, the error correction capability
101  * 'bytes' should be equal to the number of bytes required to store m * t
102  * bits, where m is such that 2^m - 1 > step_size * 8.
103  *
104  * Example: to configure 4 bit correction per 512 bytes, you should pass
105  * step_size = 512 (thus, m = 13 is the smallest integer such that 2^m - 1 > 512 * 8)
106  * bytes = 7 (7 bytes are required to store m * t = 13 * 4 = 52 bits)
107  */
108 static int nand_ecc_sw_bch_init(struct nand_device *nand)
109 {
110 	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
111 	unsigned int eccsize = nand->ecc.ctx.conf.step_size;
112 	unsigned int eccbytes = engine_conf->code_size;
113 	unsigned int m, t, i;
114 	unsigned char *erased_page;
115 	int ret;
116 
117 	m = fls(1 + (8 * eccsize));
118 	t = (eccbytes * 8) / m;
119 
120 	engine_conf->bch = bch_init(m, t, 0, false);
121 	if (!engine_conf->bch)
122 		return -EINVAL;
123 
124 	engine_conf->eccmask = kzalloc(eccbytes, GFP_KERNEL);
125 	engine_conf->errloc = kmalloc_array(t, sizeof(*engine_conf->errloc),
126 					    GFP_KERNEL);
127 	if (!engine_conf->eccmask || !engine_conf->errloc) {
128 		ret = -ENOMEM;
129 		goto cleanup;
130 	}
131 
132 	/* Compute and store the inverted ECC of an erased step */
133 	erased_page = kmalloc(eccsize, GFP_KERNEL);
134 	if (!erased_page) {
135 		ret = -ENOMEM;
136 		goto cleanup;
137 	}
138 
139 	memset(erased_page, 0xff, eccsize);
140 	bch_encode(engine_conf->bch, erased_page, eccsize,
141 		   engine_conf->eccmask);
142 	kfree(erased_page);
143 
144 	for (i = 0; i < eccbytes; i++)
145 		engine_conf->eccmask[i] ^= 0xff;
146 
147 	/* Verify that the number of code bytes has the expected value */
148 	if (engine_conf->bch->ecc_bytes != eccbytes) {
149 		pr_err("Invalid number of ECC bytes: %u, expected: %u\n",
150 		       eccbytes, engine_conf->bch->ecc_bytes);
151 		ret = -EINVAL;
152 		goto cleanup;
153 	}
154 
155 	/* Sanity checks */
156 	if (8 * (eccsize + eccbytes) >= (1 << m)) {
157 		pr_err("ECC step size is too large (%u)\n", eccsize);
158 		ret = -EINVAL;
159 		goto cleanup;
160 	}
161 
162 	return 0;
163 
164 cleanup:
165 	nand_ecc_sw_bch_cleanup(nand);
166 
167 	return ret;
168 }
169 
170 int nand_ecc_sw_bch_init_ctx(struct nand_device *nand)
171 {
172 	struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
173 	struct mtd_info *mtd = nanddev_to_mtd(nand);
174 	struct nand_ecc_sw_bch_conf *engine_conf;
175 	unsigned int code_size = 0, nsteps;
176 	int ret;
177 
178 	/* Only large page NAND chips may use BCH */
179 	if (mtd->oobsize < 64) {
180 		pr_err("BCH cannot be used with small page NAND chips\n");
181 		return -EINVAL;
182 	}
183 
184 	if (!mtd->ooblayout)
185 		mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout());
186 
187 	conf->engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
188 	conf->algo = NAND_ECC_ALGO_BCH;
189 	conf->step_size = nand->ecc.user_conf.step_size;
190 	conf->strength = nand->ecc.user_conf.strength;
191 
192 	/*
193 	 * Board driver should supply ECC size and ECC strength
194 	 * values to select how many bits are correctable.
195 	 * Otherwise, default to 512 bytes for large page devices and 256 for
196 	 * small page devices.
197 	 */
198 	if (!conf->step_size) {
199 		if (mtd->oobsize >= 64)
200 			conf->step_size = 512;
201 		else
202 			conf->step_size = 256;
203 
204 		conf->strength = 4;
205 	}
206 
207 	nsteps = mtd->writesize / conf->step_size;
208 
209 	/* Maximize */
210 	if (nand->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
211 		conf->step_size = 1024;
212 		nsteps = mtd->writesize / conf->step_size;
213 		/* Reserve 2 bytes for the BBM */
214 		code_size = (mtd->oobsize - 2) / nsteps;
215 		conf->strength = code_size * 8 / fls(8 * conf->step_size);
216 	}
217 
218 	if (!code_size)
219 		code_size = DIV_ROUND_UP(conf->strength *
220 					 fls(8 * conf->step_size), 8);
221 
222 	if (!conf->strength)
223 		conf->strength = (code_size * 8) / fls(8 * conf->step_size);
224 
225 	if (!code_size && !conf->strength) {
226 		pr_err("Missing ECC parameters\n");
227 		return -EINVAL;
228 	}
229 
230 	engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL);
231 	if (!engine_conf)
232 		return -ENOMEM;
233 
234 	ret = nand_ecc_init_req_tweaking(&engine_conf->req_ctx, nand);
235 	if (ret)
236 		goto free_engine_conf;
237 
238 	engine_conf->code_size = code_size;
239 	engine_conf->calc_buf = kzalloc(mtd->oobsize, GFP_KERNEL);
240 	engine_conf->code_buf = kzalloc(mtd->oobsize, GFP_KERNEL);
241 	if (!engine_conf->calc_buf || !engine_conf->code_buf) {
242 		ret = -ENOMEM;
243 		goto free_bufs;
244 	}
245 
246 	nand->ecc.ctx.priv = engine_conf;
247 	nand->ecc.ctx.nsteps = nsteps;
248 	nand->ecc.ctx.total = nsteps * code_size;
249 
250 	ret = nand_ecc_sw_bch_init(nand);
251 	if (ret)
252 		goto free_bufs;
253 
254 	/* Verify the layout validity */
255 	if (mtd_ooblayout_count_eccbytes(mtd) !=
256 	    nand->ecc.ctx.nsteps * engine_conf->code_size) {
257 		pr_err("Invalid ECC layout\n");
258 		ret = -EINVAL;
259 		goto cleanup_bch_ctx;
260 	}
261 
262 	return 0;
263 
264 cleanup_bch_ctx:
265 	nand_ecc_sw_bch_cleanup(nand);
266 free_bufs:
267 	nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);
268 	kfree(engine_conf->calc_buf);
269 	kfree(engine_conf->code_buf);
270 free_engine_conf:
271 	kfree(engine_conf);
272 
273 	return ret;
274 }
275 EXPORT_SYMBOL(nand_ecc_sw_bch_init_ctx);
276 
277 void nand_ecc_sw_bch_cleanup_ctx(struct nand_device *nand)
278 {
279 	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
280 
281 	if (engine_conf) {
282 		nand_ecc_sw_bch_cleanup(nand);
283 		nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);
284 		kfree(engine_conf->calc_buf);
285 		kfree(engine_conf->code_buf);
286 		kfree(engine_conf);
287 	}
288 }
289 EXPORT_SYMBOL(nand_ecc_sw_bch_cleanup_ctx);
290 
291 static int nand_ecc_sw_bch_prepare_io_req(struct nand_device *nand,
292 					  struct nand_page_io_req *req)
293 {
294 	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
295 	struct mtd_info *mtd = nanddev_to_mtd(nand);
296 	int eccsize = nand->ecc.ctx.conf.step_size;
297 	int eccbytes = engine_conf->code_size;
298 	int eccsteps = nand->ecc.ctx.nsteps;
299 	int total = nand->ecc.ctx.total;
300 	u8 *ecccalc = engine_conf->calc_buf;
301 	const u8 *data;
302 	int i;
303 
304 	/* Nothing to do for a raw operation */
305 	if (req->mode == MTD_OPS_RAW)
306 		return 0;
307 
308 	/* This engine does not provide BBM/free OOB bytes protection */
309 	if (!req->datalen)
310 		return 0;
311 
312 	nand_ecc_tweak_req(&engine_conf->req_ctx, req);
313 
314 	/* No more preparation for page read */
315 	if (req->type == NAND_PAGE_READ)
316 		return 0;
317 
318 	/* Preparation for page write: derive the ECC bytes and place them */
319 	for (i = 0, data = req->databuf.out;
320 	     eccsteps;
321 	     eccsteps--, i += eccbytes, data += eccsize)
322 		nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]);
323 
324 	return mtd_ooblayout_set_eccbytes(mtd, ecccalc, (void *)req->oobbuf.out,
325 					  0, total);
326 }
327 
328 static int nand_ecc_sw_bch_finish_io_req(struct nand_device *nand,
329 					 struct nand_page_io_req *req)
330 {
331 	struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv;
332 	struct mtd_info *mtd = nanddev_to_mtd(nand);
333 	int eccsize = nand->ecc.ctx.conf.step_size;
334 	int total = nand->ecc.ctx.total;
335 	int eccbytes = engine_conf->code_size;
336 	int eccsteps = nand->ecc.ctx.nsteps;
337 	u8 *ecccalc = engine_conf->calc_buf;
338 	u8 *ecccode = engine_conf->code_buf;
339 	unsigned int max_bitflips = 0;
340 	u8 *data = req->databuf.in;
341 	int i, ret;
342 
343 	/* Nothing to do for a raw operation */
344 	if (req->mode == MTD_OPS_RAW)
345 		return 0;
346 
347 	/* This engine does not provide BBM/free OOB bytes protection */
348 	if (!req->datalen)
349 		return 0;
350 
351 	/* No more preparation for page write */
352 	if (req->type == NAND_PAGE_WRITE) {
353 		nand_ecc_restore_req(&engine_conf->req_ctx, req);
354 		return 0;
355 	}
356 
357 	/* Finish a page read: retrieve the (raw) ECC bytes*/
358 	ret = mtd_ooblayout_get_eccbytes(mtd, ecccode, req->oobbuf.in, 0,
359 					 total);
360 	if (ret)
361 		return ret;
362 
363 	/* Calculate the ECC bytes */
364 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, data += eccsize)
365 		nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]);
366 
367 	/* Finish a page read: compare and correct */
368 	for (eccsteps = nand->ecc.ctx.nsteps, i = 0, data = req->databuf.in;
369 	     eccsteps;
370 	     eccsteps--, i += eccbytes, data += eccsize) {
371 		int stat =  nand_ecc_sw_bch_correct(nand, data,
372 						    &ecccode[i],
373 						    &ecccalc[i]);
374 		if (stat < 0) {
375 			mtd->ecc_stats.failed++;
376 		} else {
377 			mtd->ecc_stats.corrected += stat;
378 			max_bitflips = max_t(unsigned int, max_bitflips, stat);
379 		}
380 	}
381 
382 	nand_ecc_restore_req(&engine_conf->req_ctx, req);
383 
384 	return max_bitflips;
385 }
386 
387 static struct nand_ecc_engine_ops nand_ecc_sw_bch_engine_ops = {
388 	.init_ctx = nand_ecc_sw_bch_init_ctx,
389 	.cleanup_ctx = nand_ecc_sw_bch_cleanup_ctx,
390 	.prepare_io_req = nand_ecc_sw_bch_prepare_io_req,
391 	.finish_io_req = nand_ecc_sw_bch_finish_io_req,
392 };
393 
394 static struct nand_ecc_engine nand_ecc_sw_bch_engine = {
395 	.ops = &nand_ecc_sw_bch_engine_ops,
396 };
397 
398 struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void)
399 {
400 	return &nand_ecc_sw_bch_engine;
401 }
402 EXPORT_SYMBOL(nand_ecc_sw_bch_get_engine);
403 
404 MODULE_LICENSE("GPL");
405 MODULE_AUTHOR("Ivan Djelic <ivan.djelic@parrot.com>");
406 MODULE_DESCRIPTION("NAND software BCH ECC support");
407