1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * This file contains an ECC algorithm that detects and corrects 1 bit
4  * errors in a 256 byte block of data.
5  *
6  * Copyright © 2008 Koninklijke Philips Electronics NV.
7  *                  Author: Frans Meulenbroeks
8  *
9  * Completely replaces the previous ECC implementation which was written by:
10  *   Steven J. Hill (sjhill@realitydiluted.com)
11  *   Thomas Gleixner (tglx@linutronix.de)
12  *
13  * Information on how this algorithm works and how it was developed
14  * can be found in Documentation/driver-api/mtd/nand_ecc.rst
15  */
16 
17 #include <linux/types.h>
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include <linux/mtd/nand.h>
21 #include <linux/mtd/nand-ecc-sw-hamming.h>
22 #include <linux/slab.h>
23 #include <asm/byteorder.h>
24 
25 /*
26  * invparity is a 256 byte table that contains the odd parity
27  * for each byte. So if the number of bits in a byte is even,
28  * the array element is 1, and when the number of bits is odd
29  * the array eleemnt is 0.
30  */
31 static const char invparity[256] = {
32 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
33 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
34 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
35 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
36 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
37 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
38 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
39 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
40 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
41 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
42 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
43 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
44 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1,
45 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
46 	0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
47 	1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1
48 };
49 
50 /*
51  * bitsperbyte contains the number of bits per byte
52  * this is only used for testing and repairing parity
53  * (a precalculated value slightly improves performance)
54  */
55 static const char bitsperbyte[256] = {
56 	0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
57 	1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
58 	1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
59 	2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
60 	1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
61 	2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
62 	2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
63 	3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
64 	1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
65 	2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
66 	2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
67 	3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
68 	2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
69 	3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
70 	3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
71 	4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8,
72 };
73 
74 /*
75  * addressbits is a lookup table to filter out the bits from the xor-ed
76  * ECC data that identify the faulty location.
77  * this is only used for repairing parity
78  * see the comments in nand_ecc_sw_hamming_correct for more details
79  */
80 static const char addressbits[256] = {
81 	0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01,
82 	0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
83 	0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01,
84 	0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
85 	0x04, 0x04, 0x05, 0x05, 0x04, 0x04, 0x05, 0x05,
86 	0x06, 0x06, 0x07, 0x07, 0x06, 0x06, 0x07, 0x07,
87 	0x04, 0x04, 0x05, 0x05, 0x04, 0x04, 0x05, 0x05,
88 	0x06, 0x06, 0x07, 0x07, 0x06, 0x06, 0x07, 0x07,
89 	0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01,
90 	0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
91 	0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x01,
92 	0x02, 0x02, 0x03, 0x03, 0x02, 0x02, 0x03, 0x03,
93 	0x04, 0x04, 0x05, 0x05, 0x04, 0x04, 0x05, 0x05,
94 	0x06, 0x06, 0x07, 0x07, 0x06, 0x06, 0x07, 0x07,
95 	0x04, 0x04, 0x05, 0x05, 0x04, 0x04, 0x05, 0x05,
96 	0x06, 0x06, 0x07, 0x07, 0x06, 0x06, 0x07, 0x07,
97 	0x08, 0x08, 0x09, 0x09, 0x08, 0x08, 0x09, 0x09,
98 	0x0a, 0x0a, 0x0b, 0x0b, 0x0a, 0x0a, 0x0b, 0x0b,
99 	0x08, 0x08, 0x09, 0x09, 0x08, 0x08, 0x09, 0x09,
100 	0x0a, 0x0a, 0x0b, 0x0b, 0x0a, 0x0a, 0x0b, 0x0b,
101 	0x0c, 0x0c, 0x0d, 0x0d, 0x0c, 0x0c, 0x0d, 0x0d,
102 	0x0e, 0x0e, 0x0f, 0x0f, 0x0e, 0x0e, 0x0f, 0x0f,
103 	0x0c, 0x0c, 0x0d, 0x0d, 0x0c, 0x0c, 0x0d, 0x0d,
104 	0x0e, 0x0e, 0x0f, 0x0f, 0x0e, 0x0e, 0x0f, 0x0f,
105 	0x08, 0x08, 0x09, 0x09, 0x08, 0x08, 0x09, 0x09,
106 	0x0a, 0x0a, 0x0b, 0x0b, 0x0a, 0x0a, 0x0b, 0x0b,
107 	0x08, 0x08, 0x09, 0x09, 0x08, 0x08, 0x09, 0x09,
108 	0x0a, 0x0a, 0x0b, 0x0b, 0x0a, 0x0a, 0x0b, 0x0b,
109 	0x0c, 0x0c, 0x0d, 0x0d, 0x0c, 0x0c, 0x0d, 0x0d,
110 	0x0e, 0x0e, 0x0f, 0x0f, 0x0e, 0x0e, 0x0f, 0x0f,
111 	0x0c, 0x0c, 0x0d, 0x0d, 0x0c, 0x0c, 0x0d, 0x0d,
112 	0x0e, 0x0e, 0x0f, 0x0f, 0x0e, 0x0e, 0x0f, 0x0f
113 };
114 
ecc_sw_hamming_calculate(const unsigned char * buf,unsigned int step_size,unsigned char * code,bool sm_order)115 int ecc_sw_hamming_calculate(const unsigned char *buf, unsigned int step_size,
116 			     unsigned char *code, bool sm_order)
117 {
118 	const u32 *bp = (uint32_t *)buf;
119 	const u32 eccsize_mult = (step_size == 256) ? 1 : 2;
120 	/* current value in buffer */
121 	u32 cur;
122 	/* rp0..rp17 are the various accumulated parities (per byte) */
123 	u32 rp0, rp1, rp2, rp3, rp4, rp5, rp6, rp7, rp8, rp9, rp10, rp11, rp12,
124 		rp13, rp14, rp15, rp16, rp17;
125 	/* Cumulative parity for all data */
126 	u32 par;
127 	/* Cumulative parity at the end of the loop (rp12, rp14, rp16) */
128 	u32 tmppar;
129 	int i;
130 
131 	par = 0;
132 	rp4 = 0;
133 	rp6 = 0;
134 	rp8 = 0;
135 	rp10 = 0;
136 	rp12 = 0;
137 	rp14 = 0;
138 	rp16 = 0;
139 	rp17 = 0;
140 
141 	/*
142 	 * The loop is unrolled a number of times;
143 	 * This avoids if statements to decide on which rp value to update
144 	 * Also we process the data by longwords.
145 	 * Note: passing unaligned data might give a performance penalty.
146 	 * It is assumed that the buffers are aligned.
147 	 * tmppar is the cumulative sum of this iteration.
148 	 * needed for calculating rp12, rp14, rp16 and par
149 	 * also used as a performance improvement for rp6, rp8 and rp10
150 	 */
151 	for (i = 0; i < eccsize_mult << 2; i++) {
152 		cur = *bp++;
153 		tmppar = cur;
154 		rp4 ^= cur;
155 		cur = *bp++;
156 		tmppar ^= cur;
157 		rp6 ^= tmppar;
158 		cur = *bp++;
159 		tmppar ^= cur;
160 		rp4 ^= cur;
161 		cur = *bp++;
162 		tmppar ^= cur;
163 		rp8 ^= tmppar;
164 
165 		cur = *bp++;
166 		tmppar ^= cur;
167 		rp4 ^= cur;
168 		rp6 ^= cur;
169 		cur = *bp++;
170 		tmppar ^= cur;
171 		rp6 ^= cur;
172 		cur = *bp++;
173 		tmppar ^= cur;
174 		rp4 ^= cur;
175 		cur = *bp++;
176 		tmppar ^= cur;
177 		rp10 ^= tmppar;
178 
179 		cur = *bp++;
180 		tmppar ^= cur;
181 		rp4 ^= cur;
182 		rp6 ^= cur;
183 		rp8 ^= cur;
184 		cur = *bp++;
185 		tmppar ^= cur;
186 		rp6 ^= cur;
187 		rp8 ^= cur;
188 		cur = *bp++;
189 		tmppar ^= cur;
190 		rp4 ^= cur;
191 		rp8 ^= cur;
192 		cur = *bp++;
193 		tmppar ^= cur;
194 		rp8 ^= cur;
195 
196 		cur = *bp++;
197 		tmppar ^= cur;
198 		rp4 ^= cur;
199 		rp6 ^= cur;
200 		cur = *bp++;
201 		tmppar ^= cur;
202 		rp6 ^= cur;
203 		cur = *bp++;
204 		tmppar ^= cur;
205 		rp4 ^= cur;
206 		cur = *bp++;
207 		tmppar ^= cur;
208 
209 		par ^= tmppar;
210 		if ((i & 0x1) == 0)
211 			rp12 ^= tmppar;
212 		if ((i & 0x2) == 0)
213 			rp14 ^= tmppar;
214 		if (eccsize_mult == 2 && (i & 0x4) == 0)
215 			rp16 ^= tmppar;
216 	}
217 
218 	/*
219 	 * handle the fact that we use longword operations
220 	 * we'll bring rp4..rp14..rp16 back to single byte entities by
221 	 * shifting and xoring first fold the upper and lower 16 bits,
222 	 * then the upper and lower 8 bits.
223 	 */
224 	rp4 ^= (rp4 >> 16);
225 	rp4 ^= (rp4 >> 8);
226 	rp4 &= 0xff;
227 	rp6 ^= (rp6 >> 16);
228 	rp6 ^= (rp6 >> 8);
229 	rp6 &= 0xff;
230 	rp8 ^= (rp8 >> 16);
231 	rp8 ^= (rp8 >> 8);
232 	rp8 &= 0xff;
233 	rp10 ^= (rp10 >> 16);
234 	rp10 ^= (rp10 >> 8);
235 	rp10 &= 0xff;
236 	rp12 ^= (rp12 >> 16);
237 	rp12 ^= (rp12 >> 8);
238 	rp12 &= 0xff;
239 	rp14 ^= (rp14 >> 16);
240 	rp14 ^= (rp14 >> 8);
241 	rp14 &= 0xff;
242 	if (eccsize_mult == 2) {
243 		rp16 ^= (rp16 >> 16);
244 		rp16 ^= (rp16 >> 8);
245 		rp16 &= 0xff;
246 	}
247 
248 	/*
249 	 * we also need to calculate the row parity for rp0..rp3
250 	 * This is present in par, because par is now
251 	 * rp3 rp3 rp2 rp2 in little endian and
252 	 * rp2 rp2 rp3 rp3 in big endian
253 	 * as well as
254 	 * rp1 rp0 rp1 rp0 in little endian and
255 	 * rp0 rp1 rp0 rp1 in big endian
256 	 * First calculate rp2 and rp3
257 	 */
258 #ifdef __BIG_ENDIAN
259 	rp2 = (par >> 16);
260 	rp2 ^= (rp2 >> 8);
261 	rp2 &= 0xff;
262 	rp3 = par & 0xffff;
263 	rp3 ^= (rp3 >> 8);
264 	rp3 &= 0xff;
265 #else
266 	rp3 = (par >> 16);
267 	rp3 ^= (rp3 >> 8);
268 	rp3 &= 0xff;
269 	rp2 = par & 0xffff;
270 	rp2 ^= (rp2 >> 8);
271 	rp2 &= 0xff;
272 #endif
273 
274 	/* reduce par to 16 bits then calculate rp1 and rp0 */
275 	par ^= (par >> 16);
276 #ifdef __BIG_ENDIAN
277 	rp0 = (par >> 8) & 0xff;
278 	rp1 = (par & 0xff);
279 #else
280 	rp1 = (par >> 8) & 0xff;
281 	rp0 = (par & 0xff);
282 #endif
283 
284 	/* finally reduce par to 8 bits */
285 	par ^= (par >> 8);
286 	par &= 0xff;
287 
288 	/*
289 	 * and calculate rp5..rp15..rp17
290 	 * note that par = rp4 ^ rp5 and due to the commutative property
291 	 * of the ^ operator we can say:
292 	 * rp5 = (par ^ rp4);
293 	 * The & 0xff seems superfluous, but benchmarking learned that
294 	 * leaving it out gives slightly worse results. No idea why, probably
295 	 * it has to do with the way the pipeline in pentium is organized.
296 	 */
297 	rp5 = (par ^ rp4) & 0xff;
298 	rp7 = (par ^ rp6) & 0xff;
299 	rp9 = (par ^ rp8) & 0xff;
300 	rp11 = (par ^ rp10) & 0xff;
301 	rp13 = (par ^ rp12) & 0xff;
302 	rp15 = (par ^ rp14) & 0xff;
303 	if (eccsize_mult == 2)
304 		rp17 = (par ^ rp16) & 0xff;
305 
306 	/*
307 	 * Finally calculate the ECC bits.
308 	 * Again here it might seem that there are performance optimisations
309 	 * possible, but benchmarks showed that on the system this is developed
310 	 * the code below is the fastest
311 	 */
312 	if (sm_order) {
313 		code[0] = (invparity[rp7] << 7) | (invparity[rp6] << 6) |
314 			  (invparity[rp5] << 5) | (invparity[rp4] << 4) |
315 			  (invparity[rp3] << 3) | (invparity[rp2] << 2) |
316 			  (invparity[rp1] << 1) | (invparity[rp0]);
317 		code[1] = (invparity[rp15] << 7) | (invparity[rp14] << 6) |
318 			  (invparity[rp13] << 5) | (invparity[rp12] << 4) |
319 			  (invparity[rp11] << 3) | (invparity[rp10] << 2) |
320 			  (invparity[rp9] << 1) | (invparity[rp8]);
321 	} else {
322 		code[1] = (invparity[rp7] << 7) | (invparity[rp6] << 6) |
323 			  (invparity[rp5] << 5) | (invparity[rp4] << 4) |
324 			  (invparity[rp3] << 3) | (invparity[rp2] << 2) |
325 			  (invparity[rp1] << 1) | (invparity[rp0]);
326 		code[0] = (invparity[rp15] << 7) | (invparity[rp14] << 6) |
327 			  (invparity[rp13] << 5) | (invparity[rp12] << 4) |
328 			  (invparity[rp11] << 3) | (invparity[rp10] << 2) |
329 			  (invparity[rp9] << 1) | (invparity[rp8]);
330 	}
331 
332 	if (eccsize_mult == 1)
333 		code[2] =
334 		    (invparity[par & 0xf0] << 7) |
335 		    (invparity[par & 0x0f] << 6) |
336 		    (invparity[par & 0xcc] << 5) |
337 		    (invparity[par & 0x33] << 4) |
338 		    (invparity[par & 0xaa] << 3) |
339 		    (invparity[par & 0x55] << 2) |
340 		    3;
341 	else
342 		code[2] =
343 		    (invparity[par & 0xf0] << 7) |
344 		    (invparity[par & 0x0f] << 6) |
345 		    (invparity[par & 0xcc] << 5) |
346 		    (invparity[par & 0x33] << 4) |
347 		    (invparity[par & 0xaa] << 3) |
348 		    (invparity[par & 0x55] << 2) |
349 		    (invparity[rp17] << 1) |
350 		    (invparity[rp16] << 0);
351 
352 	return 0;
353 }
354 EXPORT_SYMBOL(ecc_sw_hamming_calculate);
355 
356 /**
357  * nand_ecc_sw_hamming_calculate - Calculate 3-byte ECC for 256/512-byte block
358  * @nand: NAND device
359  * @buf: Input buffer with raw data
360  * @code: Output buffer with ECC
361  */
nand_ecc_sw_hamming_calculate(struct nand_device * nand,const unsigned char * buf,unsigned char * code)362 int nand_ecc_sw_hamming_calculate(struct nand_device *nand,
363 				  const unsigned char *buf, unsigned char *code)
364 {
365 	struct nand_ecc_sw_hamming_conf *engine_conf = nand->ecc.ctx.priv;
366 	unsigned int step_size = nand->ecc.ctx.conf.step_size;
367 	bool sm_order = engine_conf ? engine_conf->sm_order : false;
368 
369 	return ecc_sw_hamming_calculate(buf, step_size, code, sm_order);
370 }
371 EXPORT_SYMBOL(nand_ecc_sw_hamming_calculate);
372 
ecc_sw_hamming_correct(unsigned char * buf,unsigned char * read_ecc,unsigned char * calc_ecc,unsigned int step_size,bool sm_order)373 int ecc_sw_hamming_correct(unsigned char *buf, unsigned char *read_ecc,
374 			   unsigned char *calc_ecc, unsigned int step_size,
375 			   bool sm_order)
376 {
377 	const u32 eccsize_mult = step_size >> 8;
378 	unsigned char b0, b1, b2, bit_addr;
379 	unsigned int byte_addr;
380 
381 	/*
382 	 * b0 to b2 indicate which bit is faulty (if any)
383 	 * we might need the xor result  more than once,
384 	 * so keep them in a local var
385 	*/
386 	if (sm_order) {
387 		b0 = read_ecc[0] ^ calc_ecc[0];
388 		b1 = read_ecc[1] ^ calc_ecc[1];
389 	} else {
390 		b0 = read_ecc[1] ^ calc_ecc[1];
391 		b1 = read_ecc[0] ^ calc_ecc[0];
392 	}
393 
394 	b2 = read_ecc[2] ^ calc_ecc[2];
395 
396 	/* check if there are any bitfaults */
397 
398 	/* repeated if statements are slightly more efficient than switch ... */
399 	/* ordered in order of likelihood */
400 
401 	if ((b0 | b1 | b2) == 0)
402 		return 0;	/* no error */
403 
404 	if ((((b0 ^ (b0 >> 1)) & 0x55) == 0x55) &&
405 	    (((b1 ^ (b1 >> 1)) & 0x55) == 0x55) &&
406 	    ((eccsize_mult == 1 && ((b2 ^ (b2 >> 1)) & 0x54) == 0x54) ||
407 	     (eccsize_mult == 2 && ((b2 ^ (b2 >> 1)) & 0x55) == 0x55))) {
408 	/* single bit error */
409 		/*
410 		 * rp17/rp15/13/11/9/7/5/3/1 indicate which byte is the faulty
411 		 * byte, cp 5/3/1 indicate the faulty bit.
412 		 * A lookup table (called addressbits) is used to filter
413 		 * the bits from the byte they are in.
414 		 * A marginal optimisation is possible by having three
415 		 * different lookup tables.
416 		 * One as we have now (for b0), one for b2
417 		 * (that would avoid the >> 1), and one for b1 (with all values
418 		 * << 4). However it was felt that introducing two more tables
419 		 * hardly justify the gain.
420 		 *
421 		 * The b2 shift is there to get rid of the lowest two bits.
422 		 * We could also do addressbits[b2] >> 1 but for the
423 		 * performance it does not make any difference
424 		 */
425 		if (eccsize_mult == 1)
426 			byte_addr = (addressbits[b1] << 4) + addressbits[b0];
427 		else
428 			byte_addr = (addressbits[b2 & 0x3] << 8) +
429 				    (addressbits[b1] << 4) + addressbits[b0];
430 		bit_addr = addressbits[b2 >> 2];
431 		/* flip the bit */
432 		buf[byte_addr] ^= (1 << bit_addr);
433 		return 1;
434 
435 	}
436 	/* count nr of bits; use table lookup, faster than calculating it */
437 	if ((bitsperbyte[b0] + bitsperbyte[b1] + bitsperbyte[b2]) == 1)
438 		return 1;	/* error in ECC data; no action needed */
439 
440 	pr_err("%s: uncorrectable ECC error\n", __func__);
441 	return -EBADMSG;
442 }
443 EXPORT_SYMBOL(ecc_sw_hamming_correct);
444 
445 /**
446  * nand_ecc_sw_hamming_correct - Detect and correct bit error(s)
447  * @nand: NAND device
448  * @buf: Raw data read from the chip
449  * @read_ecc: ECC bytes read from the chip
450  * @calc_ecc: ECC calculated from the raw data
451  *
452  * Detect and correct up to 1 bit error per 256/512-byte block.
453  */
nand_ecc_sw_hamming_correct(struct nand_device * nand,unsigned char * buf,unsigned char * read_ecc,unsigned char * calc_ecc)454 int nand_ecc_sw_hamming_correct(struct nand_device *nand, unsigned char *buf,
455 				unsigned char *read_ecc,
456 				unsigned char *calc_ecc)
457 {
458 	struct nand_ecc_sw_hamming_conf *engine_conf = nand->ecc.ctx.priv;
459 	unsigned int step_size = nand->ecc.ctx.conf.step_size;
460 	bool sm_order = engine_conf ? engine_conf->sm_order : false;
461 
462 	return ecc_sw_hamming_correct(buf, read_ecc, calc_ecc, step_size,
463 				      sm_order);
464 }
465 EXPORT_SYMBOL(nand_ecc_sw_hamming_correct);
466 
nand_ecc_sw_hamming_init_ctx(struct nand_device * nand)467 int nand_ecc_sw_hamming_init_ctx(struct nand_device *nand)
468 {
469 	struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
470 	struct nand_ecc_sw_hamming_conf *engine_conf;
471 	struct mtd_info *mtd = nanddev_to_mtd(nand);
472 	int ret;
473 
474 	if (!mtd->ooblayout) {
475 		switch (mtd->oobsize) {
476 		case 8:
477 		case 16:
478 			mtd_set_ooblayout(mtd, nand_get_small_page_ooblayout());
479 			break;
480 		case 64:
481 		case 128:
482 			mtd_set_ooblayout(mtd,
483 					  nand_get_large_page_hamming_ooblayout());
484 			break;
485 		default:
486 			return -ENOTSUPP;
487 		}
488 	}
489 
490 	conf->engine_type = NAND_ECC_ENGINE_TYPE_SOFT;
491 	conf->algo = NAND_ECC_ALGO_HAMMING;
492 	conf->step_size = nand->ecc.user_conf.step_size;
493 	conf->strength = 1;
494 
495 	/* Use the strongest configuration by default */
496 	if (conf->step_size != 256 && conf->step_size != 512)
497 		conf->step_size = 256;
498 
499 	engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL);
500 	if (!engine_conf)
501 		return -ENOMEM;
502 
503 	ret = nand_ecc_init_req_tweaking(&engine_conf->req_ctx, nand);
504 	if (ret)
505 		goto free_engine_conf;
506 
507 	engine_conf->code_size = 3;
508 	engine_conf->calc_buf = kzalloc(mtd->oobsize, GFP_KERNEL);
509 	engine_conf->code_buf = kzalloc(mtd->oobsize, GFP_KERNEL);
510 	if (!engine_conf->calc_buf || !engine_conf->code_buf) {
511 		ret = -ENOMEM;
512 		goto free_bufs;
513 	}
514 
515 	nand->ecc.ctx.priv = engine_conf;
516 	nand->ecc.ctx.nsteps = mtd->writesize / conf->step_size;
517 	nand->ecc.ctx.total = nand->ecc.ctx.nsteps * engine_conf->code_size;
518 
519 	return 0;
520 
521 free_bufs:
522 	nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);
523 	kfree(engine_conf->calc_buf);
524 	kfree(engine_conf->code_buf);
525 free_engine_conf:
526 	kfree(engine_conf);
527 
528 	return ret;
529 }
530 EXPORT_SYMBOL(nand_ecc_sw_hamming_init_ctx);
531 
nand_ecc_sw_hamming_cleanup_ctx(struct nand_device * nand)532 void nand_ecc_sw_hamming_cleanup_ctx(struct nand_device *nand)
533 {
534 	struct nand_ecc_sw_hamming_conf *engine_conf = nand->ecc.ctx.priv;
535 
536 	if (engine_conf) {
537 		nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx);
538 		kfree(engine_conf->calc_buf);
539 		kfree(engine_conf->code_buf);
540 		kfree(engine_conf);
541 	}
542 }
543 EXPORT_SYMBOL(nand_ecc_sw_hamming_cleanup_ctx);
544 
nand_ecc_sw_hamming_prepare_io_req(struct nand_device * nand,struct nand_page_io_req * req)545 static int nand_ecc_sw_hamming_prepare_io_req(struct nand_device *nand,
546 					      struct nand_page_io_req *req)
547 {
548 	struct nand_ecc_sw_hamming_conf *engine_conf = nand->ecc.ctx.priv;
549 	struct mtd_info *mtd = nanddev_to_mtd(nand);
550 	int eccsize = nand->ecc.ctx.conf.step_size;
551 	int eccbytes = engine_conf->code_size;
552 	int eccsteps = nand->ecc.ctx.nsteps;
553 	int total = nand->ecc.ctx.total;
554 	u8 *ecccalc = engine_conf->calc_buf;
555 	const u8 *data;
556 	int i;
557 
558 	/* Nothing to do for a raw operation */
559 	if (req->mode == MTD_OPS_RAW)
560 		return 0;
561 
562 	/* This engine does not provide BBM/free OOB bytes protection */
563 	if (!req->datalen)
564 		return 0;
565 
566 	nand_ecc_tweak_req(&engine_conf->req_ctx, req);
567 
568 	/* No more preparation for page read */
569 	if (req->type == NAND_PAGE_READ)
570 		return 0;
571 
572 	/* Preparation for page write: derive the ECC bytes and place them */
573 	for (i = 0, data = req->databuf.out;
574 	     eccsteps;
575 	     eccsteps--, i += eccbytes, data += eccsize)
576 		nand_ecc_sw_hamming_calculate(nand, data, &ecccalc[i]);
577 
578 	return mtd_ooblayout_set_eccbytes(mtd, ecccalc, (void *)req->oobbuf.out,
579 					  0, total);
580 }
581 
nand_ecc_sw_hamming_finish_io_req(struct nand_device * nand,struct nand_page_io_req * req)582 static int nand_ecc_sw_hamming_finish_io_req(struct nand_device *nand,
583 					     struct nand_page_io_req *req)
584 {
585 	struct nand_ecc_sw_hamming_conf *engine_conf = nand->ecc.ctx.priv;
586 	struct mtd_info *mtd = nanddev_to_mtd(nand);
587 	int eccsize = nand->ecc.ctx.conf.step_size;
588 	int total = nand->ecc.ctx.total;
589 	int eccbytes = engine_conf->code_size;
590 	int eccsteps = nand->ecc.ctx.nsteps;
591 	u8 *ecccalc = engine_conf->calc_buf;
592 	u8 *ecccode = engine_conf->code_buf;
593 	unsigned int max_bitflips = 0;
594 	u8 *data = req->databuf.in;
595 	int i, ret;
596 
597 	/* Nothing to do for a raw operation */
598 	if (req->mode == MTD_OPS_RAW)
599 		return 0;
600 
601 	/* This engine does not provide BBM/free OOB bytes protection */
602 	if (!req->datalen)
603 		return 0;
604 
605 	/* No more preparation for page write */
606 	if (req->type == NAND_PAGE_WRITE) {
607 		nand_ecc_restore_req(&engine_conf->req_ctx, req);
608 		return 0;
609 	}
610 
611 	/* Finish a page read: retrieve the (raw) ECC bytes*/
612 	ret = mtd_ooblayout_get_eccbytes(mtd, ecccode, req->oobbuf.in, 0,
613 					 total);
614 	if (ret)
615 		return ret;
616 
617 	/* Calculate the ECC bytes */
618 	for (i = 0; eccsteps; eccsteps--, i += eccbytes, data += eccsize)
619 		nand_ecc_sw_hamming_calculate(nand, data, &ecccalc[i]);
620 
621 	/* Finish a page read: compare and correct */
622 	for (eccsteps = nand->ecc.ctx.nsteps, i = 0, data = req->databuf.in;
623 	     eccsteps;
624 	     eccsteps--, i += eccbytes, data += eccsize) {
625 		int stat =  nand_ecc_sw_hamming_correct(nand, data,
626 							&ecccode[i],
627 							&ecccalc[i]);
628 		if (stat < 0) {
629 			mtd->ecc_stats.failed++;
630 		} else {
631 			mtd->ecc_stats.corrected += stat;
632 			max_bitflips = max_t(unsigned int, max_bitflips, stat);
633 		}
634 	}
635 
636 	nand_ecc_restore_req(&engine_conf->req_ctx, req);
637 
638 	return max_bitflips;
639 }
640 
641 static struct nand_ecc_engine_ops nand_ecc_sw_hamming_engine_ops = {
642 	.init_ctx = nand_ecc_sw_hamming_init_ctx,
643 	.cleanup_ctx = nand_ecc_sw_hamming_cleanup_ctx,
644 	.prepare_io_req = nand_ecc_sw_hamming_prepare_io_req,
645 	.finish_io_req = nand_ecc_sw_hamming_finish_io_req,
646 };
647 
648 static struct nand_ecc_engine nand_ecc_sw_hamming_engine = {
649 	.ops = &nand_ecc_sw_hamming_engine_ops,
650 };
651 
nand_ecc_sw_hamming_get_engine(void)652 struct nand_ecc_engine *nand_ecc_sw_hamming_get_engine(void)
653 {
654 	return &nand_ecc_sw_hamming_engine;
655 }
656 EXPORT_SYMBOL(nand_ecc_sw_hamming_get_engine);
657 
658 MODULE_LICENSE("GPL");
659 MODULE_AUTHOR("Frans Meulenbroeks <fransmeulenbroeks@gmail.com>");
660 MODULE_DESCRIPTION("NAND software Hamming ECC support");
661