1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * This file provides ECC correction for more than 1 bit per block of data, 4 * using binary BCH codes. It relies on the generic BCH library lib/bch.c. 5 * 6 * Copyright © 2011 Ivan Djelic <ivan.djelic@parrot.com> 7 */ 8 9 #include <linux/types.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/slab.h> 13 #include <linux/bitops.h> 14 #include <linux/mtd/nand.h> 15 #include <linux/mtd/nand-ecc-sw-bch.h> 16 17 /** 18 * nand_ecc_sw_bch_calculate - Calculate the ECC corresponding to a data block 19 * @nand: NAND device 20 * @buf: Input buffer with raw data 21 * @code: Output buffer with ECC 22 */ 23 int nand_ecc_sw_bch_calculate(struct nand_device *nand, 24 const unsigned char *buf, unsigned char *code) 25 { 26 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; 27 unsigned int i; 28 29 memset(code, 0, engine_conf->code_size); 30 bch_encode(engine_conf->bch, buf, nand->ecc.ctx.conf.step_size, code); 31 32 /* apply mask so that an erased page is a valid codeword */ 33 for (i = 0; i < engine_conf->code_size; i++) 34 code[i] ^= engine_conf->eccmask[i]; 35 36 return 0; 37 } 38 EXPORT_SYMBOL(nand_ecc_sw_bch_calculate); 39 40 /** 41 * nand_ecc_sw_bch_correct - Detect, correct and report bit error(s) 42 * @nand: NAND device 43 * @buf: Raw data read from the chip 44 * @read_ecc: ECC bytes from the chip 45 * @calc_ecc: ECC calculated from the raw data 46 * 47 * Detect and correct bit errors for a data block. 48 */ 49 int nand_ecc_sw_bch_correct(struct nand_device *nand, unsigned char *buf, 50 unsigned char *read_ecc, unsigned char *calc_ecc) 51 { 52 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; 53 unsigned int step_size = nand->ecc.ctx.conf.step_size; 54 unsigned int *errloc = engine_conf->errloc; 55 int i, count; 56 57 count = bch_decode(engine_conf->bch, NULL, step_size, read_ecc, 58 calc_ecc, NULL, errloc); 59 if (count > 0) { 60 for (i = 0; i < count; i++) { 61 if (errloc[i] < (step_size * 8)) 62 /* The error is in the data area: correct it */ 63 buf[errloc[i] >> 3] ^= (1 << (errloc[i] & 7)); 64 65 /* Otherwise the error is in the ECC area: nothing to do */ 66 pr_debug("%s: corrected bitflip %u\n", __func__, 67 errloc[i]); 68 } 69 } else if (count < 0) { 70 pr_err("ECC unrecoverable error\n"); 71 count = -EBADMSG; 72 } 73 74 return count; 75 } 76 EXPORT_SYMBOL(nand_ecc_sw_bch_correct); 77 78 /** 79 * nand_ecc_sw_bch_cleanup - Cleanup software BCH ECC resources 80 * @nand: NAND device 81 */ 82 static void nand_ecc_sw_bch_cleanup(struct nand_device *nand) 83 { 84 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; 85 86 bch_free(engine_conf->bch); 87 kfree(engine_conf->errloc); 88 kfree(engine_conf->eccmask); 89 } 90 91 /** 92 * nand_ecc_sw_bch_init - Initialize software BCH ECC engine 93 * @nand: NAND device 94 * 95 * Returns: a pointer to a new NAND BCH control structure, or NULL upon failure 96 * 97 * Initialize NAND BCH error correction. @nand.ecc parameters 'step_size' and 98 * 'bytes' are used to compute the following BCH parameters: 99 * m, the Galois field order 100 * t, the error correction capability 101 * 'bytes' should be equal to the number of bytes required to store m * t 102 * bits, where m is such that 2^m - 1 > step_size * 8. 103 * 104 * Example: to configure 4 bit correction per 512 bytes, you should pass 105 * step_size = 512 (thus, m = 13 is the smallest integer such that 2^m - 1 > 512 * 8) 106 * bytes = 7 (7 bytes are required to store m * t = 13 * 4 = 52 bits) 107 */ 108 static int nand_ecc_sw_bch_init(struct nand_device *nand) 109 { 110 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; 111 unsigned int eccsize = nand->ecc.ctx.conf.step_size; 112 unsigned int eccbytes = engine_conf->code_size; 113 unsigned int m, t, i; 114 unsigned char *erased_page; 115 int ret; 116 117 m = fls(1 + (8 * eccsize)); 118 t = (eccbytes * 8) / m; 119 120 engine_conf->bch = bch_init(m, t, 0, false); 121 if (!engine_conf->bch) 122 return -EINVAL; 123 124 engine_conf->eccmask = kzalloc(eccbytes, GFP_KERNEL); 125 engine_conf->errloc = kmalloc_array(t, sizeof(*engine_conf->errloc), 126 GFP_KERNEL); 127 if (!engine_conf->eccmask || !engine_conf->errloc) { 128 ret = -ENOMEM; 129 goto cleanup; 130 } 131 132 /* Compute and store the inverted ECC of an erased step */ 133 erased_page = kmalloc(eccsize, GFP_KERNEL); 134 if (!erased_page) { 135 ret = -ENOMEM; 136 goto cleanup; 137 } 138 139 memset(erased_page, 0xff, eccsize); 140 bch_encode(engine_conf->bch, erased_page, eccsize, 141 engine_conf->eccmask); 142 kfree(erased_page); 143 144 for (i = 0; i < eccbytes; i++) 145 engine_conf->eccmask[i] ^= 0xff; 146 147 /* Verify that the number of code bytes has the expected value */ 148 if (engine_conf->bch->ecc_bytes != eccbytes) { 149 pr_err("Invalid number of ECC bytes: %u, expected: %u\n", 150 eccbytes, engine_conf->bch->ecc_bytes); 151 ret = -EINVAL; 152 goto cleanup; 153 } 154 155 /* Sanity checks */ 156 if (8 * (eccsize + eccbytes) >= (1 << m)) { 157 pr_err("ECC step size is too large (%u)\n", eccsize); 158 ret = -EINVAL; 159 goto cleanup; 160 } 161 162 return 0; 163 164 cleanup: 165 nand_ecc_sw_bch_cleanup(nand); 166 167 return ret; 168 } 169 170 int nand_ecc_sw_bch_init_ctx(struct nand_device *nand) 171 { 172 struct nand_ecc_props *conf = &nand->ecc.ctx.conf; 173 struct mtd_info *mtd = nanddev_to_mtd(nand); 174 struct nand_ecc_sw_bch_conf *engine_conf; 175 unsigned int code_size = 0, nsteps; 176 int ret; 177 178 /* Only large page NAND chips may use BCH */ 179 if (mtd->oobsize < 64) { 180 pr_err("BCH cannot be used with small page NAND chips\n"); 181 return -EINVAL; 182 } 183 184 if (!mtd->ooblayout) 185 mtd_set_ooblayout(mtd, nand_get_large_page_ooblayout()); 186 187 conf->engine_type = NAND_ECC_ENGINE_TYPE_SOFT; 188 conf->algo = NAND_ECC_ALGO_BCH; 189 conf->step_size = nand->ecc.user_conf.step_size; 190 conf->strength = nand->ecc.user_conf.strength; 191 192 /* 193 * Board driver should supply ECC size and ECC strength 194 * values to select how many bits are correctable. 195 * Otherwise, default to 512 bytes for large page devices and 256 for 196 * small page devices. 197 */ 198 if (!conf->step_size) { 199 if (mtd->oobsize >= 64) 200 conf->step_size = 512; 201 else 202 conf->step_size = 256; 203 204 conf->strength = 4; 205 } 206 207 nsteps = mtd->writesize / conf->step_size; 208 209 /* Maximize */ 210 if (nand->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) { 211 conf->step_size = 1024; 212 nsteps = mtd->writesize / conf->step_size; 213 /* Reserve 2 bytes for the BBM */ 214 code_size = (mtd->oobsize - 2) / nsteps; 215 conf->strength = code_size * 8 / fls(8 * conf->step_size); 216 } 217 218 if (!code_size) 219 code_size = DIV_ROUND_UP(conf->strength * 220 fls(8 * conf->step_size), 8); 221 222 if (!conf->strength) 223 conf->strength = (code_size * 8) / fls(8 * conf->step_size); 224 225 if (!code_size && !conf->strength) { 226 pr_err("Missing ECC parameters\n"); 227 return -EINVAL; 228 } 229 230 engine_conf = kzalloc(sizeof(*engine_conf), GFP_KERNEL); 231 if (!engine_conf) 232 return -ENOMEM; 233 234 ret = nand_ecc_init_req_tweaking(&engine_conf->req_ctx, nand); 235 if (ret) 236 goto free_engine_conf; 237 238 engine_conf->code_size = code_size; 239 engine_conf->calc_buf = kzalloc(mtd->oobsize, GFP_KERNEL); 240 engine_conf->code_buf = kzalloc(mtd->oobsize, GFP_KERNEL); 241 if (!engine_conf->calc_buf || !engine_conf->code_buf) { 242 ret = -ENOMEM; 243 goto free_bufs; 244 } 245 246 nand->ecc.ctx.priv = engine_conf; 247 nand->ecc.ctx.nsteps = nsteps; 248 nand->ecc.ctx.total = nsteps * code_size; 249 250 ret = nand_ecc_sw_bch_init(nand); 251 if (ret) 252 goto free_bufs; 253 254 /* Verify the layout validity */ 255 if (mtd_ooblayout_count_eccbytes(mtd) != 256 nand->ecc.ctx.nsteps * engine_conf->code_size) { 257 pr_err("Invalid ECC layout\n"); 258 ret = -EINVAL; 259 goto cleanup_bch_ctx; 260 } 261 262 return 0; 263 264 cleanup_bch_ctx: 265 nand_ecc_sw_bch_cleanup(nand); 266 free_bufs: 267 nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx); 268 kfree(engine_conf->calc_buf); 269 kfree(engine_conf->code_buf); 270 free_engine_conf: 271 kfree(engine_conf); 272 273 return ret; 274 } 275 EXPORT_SYMBOL(nand_ecc_sw_bch_init_ctx); 276 277 void nand_ecc_sw_bch_cleanup_ctx(struct nand_device *nand) 278 { 279 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; 280 281 if (engine_conf) { 282 nand_ecc_sw_bch_cleanup(nand); 283 nand_ecc_cleanup_req_tweaking(&engine_conf->req_ctx); 284 kfree(engine_conf->calc_buf); 285 kfree(engine_conf->code_buf); 286 kfree(engine_conf); 287 } 288 } 289 EXPORT_SYMBOL(nand_ecc_sw_bch_cleanup_ctx); 290 291 static int nand_ecc_sw_bch_prepare_io_req(struct nand_device *nand, 292 struct nand_page_io_req *req) 293 { 294 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; 295 struct mtd_info *mtd = nanddev_to_mtd(nand); 296 int eccsize = nand->ecc.ctx.conf.step_size; 297 int eccbytes = engine_conf->code_size; 298 int eccsteps = nand->ecc.ctx.nsteps; 299 int total = nand->ecc.ctx.total; 300 u8 *ecccalc = engine_conf->calc_buf; 301 const u8 *data; 302 int i; 303 304 /* Nothing to do for a raw operation */ 305 if (req->mode == MTD_OPS_RAW) 306 return 0; 307 308 /* This engine does not provide BBM/free OOB bytes protection */ 309 if (!req->datalen) 310 return 0; 311 312 nand_ecc_tweak_req(&engine_conf->req_ctx, req); 313 314 /* No more preparation for page read */ 315 if (req->type == NAND_PAGE_READ) 316 return 0; 317 318 /* Preparation for page write: derive the ECC bytes and place them */ 319 for (i = 0, data = req->databuf.out; 320 eccsteps; 321 eccsteps--, i += eccbytes, data += eccsize) 322 nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]); 323 324 return mtd_ooblayout_set_eccbytes(mtd, ecccalc, (void *)req->oobbuf.out, 325 0, total); 326 } 327 328 static int nand_ecc_sw_bch_finish_io_req(struct nand_device *nand, 329 struct nand_page_io_req *req) 330 { 331 struct nand_ecc_sw_bch_conf *engine_conf = nand->ecc.ctx.priv; 332 struct mtd_info *mtd = nanddev_to_mtd(nand); 333 int eccsize = nand->ecc.ctx.conf.step_size; 334 int total = nand->ecc.ctx.total; 335 int eccbytes = engine_conf->code_size; 336 int eccsteps = nand->ecc.ctx.nsteps; 337 u8 *ecccalc = engine_conf->calc_buf; 338 u8 *ecccode = engine_conf->code_buf; 339 unsigned int max_bitflips = 0; 340 u8 *data = req->databuf.in; 341 int i, ret; 342 343 /* Nothing to do for a raw operation */ 344 if (req->mode == MTD_OPS_RAW) 345 return 0; 346 347 /* This engine does not provide BBM/free OOB bytes protection */ 348 if (!req->datalen) 349 return 0; 350 351 /* No more preparation for page write */ 352 if (req->type == NAND_PAGE_WRITE) { 353 nand_ecc_restore_req(&engine_conf->req_ctx, req); 354 return 0; 355 } 356 357 /* Finish a page read: retrieve the (raw) ECC bytes*/ 358 ret = mtd_ooblayout_get_eccbytes(mtd, ecccode, req->oobbuf.in, 0, 359 total); 360 if (ret) 361 return ret; 362 363 /* Calculate the ECC bytes */ 364 for (i = 0; eccsteps; eccsteps--, i += eccbytes, data += eccsize) 365 nand_ecc_sw_bch_calculate(nand, data, &ecccalc[i]); 366 367 /* Finish a page read: compare and correct */ 368 for (eccsteps = nand->ecc.ctx.nsteps, i = 0, data = req->databuf.in; 369 eccsteps; 370 eccsteps--, i += eccbytes, data += eccsize) { 371 int stat = nand_ecc_sw_bch_correct(nand, data, 372 &ecccode[i], 373 &ecccalc[i]); 374 if (stat < 0) { 375 mtd->ecc_stats.failed++; 376 } else { 377 mtd->ecc_stats.corrected += stat; 378 max_bitflips = max_t(unsigned int, max_bitflips, stat); 379 } 380 } 381 382 nand_ecc_restore_req(&engine_conf->req_ctx, req); 383 384 return max_bitflips; 385 } 386 387 static struct nand_ecc_engine_ops nand_ecc_sw_bch_engine_ops = { 388 .init_ctx = nand_ecc_sw_bch_init_ctx, 389 .cleanup_ctx = nand_ecc_sw_bch_cleanup_ctx, 390 .prepare_io_req = nand_ecc_sw_bch_prepare_io_req, 391 .finish_io_req = nand_ecc_sw_bch_finish_io_req, 392 }; 393 394 static struct nand_ecc_engine nand_ecc_sw_bch_engine = { 395 .ops = &nand_ecc_sw_bch_engine_ops, 396 }; 397 398 struct nand_ecc_engine *nand_ecc_sw_bch_get_engine(void) 399 { 400 return &nand_ecc_sw_bch_engine; 401 } 402 EXPORT_SYMBOL(nand_ecc_sw_bch_get_engine); 403 404 MODULE_LICENSE("GPL"); 405 MODULE_AUTHOR("Ivan Djelic <ivan.djelic@parrot.com>"); 406 MODULE_DESCRIPTION("NAND software BCH ECC support"); 407