1 /* 2 * Copyright 2017 ATMEL 3 * Copyright 2017 Free Electrons 4 * 5 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 6 * 7 * Derived from the atmel_nand.c driver which contained the following 8 * copyrights: 9 * 10 * Copyright 2003 Rick Bronson 11 * 12 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8) 13 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de) 14 * 15 * Derived from drivers/mtd/spia.c (removed in v3.8) 16 * Copyright 2000 Steven J. Hill (sjhill@cotw.com) 17 * 18 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263 19 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007 20 * 21 * Derived from Das U-Boot source code 22 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c) 23 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas 24 * 25 * Add Programmable Multibit ECC support for various AT91 SoC 26 * Copyright 2012 ATMEL, Hong Xu 27 * 28 * Add Nand Flash Controller support for SAMA5 SoC 29 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com) 30 * 31 * This program is free software; you can redistribute it and/or modify 32 * it under the terms of the GNU General Public License version 2 as 33 * published by the Free Software Foundation. 34 * 35 * The PMECC is an hardware assisted BCH engine, which means part of the 36 * ECC algorithm is left to the software. The hardware/software repartition 37 * is explained in the "PMECC Controller Functional Description" chapter in 38 * Atmel datasheets, and some of the functions in this file are directly 39 * implementing the algorithms described in the "Software Implementation" 40 * sub-section. 41 * 42 * TODO: it seems that the software BCH implementation in lib/bch.c is already 43 * providing some of the logic we are implementing here. It would be smart 44 * to expose the needed lib/bch.c helpers/functions and re-use them here. 45 */ 46 47 #include <linux/genalloc.h> 48 #include <linux/iopoll.h> 49 #include <linux/module.h> 50 #include <linux/mtd/rawnand.h> 51 #include <linux/of_irq.h> 52 #include <linux/of_platform.h> 53 #include <linux/platform_device.h> 54 #include <linux/slab.h> 55 56 #include "pmecc.h" 57 58 /* Galois field dimension */ 59 #define PMECC_GF_DIMENSION_13 13 60 #define PMECC_GF_DIMENSION_14 14 61 62 /* Primitive Polynomial used by PMECC */ 63 #define PMECC_GF_13_PRIMITIVE_POLY 0x201b 64 #define PMECC_GF_14_PRIMITIVE_POLY 0x4443 65 66 #define PMECC_LOOKUP_TABLE_SIZE_512 0x2000 67 #define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000 68 69 /* Time out value for reading PMECC status register */ 70 #define PMECC_MAX_TIMEOUT_MS 100 71 72 /* PMECC Register Definitions */ 73 #define ATMEL_PMECC_CFG 0x0 74 #define PMECC_CFG_BCH_STRENGTH(x) (x) 75 #define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0) 76 #define PMECC_CFG_SECTOR512 (0 << 4) 77 #define PMECC_CFG_SECTOR1024 (1 << 4) 78 #define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8) 79 #define PMECC_CFG_READ_OP (0 << 12) 80 #define PMECC_CFG_WRITE_OP (1 << 12) 81 #define PMECC_CFG_SPARE_ENABLE BIT(16) 82 #define PMECC_CFG_AUTO_ENABLE BIT(20) 83 84 #define ATMEL_PMECC_SAREA 0x4 85 #define ATMEL_PMECC_SADDR 0x8 86 #define ATMEL_PMECC_EADDR 0xc 87 88 #define ATMEL_PMECC_CLK 0x10 89 #define PMECC_CLK_133MHZ (2 << 0) 90 91 #define ATMEL_PMECC_CTRL 0x14 92 #define PMECC_CTRL_RST BIT(0) 93 #define PMECC_CTRL_DATA BIT(1) 94 #define PMECC_CTRL_USER BIT(2) 95 #define PMECC_CTRL_ENABLE BIT(4) 96 #define PMECC_CTRL_DISABLE BIT(5) 97 98 #define ATMEL_PMECC_SR 0x18 99 #define PMECC_SR_BUSY BIT(0) 100 #define PMECC_SR_ENABLE BIT(4) 101 102 #define ATMEL_PMECC_IER 0x1c 103 #define ATMEL_PMECC_IDR 0x20 104 #define ATMEL_PMECC_IMR 0x24 105 #define ATMEL_PMECC_ISR 0x28 106 #define PMECC_ERROR_INT BIT(0) 107 108 #define ATMEL_PMECC_ECC(sector, n) \ 109 ((((sector) + 1) * 0x40) + (n)) 110 111 #define ATMEL_PMECC_REM(sector, n) \ 112 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200) 113 114 /* PMERRLOC Register Definitions */ 115 #define ATMEL_PMERRLOC_ELCFG 0x0 116 #define PMERRLOC_ELCFG_SECTOR_512 (0 << 0) 117 #define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0) 118 #define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16) 119 120 #define ATMEL_PMERRLOC_ELPRIM 0x4 121 #define ATMEL_PMERRLOC_ELEN 0x8 122 #define ATMEL_PMERRLOC_ELDIS 0xc 123 #define PMERRLOC_DISABLE BIT(0) 124 125 #define ATMEL_PMERRLOC_ELSR 0x10 126 #define PMERRLOC_ELSR_BUSY BIT(0) 127 128 #define ATMEL_PMERRLOC_ELIER 0x14 129 #define ATMEL_PMERRLOC_ELIDR 0x18 130 #define ATMEL_PMERRLOC_ELIMR 0x1c 131 #define ATMEL_PMERRLOC_ELISR 0x20 132 #define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8) 133 #define PMERRLOC_CALC_DONE BIT(0) 134 135 #define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28) 136 137 #define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs)) 138 139 struct atmel_pmecc_gf_tables { 140 u16 *alpha_to; 141 u16 *index_of; 142 }; 143 144 struct atmel_pmecc_caps { 145 const int *strengths; 146 int nstrengths; 147 int el_offset; 148 bool correct_erased_chunks; 149 }; 150 151 struct atmel_pmecc { 152 struct device *dev; 153 const struct atmel_pmecc_caps *caps; 154 155 struct { 156 void __iomem *base; 157 void __iomem *errloc; 158 } regs; 159 160 struct mutex lock; 161 }; 162 163 struct atmel_pmecc_user_conf_cache { 164 u32 cfg; 165 u32 sarea; 166 u32 saddr; 167 u32 eaddr; 168 }; 169 170 struct atmel_pmecc_user { 171 struct atmel_pmecc_user_conf_cache cache; 172 struct atmel_pmecc *pmecc; 173 const struct atmel_pmecc_gf_tables *gf_tables; 174 int eccbytes; 175 s16 *partial_syn; 176 s16 *si; 177 s16 *lmu; 178 s16 *smu; 179 s32 *mu; 180 s32 *dmu; 181 s32 *delta; 182 u32 isr; 183 }; 184 185 static DEFINE_MUTEX(pmecc_gf_tables_lock); 186 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512; 187 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024; 188 189 static inline int deg(unsigned int poly) 190 { 191 /* polynomial degree is the most-significant bit index */ 192 return fls(poly) - 1; 193 } 194 195 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly, 196 struct atmel_pmecc_gf_tables *gf_tables) 197 { 198 unsigned int i, x = 1; 199 const unsigned int k = BIT(deg(poly)); 200 unsigned int nn = BIT(mm) - 1; 201 202 /* primitive polynomial must be of degree m */ 203 if (k != (1u << mm)) 204 return -EINVAL; 205 206 for (i = 0; i < nn; i++) { 207 gf_tables->alpha_to[i] = x; 208 gf_tables->index_of[x] = i; 209 if (i && (x == 1)) 210 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */ 211 return -EINVAL; 212 x <<= 1; 213 if (x & k) 214 x ^= poly; 215 } 216 gf_tables->alpha_to[nn] = 1; 217 gf_tables->index_of[0] = 0; 218 219 return 0; 220 } 221 222 static const struct atmel_pmecc_gf_tables * 223 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req) 224 { 225 struct atmel_pmecc_gf_tables *gf_tables; 226 unsigned int poly, degree, table_size; 227 int ret; 228 229 if (req->ecc.sectorsize == 512) { 230 degree = PMECC_GF_DIMENSION_13; 231 poly = PMECC_GF_13_PRIMITIVE_POLY; 232 table_size = PMECC_LOOKUP_TABLE_SIZE_512; 233 } else { 234 degree = PMECC_GF_DIMENSION_14; 235 poly = PMECC_GF_14_PRIMITIVE_POLY; 236 table_size = PMECC_LOOKUP_TABLE_SIZE_1024; 237 } 238 239 gf_tables = kzalloc(sizeof(*gf_tables) + 240 (2 * table_size * sizeof(u16)), 241 GFP_KERNEL); 242 if (!gf_tables) 243 return ERR_PTR(-ENOMEM); 244 245 gf_tables->alpha_to = (void *)(gf_tables + 1); 246 gf_tables->index_of = gf_tables->alpha_to + table_size; 247 248 ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables); 249 if (ret) { 250 kfree(gf_tables); 251 return ERR_PTR(ret); 252 } 253 254 return gf_tables; 255 } 256 257 static const struct atmel_pmecc_gf_tables * 258 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req) 259 { 260 const struct atmel_pmecc_gf_tables **gf_tables, *ret; 261 262 mutex_lock(&pmecc_gf_tables_lock); 263 if (req->ecc.sectorsize == 512) 264 gf_tables = &pmecc_gf_tables_512; 265 else 266 gf_tables = &pmecc_gf_tables_1024; 267 268 ret = *gf_tables; 269 270 if (!ret) { 271 ret = atmel_pmecc_create_gf_tables(req); 272 if (!IS_ERR(ret)) 273 *gf_tables = ret; 274 } 275 mutex_unlock(&pmecc_gf_tables_lock); 276 277 return ret; 278 } 279 280 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc, 281 struct atmel_pmecc_user_req *req) 282 { 283 int i, max_eccbytes, eccbytes = 0, eccstrength = 0; 284 285 if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0) 286 return -EINVAL; 287 288 if (req->ecc.ooboffset >= 0 && 289 req->ecc.ooboffset + req->ecc.bytes > req->oobsize) 290 return -EINVAL; 291 292 if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) { 293 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 294 return -EINVAL; 295 296 if (req->pagesize > 512) 297 req->ecc.sectorsize = 1024; 298 else 299 req->ecc.sectorsize = 512; 300 } 301 302 if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024) 303 return -EINVAL; 304 305 if (req->pagesize % req->ecc.sectorsize) 306 return -EINVAL; 307 308 req->ecc.nsectors = req->pagesize / req->ecc.sectorsize; 309 310 max_eccbytes = req->ecc.bytes; 311 312 for (i = 0; i < pmecc->caps->nstrengths; i++) { 313 int nbytes, strength = pmecc->caps->strengths[i]; 314 315 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH && 316 strength < req->ecc.strength) 317 continue; 318 319 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize), 320 8); 321 nbytes *= req->ecc.nsectors; 322 323 if (nbytes > max_eccbytes) 324 break; 325 326 eccstrength = strength; 327 eccbytes = nbytes; 328 329 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 330 break; 331 } 332 333 if (!eccstrength) 334 return -EINVAL; 335 336 req->ecc.bytes = eccbytes; 337 req->ecc.strength = eccstrength; 338 339 if (req->ecc.ooboffset < 0) 340 req->ecc.ooboffset = req->oobsize - eccbytes; 341 342 return 0; 343 } 344 345 struct atmel_pmecc_user * 346 atmel_pmecc_create_user(struct atmel_pmecc *pmecc, 347 struct atmel_pmecc_user_req *req) 348 { 349 struct atmel_pmecc_user *user; 350 const struct atmel_pmecc_gf_tables *gf_tables; 351 int strength, size, ret; 352 353 ret = atmel_pmecc_prepare_user_req(pmecc, req); 354 if (ret) 355 return ERR_PTR(ret); 356 357 size = sizeof(*user); 358 size = ALIGN(size, sizeof(u16)); 359 /* Reserve space for partial_syn, si and smu */ 360 size += ((2 * req->ecc.strength) + 1) * sizeof(u16) * 361 (2 + req->ecc.strength + 2); 362 /* Reserve space for lmu. */ 363 size += (req->ecc.strength + 1) * sizeof(u16); 364 /* Reserve space for mu, dmu and delta. */ 365 size = ALIGN(size, sizeof(s32)); 366 size += (req->ecc.strength + 1) * sizeof(s32) * 3; 367 368 user = kzalloc(size, GFP_KERNEL); 369 if (!user) 370 return ERR_PTR(-ENOMEM); 371 372 user->pmecc = pmecc; 373 374 user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16)); 375 user->si = user->partial_syn + ((2 * req->ecc.strength) + 1); 376 user->lmu = user->si + ((2 * req->ecc.strength) + 1); 377 user->smu = user->lmu + (req->ecc.strength + 1); 378 user->mu = (s32 *)PTR_ALIGN(user->smu + 379 (((2 * req->ecc.strength) + 1) * 380 (req->ecc.strength + 2)), 381 sizeof(s32)); 382 user->dmu = user->mu + req->ecc.strength + 1; 383 user->delta = user->dmu + req->ecc.strength + 1; 384 385 gf_tables = atmel_pmecc_get_gf_tables(req); 386 if (IS_ERR(gf_tables)) { 387 kfree(user); 388 return ERR_CAST(gf_tables); 389 } 390 391 user->gf_tables = gf_tables; 392 393 user->eccbytes = req->ecc.bytes / req->ecc.nsectors; 394 395 for (strength = 0; strength < pmecc->caps->nstrengths; strength++) { 396 if (pmecc->caps->strengths[strength] == req->ecc.strength) 397 break; 398 } 399 400 user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) | 401 PMECC_CFG_NSECTORS(req->ecc.nsectors); 402 403 if (req->ecc.sectorsize == 1024) 404 user->cache.cfg |= PMECC_CFG_SECTOR1024; 405 406 user->cache.sarea = req->oobsize - 1; 407 user->cache.saddr = req->ecc.ooboffset; 408 user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1; 409 410 return user; 411 } 412 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user); 413 414 void atmel_pmecc_destroy_user(struct atmel_pmecc_user *user) 415 { 416 kfree(user); 417 } 418 EXPORT_SYMBOL_GPL(atmel_pmecc_destroy_user); 419 420 static int get_strength(struct atmel_pmecc_user *user) 421 { 422 const int *strengths = user->pmecc->caps->strengths; 423 424 return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK]; 425 } 426 427 static int get_sectorsize(struct atmel_pmecc_user *user) 428 { 429 return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512; 430 } 431 432 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector) 433 { 434 int strength = get_strength(user); 435 u32 value; 436 int i; 437 438 /* Fill odd syndromes */ 439 for (i = 0; i < strength; i++) { 440 value = readl_relaxed(user->pmecc->regs.base + 441 ATMEL_PMECC_REM(sector, i / 2)); 442 if (i & 1) 443 value >>= 16; 444 445 user->partial_syn[(2 * i) + 1] = value; 446 } 447 } 448 449 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user) 450 { 451 int degree = get_sectorsize(user) == 512 ? 13 : 14; 452 int cw_len = BIT(degree) - 1; 453 int strength = get_strength(user); 454 s16 *alpha_to = user->gf_tables->alpha_to; 455 s16 *index_of = user->gf_tables->index_of; 456 s16 *partial_syn = user->partial_syn; 457 s16 *si; 458 int i, j; 459 460 /* 461 * si[] is a table that holds the current syndrome value, 462 * an element of that table belongs to the field 463 */ 464 si = user->si; 465 466 memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1)); 467 468 /* Computation 2t syndromes based on S(x) */ 469 /* Odd syndromes */ 470 for (i = 1; i < 2 * strength; i += 2) { 471 for (j = 0; j < degree; j++) { 472 if (partial_syn[i] & BIT(j)) 473 si[i] = alpha_to[i * j] ^ si[i]; 474 } 475 } 476 /* Even syndrome = (Odd syndrome) ** 2 */ 477 for (i = 2, j = 1; j <= strength; i = ++j << 1) { 478 if (si[j] == 0) { 479 si[i] = 0; 480 } else { 481 s16 tmp; 482 483 tmp = index_of[si[j]]; 484 tmp = (tmp * 2) % cw_len; 485 si[i] = alpha_to[tmp]; 486 } 487 } 488 } 489 490 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user) 491 { 492 s16 *lmu = user->lmu; 493 s16 *si = user->si; 494 s32 *mu = user->mu; 495 s32 *dmu = user->dmu; 496 s32 *delta = user->delta; 497 int degree = get_sectorsize(user) == 512 ? 13 : 14; 498 int cw_len = BIT(degree) - 1; 499 int strength = get_strength(user); 500 int num = 2 * strength + 1; 501 s16 *index_of = user->gf_tables->index_of; 502 s16 *alpha_to = user->gf_tables->alpha_to; 503 int i, j, k; 504 u32 dmu_0_count, tmp; 505 s16 *smu = user->smu; 506 507 /* index of largest delta */ 508 int ro; 509 int largest; 510 int diff; 511 512 dmu_0_count = 0; 513 514 /* First Row */ 515 516 /* Mu */ 517 mu[0] = -1; 518 519 memset(smu, 0, sizeof(s16) * num); 520 smu[0] = 1; 521 522 /* discrepancy set to 1 */ 523 dmu[0] = 1; 524 /* polynom order set to 0 */ 525 lmu[0] = 0; 526 delta[0] = (mu[0] * 2 - lmu[0]) >> 1; 527 528 /* Second Row */ 529 530 /* Mu */ 531 mu[1] = 0; 532 /* Sigma(x) set to 1 */ 533 memset(&smu[num], 0, sizeof(s16) * num); 534 smu[num] = 1; 535 536 /* discrepancy set to S1 */ 537 dmu[1] = si[1]; 538 539 /* polynom order set to 0 */ 540 lmu[1] = 0; 541 542 delta[1] = (mu[1] * 2 - lmu[1]) >> 1; 543 544 /* Init the Sigma(x) last row */ 545 memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num); 546 547 for (i = 1; i <= strength; i++) { 548 mu[i + 1] = i << 1; 549 /* Begin Computing Sigma (Mu+1) and L(mu) */ 550 /* check if discrepancy is set to 0 */ 551 if (dmu[i] == 0) { 552 dmu_0_count++; 553 554 tmp = ((strength - (lmu[i] >> 1) - 1) / 2); 555 if ((strength - (lmu[i] >> 1) - 1) & 0x1) 556 tmp += 2; 557 else 558 tmp += 1; 559 560 if (dmu_0_count == tmp) { 561 for (j = 0; j <= (lmu[i] >> 1) + 1; j++) 562 smu[(strength + 1) * num + j] = 563 smu[i * num + j]; 564 565 lmu[strength + 1] = lmu[i]; 566 return; 567 } 568 569 /* copy polynom */ 570 for (j = 0; j <= lmu[i] >> 1; j++) 571 smu[(i + 1) * num + j] = smu[i * num + j]; 572 573 /* copy previous polynom order to the next */ 574 lmu[i + 1] = lmu[i]; 575 } else { 576 ro = 0; 577 largest = -1; 578 /* find largest delta with dmu != 0 */ 579 for (j = 0; j < i; j++) { 580 if ((dmu[j]) && (delta[j] > largest)) { 581 largest = delta[j]; 582 ro = j; 583 } 584 } 585 586 /* compute difference */ 587 diff = (mu[i] - mu[ro]); 588 589 /* Compute degree of the new smu polynomial */ 590 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff)) 591 lmu[i + 1] = lmu[i]; 592 else 593 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2; 594 595 /* Init smu[i+1] with 0 */ 596 for (k = 0; k < num; k++) 597 smu[(i + 1) * num + k] = 0; 598 599 /* Compute smu[i+1] */ 600 for (k = 0; k <= lmu[ro] >> 1; k++) { 601 s16 a, b, c; 602 603 if (!(smu[ro * num + k] && dmu[i])) 604 continue; 605 606 a = index_of[dmu[i]]; 607 b = index_of[dmu[ro]]; 608 c = index_of[smu[ro * num + k]]; 609 tmp = a + (cw_len - b) + c; 610 a = alpha_to[tmp % cw_len]; 611 smu[(i + 1) * num + (k + diff)] = a; 612 } 613 614 for (k = 0; k <= lmu[i] >> 1; k++) 615 smu[(i + 1) * num + k] ^= smu[i * num + k]; 616 } 617 618 /* End Computing Sigma (Mu+1) and L(mu) */ 619 /* In either case compute delta */ 620 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1; 621 622 /* Do not compute discrepancy for the last iteration */ 623 if (i >= strength) 624 continue; 625 626 for (k = 0; k <= (lmu[i + 1] >> 1); k++) { 627 tmp = 2 * (i - 1); 628 if (k == 0) { 629 dmu[i + 1] = si[tmp + 3]; 630 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) { 631 s16 a, b, c; 632 633 a = index_of[smu[(i + 1) * num + k]]; 634 b = si[2 * (i - 1) + 3 - k]; 635 c = index_of[b]; 636 tmp = a + c; 637 tmp %= cw_len; 638 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1]; 639 } 640 } 641 } 642 } 643 644 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user) 645 { 646 int sector_size = get_sectorsize(user); 647 int degree = sector_size == 512 ? 13 : 14; 648 struct atmel_pmecc *pmecc = user->pmecc; 649 int strength = get_strength(user); 650 int ret, roots_nbr, i, err_nbr = 0; 651 int num = (2 * strength) + 1; 652 s16 *smu = user->smu; 653 u32 val; 654 655 writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS); 656 657 for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) { 658 writel_relaxed(smu[(strength + 1) * num + i], 659 pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i)); 660 err_nbr++; 661 } 662 663 val = (err_nbr - 1) << 16; 664 if (sector_size == 1024) 665 val |= 1; 666 667 writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG); 668 writel((sector_size * 8) + (degree * strength), 669 pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN); 670 671 ret = readl_relaxed_poll_timeout(pmecc->regs.errloc + 672 ATMEL_PMERRLOC_ELISR, 673 val, val & PMERRLOC_CALC_DONE, 0, 674 PMECC_MAX_TIMEOUT_MS * 1000); 675 if (ret) { 676 dev_err(pmecc->dev, 677 "PMECC: Timeout to calculate error location.\n"); 678 return ret; 679 } 680 681 roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8; 682 /* Number of roots == degree of smu hence <= cap */ 683 if (roots_nbr == user->lmu[strength + 1] >> 1) 684 return err_nbr - 1; 685 686 /* 687 * Number of roots does not match the degree of smu 688 * unable to correct error. 689 */ 690 return -EBADMSG; 691 } 692 693 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector, 694 void *data, void *ecc) 695 { 696 struct atmel_pmecc *pmecc = user->pmecc; 697 int sectorsize = get_sectorsize(user); 698 int eccbytes = user->eccbytes; 699 int i, nerrors; 700 701 if (!(user->isr & BIT(sector))) 702 return 0; 703 704 atmel_pmecc_gen_syndrome(user, sector); 705 atmel_pmecc_substitute(user); 706 atmel_pmecc_get_sigma(user); 707 708 nerrors = atmel_pmecc_err_location(user); 709 if (nerrors < 0) 710 return nerrors; 711 712 for (i = 0; i < nerrors; i++) { 713 const char *area; 714 int byte, bit; 715 u32 errpos; 716 u8 *ptr; 717 718 errpos = readl_relaxed(pmecc->regs.errloc + 719 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i)); 720 errpos--; 721 722 byte = errpos / 8; 723 bit = errpos % 8; 724 725 if (byte < sectorsize) { 726 ptr = data + byte; 727 area = "data"; 728 } else if (byte < sectorsize + eccbytes) { 729 ptr = ecc + byte - sectorsize; 730 area = "ECC"; 731 } else { 732 dev_dbg(pmecc->dev, 733 "Invalid errpos value (%d, max is %d)\n", 734 errpos, (sectorsize + eccbytes) * 8); 735 return -EINVAL; 736 } 737 738 dev_dbg(pmecc->dev, 739 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n", 740 area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit))); 741 742 *ptr ^= BIT(bit); 743 } 744 745 return nerrors; 746 } 747 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector); 748 749 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user) 750 { 751 return user->pmecc->caps->correct_erased_chunks; 752 } 753 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks); 754 755 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user, 756 int sector, void *ecc) 757 { 758 struct atmel_pmecc *pmecc = user->pmecc; 759 u8 *ptr = ecc; 760 int i; 761 762 for (i = 0; i < user->eccbytes; i++) 763 ptr[i] = readb_relaxed(pmecc->regs.base + 764 ATMEL_PMECC_ECC(sector, i)); 765 } 766 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes); 767 768 void atmel_pmecc_reset(struct atmel_pmecc *pmecc) 769 { 770 writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL); 771 writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 772 } 773 EXPORT_SYMBOL_GPL(atmel_pmecc_reset); 774 775 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op) 776 { 777 struct atmel_pmecc *pmecc = user->pmecc; 778 u32 cfg; 779 780 if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) { 781 dev_err(pmecc->dev, "Bad ECC operation!"); 782 return -EINVAL; 783 } 784 785 mutex_lock(&user->pmecc->lock); 786 787 cfg = user->cache.cfg; 788 if (op == NAND_ECC_WRITE) 789 cfg |= PMECC_CFG_WRITE_OP; 790 else 791 cfg |= PMECC_CFG_AUTO_ENABLE; 792 793 writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG); 794 writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA); 795 writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR); 796 writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR); 797 798 writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 799 writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL); 800 801 return 0; 802 } 803 EXPORT_SYMBOL_GPL(atmel_pmecc_enable); 804 805 void atmel_pmecc_disable(struct atmel_pmecc_user *user) 806 { 807 atmel_pmecc_reset(user->pmecc); 808 mutex_unlock(&user->pmecc->lock); 809 } 810 EXPORT_SYMBOL_GPL(atmel_pmecc_disable); 811 812 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user) 813 { 814 struct atmel_pmecc *pmecc = user->pmecc; 815 u32 status; 816 int ret; 817 818 ret = readl_relaxed_poll_timeout(pmecc->regs.base + 819 ATMEL_PMECC_SR, 820 status, !(status & PMECC_SR_BUSY), 0, 821 PMECC_MAX_TIMEOUT_MS * 1000); 822 if (ret) { 823 dev_err(pmecc->dev, 824 "Timeout while waiting for PMECC ready.\n"); 825 return ret; 826 } 827 828 user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR); 829 830 return 0; 831 } 832 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy); 833 834 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev, 835 const struct atmel_pmecc_caps *caps, 836 int pmecc_res_idx, int errloc_res_idx) 837 { 838 struct device *dev = &pdev->dev; 839 struct atmel_pmecc *pmecc; 840 struct resource *res; 841 842 pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL); 843 if (!pmecc) 844 return ERR_PTR(-ENOMEM); 845 846 pmecc->caps = caps; 847 pmecc->dev = dev; 848 mutex_init(&pmecc->lock); 849 850 res = platform_get_resource(pdev, IORESOURCE_MEM, pmecc_res_idx); 851 pmecc->regs.base = devm_ioremap_resource(dev, res); 852 if (IS_ERR(pmecc->regs.base)) 853 return ERR_CAST(pmecc->regs.base); 854 855 res = platform_get_resource(pdev, IORESOURCE_MEM, errloc_res_idx); 856 pmecc->regs.errloc = devm_ioremap_resource(dev, res); 857 if (IS_ERR(pmecc->regs.errloc)) 858 return ERR_CAST(pmecc->regs.errloc); 859 860 /* Disable all interrupts before registering the PMECC handler. */ 861 writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR); 862 atmel_pmecc_reset(pmecc); 863 864 return pmecc; 865 } 866 867 static void devm_atmel_pmecc_put(struct device *dev, void *res) 868 { 869 struct atmel_pmecc **pmecc = res; 870 871 put_device((*pmecc)->dev); 872 } 873 874 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev, 875 struct device_node *np) 876 { 877 struct platform_device *pdev; 878 struct atmel_pmecc *pmecc, **ptr; 879 880 pdev = of_find_device_by_node(np); 881 if (!pdev || !platform_get_drvdata(pdev)) 882 return ERR_PTR(-EPROBE_DEFER); 883 884 ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL); 885 if (!ptr) 886 return ERR_PTR(-ENOMEM); 887 888 get_device(&pdev->dev); 889 pmecc = platform_get_drvdata(pdev); 890 891 *ptr = pmecc; 892 893 devres_add(userdev, ptr); 894 895 return pmecc; 896 } 897 898 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 }; 899 900 static struct atmel_pmecc_caps at91sam9g45_caps = { 901 .strengths = atmel_pmecc_strengths, 902 .nstrengths = 5, 903 .el_offset = 0x8c, 904 }; 905 906 static struct atmel_pmecc_caps sama5d4_caps = { 907 .strengths = atmel_pmecc_strengths, 908 .nstrengths = 5, 909 .el_offset = 0x8c, 910 .correct_erased_chunks = true, 911 }; 912 913 static struct atmel_pmecc_caps sama5d2_caps = { 914 .strengths = atmel_pmecc_strengths, 915 .nstrengths = 6, 916 .el_offset = 0xac, 917 .correct_erased_chunks = true, 918 }; 919 920 static const struct of_device_id atmel_pmecc_legacy_match[] = { 921 { .compatible = "atmel,sama5d4-nand", &sama5d4_caps }, 922 { .compatible = "atmel,sama5d2-nand", &sama5d2_caps }, 923 { /* sentinel */ } 924 }; 925 926 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev) 927 { 928 struct atmel_pmecc *pmecc; 929 struct device_node *np; 930 931 if (!userdev) 932 return ERR_PTR(-EINVAL); 933 934 if (!userdev->of_node) 935 return NULL; 936 937 np = of_parse_phandle(userdev->of_node, "ecc-engine", 0); 938 if (np) { 939 pmecc = atmel_pmecc_get_by_node(userdev, np); 940 of_node_put(np); 941 } else { 942 /* 943 * Support old DT bindings: in this case the PMECC iomem 944 * resources are directly defined in the user pdev at position 945 * 1 and 2. Extract all relevant information from there. 946 */ 947 struct platform_device *pdev = to_platform_device(userdev); 948 const struct atmel_pmecc_caps *caps; 949 const struct of_device_id *match; 950 951 /* No PMECC engine available. */ 952 if (!of_property_read_bool(userdev->of_node, 953 "atmel,has-pmecc")) 954 return NULL; 955 956 caps = &at91sam9g45_caps; 957 958 /* Find the caps associated to the NAND dev node. */ 959 match = of_match_node(atmel_pmecc_legacy_match, 960 userdev->of_node); 961 if (match && match->data) 962 caps = match->data; 963 964 pmecc = atmel_pmecc_create(pdev, caps, 1, 2); 965 } 966 967 return pmecc; 968 } 969 EXPORT_SYMBOL(devm_atmel_pmecc_get); 970 971 static const struct of_device_id atmel_pmecc_match[] = { 972 { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps }, 973 { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps }, 974 { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps }, 975 { /* sentinel */ } 976 }; 977 MODULE_DEVICE_TABLE(of, atmel_pmecc_match); 978 979 static int atmel_pmecc_probe(struct platform_device *pdev) 980 { 981 struct device *dev = &pdev->dev; 982 const struct atmel_pmecc_caps *caps; 983 struct atmel_pmecc *pmecc; 984 985 caps = of_device_get_match_data(&pdev->dev); 986 if (!caps) { 987 dev_err(dev, "Invalid caps\n"); 988 return -EINVAL; 989 } 990 991 pmecc = atmel_pmecc_create(pdev, caps, 0, 1); 992 if (IS_ERR(pmecc)) 993 return PTR_ERR(pmecc); 994 995 platform_set_drvdata(pdev, pmecc); 996 997 return 0; 998 } 999 1000 static struct platform_driver atmel_pmecc_driver = { 1001 .driver = { 1002 .name = "atmel-pmecc", 1003 .of_match_table = of_match_ptr(atmel_pmecc_match), 1004 }, 1005 .probe = atmel_pmecc_probe, 1006 }; 1007 module_platform_driver(atmel_pmecc_driver); 1008 1009 MODULE_LICENSE("GPL"); 1010 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>"); 1011 MODULE_DESCRIPTION("PMECC engine driver"); 1012 MODULE_ALIAS("platform:atmel_pmecc"); 1013