1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2017 ATMEL 4 * Copyright 2017 Free Electrons 5 * 6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 7 * 8 * Derived from the atmel_nand.c driver which contained the following 9 * copyrights: 10 * 11 * Copyright 2003 Rick Bronson 12 * 13 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8) 14 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de) 15 * 16 * Derived from drivers/mtd/spia.c (removed in v3.8) 17 * Copyright 2000 Steven J. Hill (sjhill@cotw.com) 18 * 19 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263 20 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007 21 * 22 * Derived from Das U-Boot source code 23 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c) 24 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas 25 * 26 * Add Programmable Multibit ECC support for various AT91 SoC 27 * Copyright 2012 ATMEL, Hong Xu 28 * 29 * Add Nand Flash Controller support for SAMA5 SoC 30 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com) 31 * 32 * The PMECC is an hardware assisted BCH engine, which means part of the 33 * ECC algorithm is left to the software. The hardware/software repartition 34 * is explained in the "PMECC Controller Functional Description" chapter in 35 * Atmel datasheets, and some of the functions in this file are directly 36 * implementing the algorithms described in the "Software Implementation" 37 * sub-section. 38 * 39 * TODO: it seems that the software BCH implementation in lib/bch.c is already 40 * providing some of the logic we are implementing here. It would be smart 41 * to expose the needed lib/bch.c helpers/functions and re-use them here. 42 */ 43 44 #include <linux/genalloc.h> 45 #include <linux/iopoll.h> 46 #include <linux/module.h> 47 #include <linux/mtd/rawnand.h> 48 #include <linux/of_irq.h> 49 #include <linux/of_platform.h> 50 #include <linux/platform_device.h> 51 #include <linux/slab.h> 52 53 #include "pmecc.h" 54 55 /* Galois field dimension */ 56 #define PMECC_GF_DIMENSION_13 13 57 #define PMECC_GF_DIMENSION_14 14 58 59 /* Primitive Polynomial used by PMECC */ 60 #define PMECC_GF_13_PRIMITIVE_POLY 0x201b 61 #define PMECC_GF_14_PRIMITIVE_POLY 0x4443 62 63 #define PMECC_LOOKUP_TABLE_SIZE_512 0x2000 64 #define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000 65 66 /* Time out value for reading PMECC status register */ 67 #define PMECC_MAX_TIMEOUT_MS 100 68 69 /* PMECC Register Definitions */ 70 #define ATMEL_PMECC_CFG 0x0 71 #define PMECC_CFG_BCH_STRENGTH(x) (x) 72 #define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0) 73 #define PMECC_CFG_SECTOR512 (0 << 4) 74 #define PMECC_CFG_SECTOR1024 (1 << 4) 75 #define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8) 76 #define PMECC_CFG_READ_OP (0 << 12) 77 #define PMECC_CFG_WRITE_OP (1 << 12) 78 #define PMECC_CFG_SPARE_ENABLE BIT(16) 79 #define PMECC_CFG_AUTO_ENABLE BIT(20) 80 81 #define ATMEL_PMECC_SAREA 0x4 82 #define ATMEL_PMECC_SADDR 0x8 83 #define ATMEL_PMECC_EADDR 0xc 84 85 #define ATMEL_PMECC_CLK 0x10 86 #define PMECC_CLK_133MHZ (2 << 0) 87 88 #define ATMEL_PMECC_CTRL 0x14 89 #define PMECC_CTRL_RST BIT(0) 90 #define PMECC_CTRL_DATA BIT(1) 91 #define PMECC_CTRL_USER BIT(2) 92 #define PMECC_CTRL_ENABLE BIT(4) 93 #define PMECC_CTRL_DISABLE BIT(5) 94 95 #define ATMEL_PMECC_SR 0x18 96 #define PMECC_SR_BUSY BIT(0) 97 #define PMECC_SR_ENABLE BIT(4) 98 99 #define ATMEL_PMECC_IER 0x1c 100 #define ATMEL_PMECC_IDR 0x20 101 #define ATMEL_PMECC_IMR 0x24 102 #define ATMEL_PMECC_ISR 0x28 103 #define PMECC_ERROR_INT BIT(0) 104 105 #define ATMEL_PMECC_ECC(sector, n) \ 106 ((((sector) + 1) * 0x40) + (n)) 107 108 #define ATMEL_PMECC_REM(sector, n) \ 109 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200) 110 111 /* PMERRLOC Register Definitions */ 112 #define ATMEL_PMERRLOC_ELCFG 0x0 113 #define PMERRLOC_ELCFG_SECTOR_512 (0 << 0) 114 #define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0) 115 #define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16) 116 117 #define ATMEL_PMERRLOC_ELPRIM 0x4 118 #define ATMEL_PMERRLOC_ELEN 0x8 119 #define ATMEL_PMERRLOC_ELDIS 0xc 120 #define PMERRLOC_DISABLE BIT(0) 121 122 #define ATMEL_PMERRLOC_ELSR 0x10 123 #define PMERRLOC_ELSR_BUSY BIT(0) 124 125 #define ATMEL_PMERRLOC_ELIER 0x14 126 #define ATMEL_PMERRLOC_ELIDR 0x18 127 #define ATMEL_PMERRLOC_ELIMR 0x1c 128 #define ATMEL_PMERRLOC_ELISR 0x20 129 #define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8) 130 #define PMERRLOC_CALC_DONE BIT(0) 131 132 #define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28) 133 134 #define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs)) 135 136 struct atmel_pmecc_gf_tables { 137 u16 *alpha_to; 138 u16 *index_of; 139 }; 140 141 struct atmel_pmecc_caps { 142 const int *strengths; 143 int nstrengths; 144 int el_offset; 145 bool correct_erased_chunks; 146 }; 147 148 struct atmel_pmecc { 149 struct device *dev; 150 const struct atmel_pmecc_caps *caps; 151 152 struct { 153 void __iomem *base; 154 void __iomem *errloc; 155 } regs; 156 157 struct mutex lock; 158 }; 159 160 struct atmel_pmecc_user_conf_cache { 161 u32 cfg; 162 u32 sarea; 163 u32 saddr; 164 u32 eaddr; 165 }; 166 167 struct atmel_pmecc_user { 168 struct atmel_pmecc_user_conf_cache cache; 169 struct atmel_pmecc *pmecc; 170 const struct atmel_pmecc_gf_tables *gf_tables; 171 int eccbytes; 172 s16 *partial_syn; 173 s16 *si; 174 s16 *lmu; 175 s16 *smu; 176 s32 *mu; 177 s32 *dmu; 178 s32 *delta; 179 u32 isr; 180 }; 181 182 static DEFINE_MUTEX(pmecc_gf_tables_lock); 183 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512; 184 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024; 185 186 static inline int deg(unsigned int poly) 187 { 188 /* polynomial degree is the most-significant bit index */ 189 return fls(poly) - 1; 190 } 191 192 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly, 193 struct atmel_pmecc_gf_tables *gf_tables) 194 { 195 unsigned int i, x = 1; 196 const unsigned int k = BIT(deg(poly)); 197 unsigned int nn = BIT(mm) - 1; 198 199 /* primitive polynomial must be of degree m */ 200 if (k != (1u << mm)) 201 return -EINVAL; 202 203 for (i = 0; i < nn; i++) { 204 gf_tables->alpha_to[i] = x; 205 gf_tables->index_of[x] = i; 206 if (i && (x == 1)) 207 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */ 208 return -EINVAL; 209 x <<= 1; 210 if (x & k) 211 x ^= poly; 212 } 213 gf_tables->alpha_to[nn] = 1; 214 gf_tables->index_of[0] = 0; 215 216 return 0; 217 } 218 219 static const struct atmel_pmecc_gf_tables * 220 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req) 221 { 222 struct atmel_pmecc_gf_tables *gf_tables; 223 unsigned int poly, degree, table_size; 224 int ret; 225 226 if (req->ecc.sectorsize == 512) { 227 degree = PMECC_GF_DIMENSION_13; 228 poly = PMECC_GF_13_PRIMITIVE_POLY; 229 table_size = PMECC_LOOKUP_TABLE_SIZE_512; 230 } else { 231 degree = PMECC_GF_DIMENSION_14; 232 poly = PMECC_GF_14_PRIMITIVE_POLY; 233 table_size = PMECC_LOOKUP_TABLE_SIZE_1024; 234 } 235 236 gf_tables = kzalloc(sizeof(*gf_tables) + 237 (2 * table_size * sizeof(u16)), 238 GFP_KERNEL); 239 if (!gf_tables) 240 return ERR_PTR(-ENOMEM); 241 242 gf_tables->alpha_to = (void *)(gf_tables + 1); 243 gf_tables->index_of = gf_tables->alpha_to + table_size; 244 245 ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables); 246 if (ret) { 247 kfree(gf_tables); 248 return ERR_PTR(ret); 249 } 250 251 return gf_tables; 252 } 253 254 static const struct atmel_pmecc_gf_tables * 255 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req) 256 { 257 const struct atmel_pmecc_gf_tables **gf_tables, *ret; 258 259 mutex_lock(&pmecc_gf_tables_lock); 260 if (req->ecc.sectorsize == 512) 261 gf_tables = &pmecc_gf_tables_512; 262 else 263 gf_tables = &pmecc_gf_tables_1024; 264 265 ret = *gf_tables; 266 267 if (!ret) { 268 ret = atmel_pmecc_create_gf_tables(req); 269 if (!IS_ERR(ret)) 270 *gf_tables = ret; 271 } 272 mutex_unlock(&pmecc_gf_tables_lock); 273 274 return ret; 275 } 276 277 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc, 278 struct atmel_pmecc_user_req *req) 279 { 280 int i, max_eccbytes, eccbytes = 0, eccstrength = 0; 281 282 if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0) 283 return -EINVAL; 284 285 if (req->ecc.ooboffset >= 0 && 286 req->ecc.ooboffset + req->ecc.bytes > req->oobsize) 287 return -EINVAL; 288 289 if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) { 290 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 291 return -EINVAL; 292 293 if (req->pagesize > 512) 294 req->ecc.sectorsize = 1024; 295 else 296 req->ecc.sectorsize = 512; 297 } 298 299 if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024) 300 return -EINVAL; 301 302 if (req->pagesize % req->ecc.sectorsize) 303 return -EINVAL; 304 305 req->ecc.nsectors = req->pagesize / req->ecc.sectorsize; 306 307 max_eccbytes = req->ecc.bytes; 308 309 for (i = 0; i < pmecc->caps->nstrengths; i++) { 310 int nbytes, strength = pmecc->caps->strengths[i]; 311 312 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH && 313 strength < req->ecc.strength) 314 continue; 315 316 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize), 317 8); 318 nbytes *= req->ecc.nsectors; 319 320 if (nbytes > max_eccbytes) 321 break; 322 323 eccstrength = strength; 324 eccbytes = nbytes; 325 326 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH) 327 break; 328 } 329 330 if (!eccstrength) 331 return -EINVAL; 332 333 req->ecc.bytes = eccbytes; 334 req->ecc.strength = eccstrength; 335 336 if (req->ecc.ooboffset < 0) 337 req->ecc.ooboffset = req->oobsize - eccbytes; 338 339 return 0; 340 } 341 342 struct atmel_pmecc_user * 343 atmel_pmecc_create_user(struct atmel_pmecc *pmecc, 344 struct atmel_pmecc_user_req *req) 345 { 346 struct atmel_pmecc_user *user; 347 const struct atmel_pmecc_gf_tables *gf_tables; 348 int strength, size, ret; 349 350 ret = atmel_pmecc_prepare_user_req(pmecc, req); 351 if (ret) 352 return ERR_PTR(ret); 353 354 size = sizeof(*user); 355 size = ALIGN(size, sizeof(u16)); 356 /* Reserve space for partial_syn, si and smu */ 357 size += ((2 * req->ecc.strength) + 1) * sizeof(u16) * 358 (2 + req->ecc.strength + 2); 359 /* Reserve space for lmu. */ 360 size += (req->ecc.strength + 1) * sizeof(u16); 361 /* Reserve space for mu, dmu and delta. */ 362 size = ALIGN(size, sizeof(s32)); 363 size += (req->ecc.strength + 1) * sizeof(s32) * 3; 364 365 user = devm_kzalloc(pmecc->dev, size, GFP_KERNEL); 366 if (!user) 367 return ERR_PTR(-ENOMEM); 368 369 user->pmecc = pmecc; 370 371 user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16)); 372 user->si = user->partial_syn + ((2 * req->ecc.strength) + 1); 373 user->lmu = user->si + ((2 * req->ecc.strength) + 1); 374 user->smu = user->lmu + (req->ecc.strength + 1); 375 user->mu = (s32 *)PTR_ALIGN(user->smu + 376 (((2 * req->ecc.strength) + 1) * 377 (req->ecc.strength + 2)), 378 sizeof(s32)); 379 user->dmu = user->mu + req->ecc.strength + 1; 380 user->delta = user->dmu + req->ecc.strength + 1; 381 382 gf_tables = atmel_pmecc_get_gf_tables(req); 383 if (IS_ERR(gf_tables)) { 384 kfree(user); 385 return ERR_CAST(gf_tables); 386 } 387 388 user->gf_tables = gf_tables; 389 390 user->eccbytes = req->ecc.bytes / req->ecc.nsectors; 391 392 for (strength = 0; strength < pmecc->caps->nstrengths; strength++) { 393 if (pmecc->caps->strengths[strength] == req->ecc.strength) 394 break; 395 } 396 397 user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) | 398 PMECC_CFG_NSECTORS(req->ecc.nsectors); 399 400 if (req->ecc.sectorsize == 1024) 401 user->cache.cfg |= PMECC_CFG_SECTOR1024; 402 403 user->cache.sarea = req->oobsize - 1; 404 user->cache.saddr = req->ecc.ooboffset; 405 user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1; 406 407 return user; 408 } 409 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user); 410 411 static int get_strength(struct atmel_pmecc_user *user) 412 { 413 const int *strengths = user->pmecc->caps->strengths; 414 415 return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK]; 416 } 417 418 static int get_sectorsize(struct atmel_pmecc_user *user) 419 { 420 return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512; 421 } 422 423 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector) 424 { 425 int strength = get_strength(user); 426 u32 value; 427 int i; 428 429 /* Fill odd syndromes */ 430 for (i = 0; i < strength; i++) { 431 value = readl_relaxed(user->pmecc->regs.base + 432 ATMEL_PMECC_REM(sector, i / 2)); 433 if (i & 1) 434 value >>= 16; 435 436 user->partial_syn[(2 * i) + 1] = value; 437 } 438 } 439 440 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user) 441 { 442 int degree = get_sectorsize(user) == 512 ? 13 : 14; 443 int cw_len = BIT(degree) - 1; 444 int strength = get_strength(user); 445 s16 *alpha_to = user->gf_tables->alpha_to; 446 s16 *index_of = user->gf_tables->index_of; 447 s16 *partial_syn = user->partial_syn; 448 s16 *si; 449 int i, j; 450 451 /* 452 * si[] is a table that holds the current syndrome value, 453 * an element of that table belongs to the field 454 */ 455 si = user->si; 456 457 memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1)); 458 459 /* Computation 2t syndromes based on S(x) */ 460 /* Odd syndromes */ 461 for (i = 1; i < 2 * strength; i += 2) { 462 for (j = 0; j < degree; j++) { 463 if (partial_syn[i] & BIT(j)) 464 si[i] = alpha_to[i * j] ^ si[i]; 465 } 466 } 467 /* Even syndrome = (Odd syndrome) ** 2 */ 468 for (i = 2, j = 1; j <= strength; i = ++j << 1) { 469 if (si[j] == 0) { 470 si[i] = 0; 471 } else { 472 s16 tmp; 473 474 tmp = index_of[si[j]]; 475 tmp = (tmp * 2) % cw_len; 476 si[i] = alpha_to[tmp]; 477 } 478 } 479 } 480 481 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user) 482 { 483 s16 *lmu = user->lmu; 484 s16 *si = user->si; 485 s32 *mu = user->mu; 486 s32 *dmu = user->dmu; 487 s32 *delta = user->delta; 488 int degree = get_sectorsize(user) == 512 ? 13 : 14; 489 int cw_len = BIT(degree) - 1; 490 int strength = get_strength(user); 491 int num = 2 * strength + 1; 492 s16 *index_of = user->gf_tables->index_of; 493 s16 *alpha_to = user->gf_tables->alpha_to; 494 int i, j, k; 495 u32 dmu_0_count, tmp; 496 s16 *smu = user->smu; 497 498 /* index of largest delta */ 499 int ro; 500 int largest; 501 int diff; 502 503 dmu_0_count = 0; 504 505 /* First Row */ 506 507 /* Mu */ 508 mu[0] = -1; 509 510 memset(smu, 0, sizeof(s16) * num); 511 smu[0] = 1; 512 513 /* discrepancy set to 1 */ 514 dmu[0] = 1; 515 /* polynom order set to 0 */ 516 lmu[0] = 0; 517 delta[0] = (mu[0] * 2 - lmu[0]) >> 1; 518 519 /* Second Row */ 520 521 /* Mu */ 522 mu[1] = 0; 523 /* Sigma(x) set to 1 */ 524 memset(&smu[num], 0, sizeof(s16) * num); 525 smu[num] = 1; 526 527 /* discrepancy set to S1 */ 528 dmu[1] = si[1]; 529 530 /* polynom order set to 0 */ 531 lmu[1] = 0; 532 533 delta[1] = (mu[1] * 2 - lmu[1]) >> 1; 534 535 /* Init the Sigma(x) last row */ 536 memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num); 537 538 for (i = 1; i <= strength; i++) { 539 mu[i + 1] = i << 1; 540 /* Begin Computing Sigma (Mu+1) and L(mu) */ 541 /* check if discrepancy is set to 0 */ 542 if (dmu[i] == 0) { 543 dmu_0_count++; 544 545 tmp = ((strength - (lmu[i] >> 1) - 1) / 2); 546 if ((strength - (lmu[i] >> 1) - 1) & 0x1) 547 tmp += 2; 548 else 549 tmp += 1; 550 551 if (dmu_0_count == tmp) { 552 for (j = 0; j <= (lmu[i] >> 1) + 1; j++) 553 smu[(strength + 1) * num + j] = 554 smu[i * num + j]; 555 556 lmu[strength + 1] = lmu[i]; 557 return; 558 } 559 560 /* copy polynom */ 561 for (j = 0; j <= lmu[i] >> 1; j++) 562 smu[(i + 1) * num + j] = smu[i * num + j]; 563 564 /* copy previous polynom order to the next */ 565 lmu[i + 1] = lmu[i]; 566 } else { 567 ro = 0; 568 largest = -1; 569 /* find largest delta with dmu != 0 */ 570 for (j = 0; j < i; j++) { 571 if ((dmu[j]) && (delta[j] > largest)) { 572 largest = delta[j]; 573 ro = j; 574 } 575 } 576 577 /* compute difference */ 578 diff = (mu[i] - mu[ro]); 579 580 /* Compute degree of the new smu polynomial */ 581 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff)) 582 lmu[i + 1] = lmu[i]; 583 else 584 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2; 585 586 /* Init smu[i+1] with 0 */ 587 for (k = 0; k < num; k++) 588 smu[(i + 1) * num + k] = 0; 589 590 /* Compute smu[i+1] */ 591 for (k = 0; k <= lmu[ro] >> 1; k++) { 592 s16 a, b, c; 593 594 if (!(smu[ro * num + k] && dmu[i])) 595 continue; 596 597 a = index_of[dmu[i]]; 598 b = index_of[dmu[ro]]; 599 c = index_of[smu[ro * num + k]]; 600 tmp = a + (cw_len - b) + c; 601 a = alpha_to[tmp % cw_len]; 602 smu[(i + 1) * num + (k + diff)] = a; 603 } 604 605 for (k = 0; k <= lmu[i] >> 1; k++) 606 smu[(i + 1) * num + k] ^= smu[i * num + k]; 607 } 608 609 /* End Computing Sigma (Mu+1) and L(mu) */ 610 /* In either case compute delta */ 611 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1; 612 613 /* Do not compute discrepancy for the last iteration */ 614 if (i >= strength) 615 continue; 616 617 for (k = 0; k <= (lmu[i + 1] >> 1); k++) { 618 tmp = 2 * (i - 1); 619 if (k == 0) { 620 dmu[i + 1] = si[tmp + 3]; 621 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) { 622 s16 a, b, c; 623 624 a = index_of[smu[(i + 1) * num + k]]; 625 b = si[2 * (i - 1) + 3 - k]; 626 c = index_of[b]; 627 tmp = a + c; 628 tmp %= cw_len; 629 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1]; 630 } 631 } 632 } 633 } 634 635 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user) 636 { 637 int sector_size = get_sectorsize(user); 638 int degree = sector_size == 512 ? 13 : 14; 639 struct atmel_pmecc *pmecc = user->pmecc; 640 int strength = get_strength(user); 641 int ret, roots_nbr, i, err_nbr = 0; 642 int num = (2 * strength) + 1; 643 s16 *smu = user->smu; 644 u32 val; 645 646 writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS); 647 648 for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) { 649 writel_relaxed(smu[(strength + 1) * num + i], 650 pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i)); 651 err_nbr++; 652 } 653 654 val = (err_nbr - 1) << 16; 655 if (sector_size == 1024) 656 val |= 1; 657 658 writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG); 659 writel((sector_size * 8) + (degree * strength), 660 pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN); 661 662 ret = readl_relaxed_poll_timeout(pmecc->regs.errloc + 663 ATMEL_PMERRLOC_ELISR, 664 val, val & PMERRLOC_CALC_DONE, 0, 665 PMECC_MAX_TIMEOUT_MS * 1000); 666 if (ret) { 667 dev_err(pmecc->dev, 668 "PMECC: Timeout to calculate error location.\n"); 669 return ret; 670 } 671 672 roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8; 673 /* Number of roots == degree of smu hence <= cap */ 674 if (roots_nbr == user->lmu[strength + 1] >> 1) 675 return err_nbr - 1; 676 677 /* 678 * Number of roots does not match the degree of smu 679 * unable to correct error. 680 */ 681 return -EBADMSG; 682 } 683 684 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector, 685 void *data, void *ecc) 686 { 687 struct atmel_pmecc *pmecc = user->pmecc; 688 int sectorsize = get_sectorsize(user); 689 int eccbytes = user->eccbytes; 690 int i, nerrors; 691 692 if (!(user->isr & BIT(sector))) 693 return 0; 694 695 atmel_pmecc_gen_syndrome(user, sector); 696 atmel_pmecc_substitute(user); 697 atmel_pmecc_get_sigma(user); 698 699 nerrors = atmel_pmecc_err_location(user); 700 if (nerrors < 0) 701 return nerrors; 702 703 for (i = 0; i < nerrors; i++) { 704 const char *area; 705 int byte, bit; 706 u32 errpos; 707 u8 *ptr; 708 709 errpos = readl_relaxed(pmecc->regs.errloc + 710 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i)); 711 errpos--; 712 713 byte = errpos / 8; 714 bit = errpos % 8; 715 716 if (byte < sectorsize) { 717 ptr = data + byte; 718 area = "data"; 719 } else if (byte < sectorsize + eccbytes) { 720 ptr = ecc + byte - sectorsize; 721 area = "ECC"; 722 } else { 723 dev_dbg(pmecc->dev, 724 "Invalid errpos value (%d, max is %d)\n", 725 errpos, (sectorsize + eccbytes) * 8); 726 return -EINVAL; 727 } 728 729 dev_dbg(pmecc->dev, 730 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n", 731 area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit))); 732 733 *ptr ^= BIT(bit); 734 } 735 736 return nerrors; 737 } 738 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector); 739 740 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user) 741 { 742 return user->pmecc->caps->correct_erased_chunks; 743 } 744 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks); 745 746 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user, 747 int sector, void *ecc) 748 { 749 struct atmel_pmecc *pmecc = user->pmecc; 750 u8 *ptr = ecc; 751 int i; 752 753 for (i = 0; i < user->eccbytes; i++) 754 ptr[i] = readb_relaxed(pmecc->regs.base + 755 ATMEL_PMECC_ECC(sector, i)); 756 } 757 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes); 758 759 void atmel_pmecc_reset(struct atmel_pmecc *pmecc) 760 { 761 writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL); 762 writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 763 } 764 EXPORT_SYMBOL_GPL(atmel_pmecc_reset); 765 766 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op) 767 { 768 struct atmel_pmecc *pmecc = user->pmecc; 769 u32 cfg; 770 771 if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) { 772 dev_err(pmecc->dev, "Bad ECC operation!"); 773 return -EINVAL; 774 } 775 776 mutex_lock(&user->pmecc->lock); 777 778 cfg = user->cache.cfg; 779 if (op == NAND_ECC_WRITE) 780 cfg |= PMECC_CFG_WRITE_OP; 781 else 782 cfg |= PMECC_CFG_AUTO_ENABLE; 783 784 writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG); 785 writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA); 786 writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR); 787 writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR); 788 789 writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL); 790 writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL); 791 792 return 0; 793 } 794 EXPORT_SYMBOL_GPL(atmel_pmecc_enable); 795 796 void atmel_pmecc_disable(struct atmel_pmecc_user *user) 797 { 798 atmel_pmecc_reset(user->pmecc); 799 mutex_unlock(&user->pmecc->lock); 800 } 801 EXPORT_SYMBOL_GPL(atmel_pmecc_disable); 802 803 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user) 804 { 805 struct atmel_pmecc *pmecc = user->pmecc; 806 u32 status; 807 int ret; 808 809 ret = readl_relaxed_poll_timeout(pmecc->regs.base + 810 ATMEL_PMECC_SR, 811 status, !(status & PMECC_SR_BUSY), 0, 812 PMECC_MAX_TIMEOUT_MS * 1000); 813 if (ret) { 814 dev_err(pmecc->dev, 815 "Timeout while waiting for PMECC ready.\n"); 816 return ret; 817 } 818 819 user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR); 820 821 return 0; 822 } 823 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy); 824 825 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev, 826 const struct atmel_pmecc_caps *caps, 827 int pmecc_res_idx, int errloc_res_idx) 828 { 829 struct device *dev = &pdev->dev; 830 struct atmel_pmecc *pmecc; 831 832 pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL); 833 if (!pmecc) 834 return ERR_PTR(-ENOMEM); 835 836 pmecc->caps = caps; 837 pmecc->dev = dev; 838 mutex_init(&pmecc->lock); 839 840 pmecc->regs.base = devm_platform_ioremap_resource(pdev, pmecc_res_idx); 841 if (IS_ERR(pmecc->regs.base)) 842 return ERR_CAST(pmecc->regs.base); 843 844 pmecc->regs.errloc = devm_platform_ioremap_resource(pdev, errloc_res_idx); 845 if (IS_ERR(pmecc->regs.errloc)) 846 return ERR_CAST(pmecc->regs.errloc); 847 848 /* Disable all interrupts before registering the PMECC handler. */ 849 writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR); 850 atmel_pmecc_reset(pmecc); 851 852 return pmecc; 853 } 854 855 static void devm_atmel_pmecc_put(struct device *dev, void *res) 856 { 857 struct atmel_pmecc **pmecc = res; 858 859 put_device((*pmecc)->dev); 860 } 861 862 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev, 863 struct device_node *np) 864 { 865 struct platform_device *pdev; 866 struct atmel_pmecc *pmecc, **ptr; 867 int ret; 868 869 pdev = of_find_device_by_node(np); 870 if (!pdev) 871 return ERR_PTR(-EPROBE_DEFER); 872 pmecc = platform_get_drvdata(pdev); 873 if (!pmecc) { 874 ret = -EPROBE_DEFER; 875 goto err_put_device; 876 } 877 878 ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL); 879 if (!ptr) { 880 ret = -ENOMEM; 881 goto err_put_device; 882 } 883 884 *ptr = pmecc; 885 886 devres_add(userdev, ptr); 887 888 return pmecc; 889 890 err_put_device: 891 put_device(&pdev->dev); 892 return ERR_PTR(ret); 893 } 894 895 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 }; 896 897 static struct atmel_pmecc_caps at91sam9g45_caps = { 898 .strengths = atmel_pmecc_strengths, 899 .nstrengths = 5, 900 .el_offset = 0x8c, 901 }; 902 903 static struct atmel_pmecc_caps sama5d4_caps = { 904 .strengths = atmel_pmecc_strengths, 905 .nstrengths = 5, 906 .el_offset = 0x8c, 907 .correct_erased_chunks = true, 908 }; 909 910 static struct atmel_pmecc_caps sama5d2_caps = { 911 .strengths = atmel_pmecc_strengths, 912 .nstrengths = 6, 913 .el_offset = 0xac, 914 .correct_erased_chunks = true, 915 }; 916 917 static const struct of_device_id __maybe_unused atmel_pmecc_legacy_match[] = { 918 { .compatible = "atmel,sama5d4-nand", &sama5d4_caps }, 919 { .compatible = "atmel,sama5d2-nand", &sama5d2_caps }, 920 { /* sentinel */ } 921 }; 922 923 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev) 924 { 925 struct atmel_pmecc *pmecc; 926 struct device_node *np; 927 928 if (!userdev) 929 return ERR_PTR(-EINVAL); 930 931 if (!userdev->of_node) 932 return NULL; 933 934 np = of_parse_phandle(userdev->of_node, "ecc-engine", 0); 935 if (np) { 936 pmecc = atmel_pmecc_get_by_node(userdev, np); 937 of_node_put(np); 938 } else { 939 /* 940 * Support old DT bindings: in this case the PMECC iomem 941 * resources are directly defined in the user pdev at position 942 * 1 and 2. Extract all relevant information from there. 943 */ 944 struct platform_device *pdev = to_platform_device(userdev); 945 const struct atmel_pmecc_caps *caps; 946 const struct of_device_id *match; 947 948 /* No PMECC engine available. */ 949 if (!of_property_read_bool(userdev->of_node, 950 "atmel,has-pmecc")) 951 return NULL; 952 953 caps = &at91sam9g45_caps; 954 955 /* Find the caps associated to the NAND dev node. */ 956 match = of_match_node(atmel_pmecc_legacy_match, 957 userdev->of_node); 958 if (match && match->data) 959 caps = match->data; 960 961 pmecc = atmel_pmecc_create(pdev, caps, 1, 2); 962 } 963 964 return pmecc; 965 } 966 EXPORT_SYMBOL(devm_atmel_pmecc_get); 967 968 static const struct of_device_id atmel_pmecc_match[] = { 969 { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps }, 970 { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps }, 971 { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps }, 972 { /* sentinel */ } 973 }; 974 MODULE_DEVICE_TABLE(of, atmel_pmecc_match); 975 976 static int atmel_pmecc_probe(struct platform_device *pdev) 977 { 978 struct device *dev = &pdev->dev; 979 const struct atmel_pmecc_caps *caps; 980 struct atmel_pmecc *pmecc; 981 982 caps = of_device_get_match_data(&pdev->dev); 983 if (!caps) { 984 dev_err(dev, "Invalid caps\n"); 985 return -EINVAL; 986 } 987 988 pmecc = atmel_pmecc_create(pdev, caps, 0, 1); 989 if (IS_ERR(pmecc)) 990 return PTR_ERR(pmecc); 991 992 platform_set_drvdata(pdev, pmecc); 993 994 return 0; 995 } 996 997 static struct platform_driver atmel_pmecc_driver = { 998 .driver = { 999 .name = "atmel-pmecc", 1000 .of_match_table = atmel_pmecc_match, 1001 }, 1002 .probe = atmel_pmecc_probe, 1003 }; 1004 module_platform_driver(atmel_pmecc_driver); 1005 1006 MODULE_LICENSE("GPL"); 1007 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>"); 1008 MODULE_DESCRIPTION("PMECC engine driver"); 1009 MODULE_ALIAS("platform:atmel_pmecc"); 1010