1 /* 2 * Copyright (C) 2012 Google, Inc. 3 * 4 * This software is licensed under the terms of the GNU General Public 5 * License version 2, as published by the Free Software Foundation, and 6 * may be copied, distributed, and modified under those terms. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 */ 14 15 #define pr_fmt(fmt) "persistent_ram: " fmt 16 17 #include <linux/device.h> 18 #include <linux/err.h> 19 #include <linux/errno.h> 20 #include <linux/init.h> 21 #include <linux/io.h> 22 #include <linux/kernel.h> 23 #include <linux/list.h> 24 #include <linux/memblock.h> 25 #include <linux/pstore_ram.h> 26 #include <linux/rslib.h> 27 #include <linux/slab.h> 28 #include <linux/uaccess.h> 29 #include <linux/vmalloc.h> 30 #include <asm/page.h> 31 32 struct persistent_ram_buffer { 33 uint32_t sig; 34 atomic_t start; 35 atomic_t size; 36 uint8_t data[0]; 37 }; 38 39 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ 40 41 static inline size_t buffer_size(struct persistent_ram_zone *prz) 42 { 43 return atomic_read(&prz->buffer->size); 44 } 45 46 static inline size_t buffer_start(struct persistent_ram_zone *prz) 47 { 48 return atomic_read(&prz->buffer->start); 49 } 50 51 /* increase and wrap the start pointer, returning the old value */ 52 static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) 53 { 54 int old; 55 int new; 56 unsigned long flags = 0; 57 58 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 59 raw_spin_lock_irqsave(&prz->buffer_lock, flags); 60 61 old = atomic_read(&prz->buffer->start); 62 new = old + a; 63 while (unlikely(new >= prz->buffer_size)) 64 new -= prz->buffer_size; 65 atomic_set(&prz->buffer->start, new); 66 67 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 68 raw_spin_unlock_irqrestore(&prz->buffer_lock, flags); 69 70 return old; 71 } 72 73 /* increase the size counter until it hits the max size */ 74 static void buffer_size_add(struct persistent_ram_zone *prz, size_t a) 75 { 76 size_t old; 77 size_t new; 78 unsigned long flags = 0; 79 80 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 81 raw_spin_lock_irqsave(&prz->buffer_lock, flags); 82 83 old = atomic_read(&prz->buffer->size); 84 if (old == prz->buffer_size) 85 goto exit; 86 87 new = old + a; 88 if (new > prz->buffer_size) 89 new = prz->buffer_size; 90 atomic_set(&prz->buffer->size, new); 91 92 exit: 93 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 94 raw_spin_unlock_irqrestore(&prz->buffer_lock, flags); 95 } 96 97 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, 98 uint8_t *data, size_t len, uint8_t *ecc) 99 { 100 int i; 101 102 /* Initialize the parity buffer */ 103 memset(prz->ecc_info.par, 0, 104 prz->ecc_info.ecc_size * sizeof(prz->ecc_info.par[0])); 105 encode_rs8(prz->rs_decoder, data, len, prz->ecc_info.par, 0); 106 for (i = 0; i < prz->ecc_info.ecc_size; i++) 107 ecc[i] = prz->ecc_info.par[i]; 108 } 109 110 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, 111 void *data, size_t len, uint8_t *ecc) 112 { 113 int i; 114 115 for (i = 0; i < prz->ecc_info.ecc_size; i++) 116 prz->ecc_info.par[i] = ecc[i]; 117 return decode_rs8(prz->rs_decoder, data, prz->ecc_info.par, len, 118 NULL, 0, NULL, 0, NULL); 119 } 120 121 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, 122 unsigned int start, unsigned int count) 123 { 124 struct persistent_ram_buffer *buffer = prz->buffer; 125 uint8_t *buffer_end = buffer->data + prz->buffer_size; 126 uint8_t *block; 127 uint8_t *par; 128 int ecc_block_size = prz->ecc_info.block_size; 129 int ecc_size = prz->ecc_info.ecc_size; 130 int size = ecc_block_size; 131 132 if (!ecc_size) 133 return; 134 135 block = buffer->data + (start & ~(ecc_block_size - 1)); 136 par = prz->par_buffer + (start / ecc_block_size) * ecc_size; 137 138 do { 139 if (block + ecc_block_size > buffer_end) 140 size = buffer_end - block; 141 persistent_ram_encode_rs8(prz, block, size, par); 142 block += ecc_block_size; 143 par += ecc_size; 144 } while (block < buffer->data + start + count); 145 } 146 147 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) 148 { 149 struct persistent_ram_buffer *buffer = prz->buffer; 150 151 if (!prz->ecc_info.ecc_size) 152 return; 153 154 persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), 155 prz->par_header); 156 } 157 158 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) 159 { 160 struct persistent_ram_buffer *buffer = prz->buffer; 161 uint8_t *block; 162 uint8_t *par; 163 164 if (!prz->ecc_info.ecc_size) 165 return; 166 167 block = buffer->data; 168 par = prz->par_buffer; 169 while (block < buffer->data + buffer_size(prz)) { 170 int numerr; 171 int size = prz->ecc_info.block_size; 172 if (block + size > buffer->data + prz->buffer_size) 173 size = buffer->data + prz->buffer_size - block; 174 numerr = persistent_ram_decode_rs8(prz, block, size, par); 175 if (numerr > 0) { 176 pr_devel("error in block %p, %d\n", block, numerr); 177 prz->corrected_bytes += numerr; 178 } else if (numerr < 0) { 179 pr_devel("uncorrectable error in block %p\n", block); 180 prz->bad_blocks++; 181 } 182 block += prz->ecc_info.block_size; 183 par += prz->ecc_info.ecc_size; 184 } 185 } 186 187 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, 188 struct persistent_ram_ecc_info *ecc_info) 189 { 190 int numerr; 191 struct persistent_ram_buffer *buffer = prz->buffer; 192 int ecc_blocks; 193 size_t ecc_total; 194 195 if (!ecc_info || !ecc_info->ecc_size) 196 return 0; 197 198 prz->ecc_info.block_size = ecc_info->block_size ?: 128; 199 prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16; 200 prz->ecc_info.symsize = ecc_info->symsize ?: 8; 201 prz->ecc_info.poly = ecc_info->poly ?: 0x11d; 202 203 ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size, 204 prz->ecc_info.block_size + 205 prz->ecc_info.ecc_size); 206 ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size; 207 if (ecc_total >= prz->buffer_size) { 208 pr_err("%s: invalid ecc_size %u (total %zu, buffer size %zu)\n", 209 __func__, prz->ecc_info.ecc_size, 210 ecc_total, prz->buffer_size); 211 return -EINVAL; 212 } 213 214 prz->buffer_size -= ecc_total; 215 prz->par_buffer = buffer->data + prz->buffer_size; 216 prz->par_header = prz->par_buffer + 217 ecc_blocks * prz->ecc_info.ecc_size; 218 219 /* 220 * first consecutive root is 0 221 * primitive element to generate roots = 1 222 */ 223 prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly, 224 0, 1, prz->ecc_info.ecc_size); 225 if (prz->rs_decoder == NULL) { 226 pr_info("init_rs failed\n"); 227 return -EINVAL; 228 } 229 230 /* allocate workspace instead of using stack VLA */ 231 prz->ecc_info.par = kmalloc_array(prz->ecc_info.ecc_size, 232 sizeof(*prz->ecc_info.par), 233 GFP_KERNEL); 234 if (!prz->ecc_info.par) { 235 pr_err("cannot allocate ECC parity workspace\n"); 236 return -ENOMEM; 237 } 238 239 prz->corrected_bytes = 0; 240 prz->bad_blocks = 0; 241 242 numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), 243 prz->par_header); 244 if (numerr > 0) { 245 pr_info("error in header, %d\n", numerr); 246 prz->corrected_bytes += numerr; 247 } else if (numerr < 0) { 248 pr_info("uncorrectable error in header\n"); 249 prz->bad_blocks++; 250 } 251 252 return 0; 253 } 254 255 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, 256 char *str, size_t len) 257 { 258 ssize_t ret; 259 260 if (!prz->ecc_info.ecc_size) 261 return 0; 262 263 if (prz->corrected_bytes || prz->bad_blocks) 264 ret = snprintf(str, len, "" 265 "\n%d Corrected bytes, %d unrecoverable blocks\n", 266 prz->corrected_bytes, prz->bad_blocks); 267 else 268 ret = snprintf(str, len, "\nNo errors detected\n"); 269 270 return ret; 271 } 272 273 static void notrace persistent_ram_update(struct persistent_ram_zone *prz, 274 const void *s, unsigned int start, unsigned int count) 275 { 276 struct persistent_ram_buffer *buffer = prz->buffer; 277 memcpy_toio(buffer->data + start, s, count); 278 persistent_ram_update_ecc(prz, start, count); 279 } 280 281 static int notrace persistent_ram_update_user(struct persistent_ram_zone *prz, 282 const void __user *s, unsigned int start, unsigned int count) 283 { 284 struct persistent_ram_buffer *buffer = prz->buffer; 285 int ret = unlikely(__copy_from_user(buffer->data + start, s, count)) ? 286 -EFAULT : 0; 287 persistent_ram_update_ecc(prz, start, count); 288 return ret; 289 } 290 291 void persistent_ram_save_old(struct persistent_ram_zone *prz) 292 { 293 struct persistent_ram_buffer *buffer = prz->buffer; 294 size_t size = buffer_size(prz); 295 size_t start = buffer_start(prz); 296 297 if (!size) 298 return; 299 300 if (!prz->old_log) { 301 persistent_ram_ecc_old(prz); 302 prz->old_log = kmalloc(size, GFP_KERNEL); 303 } 304 if (!prz->old_log) { 305 pr_err("failed to allocate buffer\n"); 306 return; 307 } 308 309 prz->old_log_size = size; 310 memcpy_fromio(prz->old_log, &buffer->data[start], size - start); 311 memcpy_fromio(prz->old_log + size - start, &buffer->data[0], start); 312 } 313 314 int notrace persistent_ram_write(struct persistent_ram_zone *prz, 315 const void *s, unsigned int count) 316 { 317 int rem; 318 int c = count; 319 size_t start; 320 321 if (unlikely(c > prz->buffer_size)) { 322 s += c - prz->buffer_size; 323 c = prz->buffer_size; 324 } 325 326 buffer_size_add(prz, c); 327 328 start = buffer_start_add(prz, c); 329 330 rem = prz->buffer_size - start; 331 if (unlikely(rem < c)) { 332 persistent_ram_update(prz, s, start, rem); 333 s += rem; 334 c -= rem; 335 start = 0; 336 } 337 persistent_ram_update(prz, s, start, c); 338 339 persistent_ram_update_header_ecc(prz); 340 341 return count; 342 } 343 344 int notrace persistent_ram_write_user(struct persistent_ram_zone *prz, 345 const void __user *s, unsigned int count) 346 { 347 int rem, ret = 0, c = count; 348 size_t start; 349 350 if (unlikely(!access_ok(VERIFY_READ, s, count))) 351 return -EFAULT; 352 if (unlikely(c > prz->buffer_size)) { 353 s += c - prz->buffer_size; 354 c = prz->buffer_size; 355 } 356 357 buffer_size_add(prz, c); 358 359 start = buffer_start_add(prz, c); 360 361 rem = prz->buffer_size - start; 362 if (unlikely(rem < c)) { 363 ret = persistent_ram_update_user(prz, s, start, rem); 364 s += rem; 365 c -= rem; 366 start = 0; 367 } 368 if (likely(!ret)) 369 ret = persistent_ram_update_user(prz, s, start, c); 370 371 persistent_ram_update_header_ecc(prz); 372 373 return unlikely(ret) ? ret : count; 374 } 375 376 size_t persistent_ram_old_size(struct persistent_ram_zone *prz) 377 { 378 return prz->old_log_size; 379 } 380 381 void *persistent_ram_old(struct persistent_ram_zone *prz) 382 { 383 return prz->old_log; 384 } 385 386 void persistent_ram_free_old(struct persistent_ram_zone *prz) 387 { 388 kfree(prz->old_log); 389 prz->old_log = NULL; 390 prz->old_log_size = 0; 391 } 392 393 void persistent_ram_zap(struct persistent_ram_zone *prz) 394 { 395 atomic_set(&prz->buffer->start, 0); 396 atomic_set(&prz->buffer->size, 0); 397 persistent_ram_update_header_ecc(prz); 398 } 399 400 static void *persistent_ram_vmap(phys_addr_t start, size_t size, 401 unsigned int memtype) 402 { 403 struct page **pages; 404 phys_addr_t page_start; 405 unsigned int page_count; 406 pgprot_t prot; 407 unsigned int i; 408 void *vaddr; 409 410 page_start = start - offset_in_page(start); 411 page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); 412 413 if (memtype) 414 prot = pgprot_noncached(PAGE_KERNEL); 415 else 416 prot = pgprot_writecombine(PAGE_KERNEL); 417 418 pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); 419 if (!pages) { 420 pr_err("%s: Failed to allocate array for %u pages\n", 421 __func__, page_count); 422 return NULL; 423 } 424 425 for (i = 0; i < page_count; i++) { 426 phys_addr_t addr = page_start + i * PAGE_SIZE; 427 pages[i] = pfn_to_page(addr >> PAGE_SHIFT); 428 } 429 vaddr = vmap(pages, page_count, VM_MAP, prot); 430 kfree(pages); 431 432 return vaddr; 433 } 434 435 static void *persistent_ram_iomap(phys_addr_t start, size_t size, 436 unsigned int memtype) 437 { 438 void *va; 439 440 if (!request_mem_region(start, size, "persistent_ram")) { 441 pr_err("request mem region (0x%llx@0x%llx) failed\n", 442 (unsigned long long)size, (unsigned long long)start); 443 return NULL; 444 } 445 446 if (memtype) 447 va = ioremap(start, size); 448 else 449 va = ioremap_wc(start, size); 450 451 return va; 452 } 453 454 static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, 455 struct persistent_ram_zone *prz, int memtype) 456 { 457 prz->paddr = start; 458 prz->size = size; 459 460 if (pfn_valid(start >> PAGE_SHIFT)) 461 prz->vaddr = persistent_ram_vmap(start, size, memtype); 462 else 463 prz->vaddr = persistent_ram_iomap(start, size, memtype); 464 465 if (!prz->vaddr) { 466 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__, 467 (unsigned long long)size, (unsigned long long)start); 468 return -ENOMEM; 469 } 470 471 prz->buffer = prz->vaddr + offset_in_page(start); 472 prz->buffer_size = size - sizeof(struct persistent_ram_buffer); 473 474 return 0; 475 } 476 477 static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig, 478 struct persistent_ram_ecc_info *ecc_info) 479 { 480 int ret; 481 482 ret = persistent_ram_init_ecc(prz, ecc_info); 483 if (ret) 484 return ret; 485 486 sig ^= PERSISTENT_RAM_SIG; 487 488 if (prz->buffer->sig == sig) { 489 if (buffer_size(prz) > prz->buffer_size || 490 buffer_start(prz) > buffer_size(prz)) 491 pr_info("found existing invalid buffer, size %zu, start %zu\n", 492 buffer_size(prz), buffer_start(prz)); 493 else { 494 pr_debug("found existing buffer, size %zu, start %zu\n", 495 buffer_size(prz), buffer_start(prz)); 496 persistent_ram_save_old(prz); 497 return 0; 498 } 499 } else { 500 pr_debug("no valid data in buffer (sig = 0x%08x)\n", 501 prz->buffer->sig); 502 } 503 504 /* Rewind missing or invalid memory area. */ 505 prz->buffer->sig = sig; 506 persistent_ram_zap(prz); 507 508 return 0; 509 } 510 511 void persistent_ram_free(struct persistent_ram_zone *prz) 512 { 513 if (!prz) 514 return; 515 516 if (prz->vaddr) { 517 if (pfn_valid(prz->paddr >> PAGE_SHIFT)) { 518 vunmap(prz->vaddr); 519 } else { 520 iounmap(prz->vaddr); 521 release_mem_region(prz->paddr, prz->size); 522 } 523 prz->vaddr = NULL; 524 } 525 if (prz->rs_decoder) { 526 free_rs(prz->rs_decoder); 527 prz->rs_decoder = NULL; 528 } 529 kfree(prz->ecc_info.par); 530 prz->ecc_info.par = NULL; 531 532 persistent_ram_free_old(prz); 533 kfree(prz); 534 } 535 536 struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size, 537 u32 sig, struct persistent_ram_ecc_info *ecc_info, 538 unsigned int memtype, u32 flags) 539 { 540 struct persistent_ram_zone *prz; 541 int ret = -ENOMEM; 542 543 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); 544 if (!prz) { 545 pr_err("failed to allocate persistent ram zone\n"); 546 goto err; 547 } 548 549 /* Initialize general buffer state. */ 550 raw_spin_lock_init(&prz->buffer_lock); 551 prz->flags = flags; 552 553 ret = persistent_ram_buffer_map(start, size, prz, memtype); 554 if (ret) 555 goto err; 556 557 ret = persistent_ram_post_init(prz, sig, ecc_info); 558 if (ret) 559 goto err; 560 561 return prz; 562 err: 563 persistent_ram_free(prz); 564 return ERR_PTR(ret); 565 } 566