1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Google, Inc. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/device.h> 9 #include <linux/err.h> 10 #include <linux/errno.h> 11 #include <linux/init.h> 12 #include <linux/io.h> 13 #include <linux/kernel.h> 14 #include <linux/list.h> 15 #include <linux/memblock.h> 16 #include <linux/pstore_ram.h> 17 #include <linux/rslib.h> 18 #include <linux/slab.h> 19 #include <linux/uaccess.h> 20 #include <linux/vmalloc.h> 21 #include <asm/page.h> 22 23 /** 24 * struct persistent_ram_buffer - persistent circular RAM buffer 25 * 26 * @sig: 27 * signature to indicate header (PERSISTENT_RAM_SIG xor PRZ-type value) 28 * @start: 29 * offset into @data where the beginning of the stored bytes begin 30 * @size: 31 * number of valid bytes stored in @data 32 */ 33 struct persistent_ram_buffer { 34 uint32_t sig; 35 atomic_t start; 36 atomic_t size; 37 uint8_t data[]; 38 }; 39 40 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ 41 42 static inline size_t buffer_size(struct persistent_ram_zone *prz) 43 { 44 return atomic_read(&prz->buffer->size); 45 } 46 47 static inline size_t buffer_start(struct persistent_ram_zone *prz) 48 { 49 return atomic_read(&prz->buffer->start); 50 } 51 52 /* increase and wrap the start pointer, returning the old value */ 53 static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) 54 { 55 int old; 56 int new; 57 unsigned long flags = 0; 58 59 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 60 raw_spin_lock_irqsave(&prz->buffer_lock, flags); 61 62 old = atomic_read(&prz->buffer->start); 63 new = old + a; 64 while (unlikely(new >= prz->buffer_size)) 65 new -= prz->buffer_size; 66 atomic_set(&prz->buffer->start, new); 67 68 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 69 raw_spin_unlock_irqrestore(&prz->buffer_lock, flags); 70 71 return old; 72 } 73 74 /* increase the size counter until it hits the max size */ 75 static void buffer_size_add(struct persistent_ram_zone *prz, size_t a) 76 { 77 size_t old; 78 size_t new; 79 unsigned long flags = 0; 80 81 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 82 raw_spin_lock_irqsave(&prz->buffer_lock, flags); 83 84 old = atomic_read(&prz->buffer->size); 85 if (old == prz->buffer_size) 86 goto exit; 87 88 new = old + a; 89 if (new > prz->buffer_size) 90 new = prz->buffer_size; 91 atomic_set(&prz->buffer->size, new); 92 93 exit: 94 if (!(prz->flags & PRZ_FLAG_NO_LOCK)) 95 raw_spin_unlock_irqrestore(&prz->buffer_lock, flags); 96 } 97 98 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, 99 uint8_t *data, size_t len, uint8_t *ecc) 100 { 101 int i; 102 103 /* Initialize the parity buffer */ 104 memset(prz->ecc_info.par, 0, 105 prz->ecc_info.ecc_size * sizeof(prz->ecc_info.par[0])); 106 encode_rs8(prz->rs_decoder, data, len, prz->ecc_info.par, 0); 107 for (i = 0; i < prz->ecc_info.ecc_size; i++) 108 ecc[i] = prz->ecc_info.par[i]; 109 } 110 111 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, 112 void *data, size_t len, uint8_t *ecc) 113 { 114 int i; 115 116 for (i = 0; i < prz->ecc_info.ecc_size; i++) 117 prz->ecc_info.par[i] = ecc[i]; 118 return decode_rs8(prz->rs_decoder, data, prz->ecc_info.par, len, 119 NULL, 0, NULL, 0, NULL); 120 } 121 122 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, 123 unsigned int start, unsigned int count) 124 { 125 struct persistent_ram_buffer *buffer = prz->buffer; 126 uint8_t *buffer_end = buffer->data + prz->buffer_size; 127 uint8_t *block; 128 uint8_t *par; 129 int ecc_block_size = prz->ecc_info.block_size; 130 int ecc_size = prz->ecc_info.ecc_size; 131 int size = ecc_block_size; 132 133 if (!ecc_size) 134 return; 135 136 block = buffer->data + (start & ~(ecc_block_size - 1)); 137 par = prz->par_buffer + (start / ecc_block_size) * ecc_size; 138 139 do { 140 if (block + ecc_block_size > buffer_end) 141 size = buffer_end - block; 142 persistent_ram_encode_rs8(prz, block, size, par); 143 block += ecc_block_size; 144 par += ecc_size; 145 } while (block < buffer->data + start + count); 146 } 147 148 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) 149 { 150 struct persistent_ram_buffer *buffer = prz->buffer; 151 152 if (!prz->ecc_info.ecc_size) 153 return; 154 155 persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), 156 prz->par_header); 157 } 158 159 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) 160 { 161 struct persistent_ram_buffer *buffer = prz->buffer; 162 uint8_t *block; 163 uint8_t *par; 164 165 if (!prz->ecc_info.ecc_size) 166 return; 167 168 block = buffer->data; 169 par = prz->par_buffer; 170 while (block < buffer->data + buffer_size(prz)) { 171 int numerr; 172 int size = prz->ecc_info.block_size; 173 if (block + size > buffer->data + prz->buffer_size) 174 size = buffer->data + prz->buffer_size - block; 175 numerr = persistent_ram_decode_rs8(prz, block, size, par); 176 if (numerr > 0) { 177 pr_devel("error in block %p, %d\n", block, numerr); 178 prz->corrected_bytes += numerr; 179 } else if (numerr < 0) { 180 pr_devel("uncorrectable error in block %p\n", block); 181 prz->bad_blocks++; 182 } 183 block += prz->ecc_info.block_size; 184 par += prz->ecc_info.ecc_size; 185 } 186 } 187 188 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, 189 struct persistent_ram_ecc_info *ecc_info) 190 { 191 int numerr; 192 struct persistent_ram_buffer *buffer = prz->buffer; 193 int ecc_blocks; 194 size_t ecc_total; 195 196 if (!ecc_info || !ecc_info->ecc_size) 197 return 0; 198 199 prz->ecc_info.block_size = ecc_info->block_size ?: 128; 200 prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16; 201 prz->ecc_info.symsize = ecc_info->symsize ?: 8; 202 prz->ecc_info.poly = ecc_info->poly ?: 0x11d; 203 204 ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size, 205 prz->ecc_info.block_size + 206 prz->ecc_info.ecc_size); 207 ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size; 208 if (ecc_total >= prz->buffer_size) { 209 pr_err("%s: invalid ecc_size %u (total %zu, buffer size %zu)\n", 210 __func__, prz->ecc_info.ecc_size, 211 ecc_total, prz->buffer_size); 212 return -EINVAL; 213 } 214 215 prz->buffer_size -= ecc_total; 216 prz->par_buffer = buffer->data + prz->buffer_size; 217 prz->par_header = prz->par_buffer + 218 ecc_blocks * prz->ecc_info.ecc_size; 219 220 /* 221 * first consecutive root is 0 222 * primitive element to generate roots = 1 223 */ 224 prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly, 225 0, 1, prz->ecc_info.ecc_size); 226 if (prz->rs_decoder == NULL) { 227 pr_info("init_rs failed\n"); 228 return -EINVAL; 229 } 230 231 /* allocate workspace instead of using stack VLA */ 232 prz->ecc_info.par = kmalloc_array(prz->ecc_info.ecc_size, 233 sizeof(*prz->ecc_info.par), 234 GFP_KERNEL); 235 if (!prz->ecc_info.par) { 236 pr_err("cannot allocate ECC parity workspace\n"); 237 return -ENOMEM; 238 } 239 240 prz->corrected_bytes = 0; 241 prz->bad_blocks = 0; 242 243 numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), 244 prz->par_header); 245 if (numerr > 0) { 246 pr_info("error in header, %d\n", numerr); 247 prz->corrected_bytes += numerr; 248 } else if (numerr < 0) { 249 pr_info_ratelimited("uncorrectable error in header\n"); 250 prz->bad_blocks++; 251 } 252 253 return 0; 254 } 255 256 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, 257 char *str, size_t len) 258 { 259 ssize_t ret; 260 261 if (!prz->ecc_info.ecc_size) 262 return 0; 263 264 if (prz->corrected_bytes || prz->bad_blocks) 265 ret = snprintf(str, len, "" 266 "\n%d Corrected bytes, %d unrecoverable blocks\n", 267 prz->corrected_bytes, prz->bad_blocks); 268 else 269 ret = snprintf(str, len, "\nNo errors detected\n"); 270 271 return ret; 272 } 273 274 static void notrace persistent_ram_update(struct persistent_ram_zone *prz, 275 const void *s, unsigned int start, unsigned int count) 276 { 277 struct persistent_ram_buffer *buffer = prz->buffer; 278 memcpy_toio(buffer->data + start, s, count); 279 persistent_ram_update_ecc(prz, start, count); 280 } 281 282 static int notrace persistent_ram_update_user(struct persistent_ram_zone *prz, 283 const void __user *s, unsigned int start, unsigned int count) 284 { 285 struct persistent_ram_buffer *buffer = prz->buffer; 286 int ret = unlikely(copy_from_user(buffer->data + start, s, count)) ? 287 -EFAULT : 0; 288 persistent_ram_update_ecc(prz, start, count); 289 return ret; 290 } 291 292 void persistent_ram_save_old(struct persistent_ram_zone *prz) 293 { 294 struct persistent_ram_buffer *buffer = prz->buffer; 295 size_t size = buffer_size(prz); 296 size_t start = buffer_start(prz); 297 298 if (!size) 299 return; 300 301 if (!prz->old_log) { 302 persistent_ram_ecc_old(prz); 303 prz->old_log = kmalloc(size, GFP_KERNEL); 304 } 305 if (!prz->old_log) { 306 pr_err("failed to allocate buffer\n"); 307 return; 308 } 309 310 prz->old_log_size = size; 311 memcpy_fromio(prz->old_log, &buffer->data[start], size - start); 312 memcpy_fromio(prz->old_log + size - start, &buffer->data[0], start); 313 } 314 315 int notrace persistent_ram_write(struct persistent_ram_zone *prz, 316 const void *s, unsigned int count) 317 { 318 int rem; 319 int c = count; 320 size_t start; 321 322 if (unlikely(c > prz->buffer_size)) { 323 s += c - prz->buffer_size; 324 c = prz->buffer_size; 325 } 326 327 buffer_size_add(prz, c); 328 329 start = buffer_start_add(prz, c); 330 331 rem = prz->buffer_size - start; 332 if (unlikely(rem < c)) { 333 persistent_ram_update(prz, s, start, rem); 334 s += rem; 335 c -= rem; 336 start = 0; 337 } 338 persistent_ram_update(prz, s, start, c); 339 340 persistent_ram_update_header_ecc(prz); 341 342 return count; 343 } 344 345 int notrace persistent_ram_write_user(struct persistent_ram_zone *prz, 346 const void __user *s, unsigned int count) 347 { 348 int rem, ret = 0, c = count; 349 size_t start; 350 351 if (unlikely(c > prz->buffer_size)) { 352 s += c - prz->buffer_size; 353 c = prz->buffer_size; 354 } 355 356 buffer_size_add(prz, c); 357 358 start = buffer_start_add(prz, c); 359 360 rem = prz->buffer_size - start; 361 if (unlikely(rem < c)) { 362 ret = persistent_ram_update_user(prz, s, start, rem); 363 s += rem; 364 c -= rem; 365 start = 0; 366 } 367 if (likely(!ret)) 368 ret = persistent_ram_update_user(prz, s, start, c); 369 370 persistent_ram_update_header_ecc(prz); 371 372 return unlikely(ret) ? ret : count; 373 } 374 375 size_t persistent_ram_old_size(struct persistent_ram_zone *prz) 376 { 377 return prz->old_log_size; 378 } 379 380 void *persistent_ram_old(struct persistent_ram_zone *prz) 381 { 382 return prz->old_log; 383 } 384 385 void persistent_ram_free_old(struct persistent_ram_zone *prz) 386 { 387 kfree(prz->old_log); 388 prz->old_log = NULL; 389 prz->old_log_size = 0; 390 } 391 392 void persistent_ram_zap(struct persistent_ram_zone *prz) 393 { 394 atomic_set(&prz->buffer->start, 0); 395 atomic_set(&prz->buffer->size, 0); 396 persistent_ram_update_header_ecc(prz); 397 } 398 399 static void *persistent_ram_vmap(phys_addr_t start, size_t size, 400 unsigned int memtype) 401 { 402 struct page **pages; 403 phys_addr_t page_start; 404 unsigned int page_count; 405 pgprot_t prot; 406 unsigned int i; 407 void *vaddr; 408 409 page_start = start - offset_in_page(start); 410 page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); 411 412 if (memtype) 413 prot = pgprot_noncached(PAGE_KERNEL); 414 else 415 prot = pgprot_writecombine(PAGE_KERNEL); 416 417 pages = kmalloc_array(page_count, sizeof(struct page *), GFP_KERNEL); 418 if (!pages) { 419 pr_err("%s: Failed to allocate array for %u pages\n", 420 __func__, page_count); 421 return NULL; 422 } 423 424 for (i = 0; i < page_count; i++) { 425 phys_addr_t addr = page_start + i * PAGE_SIZE; 426 pages[i] = pfn_to_page(addr >> PAGE_SHIFT); 427 } 428 vaddr = vmap(pages, page_count, VM_MAP, prot); 429 kfree(pages); 430 431 /* 432 * Since vmap() uses page granularity, we must add the offset 433 * into the page here, to get the byte granularity address 434 * into the mapping to represent the actual "start" location. 435 */ 436 return vaddr + offset_in_page(start); 437 } 438 439 static void *persistent_ram_iomap(phys_addr_t start, size_t size, 440 unsigned int memtype, char *label) 441 { 442 void *va; 443 444 if (!request_mem_region(start, size, label ?: "ramoops")) { 445 pr_err("request mem region (%s 0x%llx@0x%llx) failed\n", 446 label ?: "ramoops", 447 (unsigned long long)size, (unsigned long long)start); 448 return NULL; 449 } 450 451 if (memtype) 452 va = ioremap(start, size); 453 else 454 va = ioremap_wc(start, size); 455 456 /* 457 * Since request_mem_region() and ioremap() are byte-granularity 458 * there is no need handle anything special like we do when the 459 * vmap() case in persistent_ram_vmap() above. 460 */ 461 return va; 462 } 463 464 static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, 465 struct persistent_ram_zone *prz, int memtype) 466 { 467 prz->paddr = start; 468 prz->size = size; 469 470 if (pfn_valid(start >> PAGE_SHIFT)) 471 prz->vaddr = persistent_ram_vmap(start, size, memtype); 472 else 473 prz->vaddr = persistent_ram_iomap(start, size, memtype, 474 prz->label); 475 476 if (!prz->vaddr) { 477 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__, 478 (unsigned long long)size, (unsigned long long)start); 479 return -ENOMEM; 480 } 481 482 prz->buffer = prz->vaddr; 483 prz->buffer_size = size - sizeof(struct persistent_ram_buffer); 484 485 return 0; 486 } 487 488 static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig, 489 struct persistent_ram_ecc_info *ecc_info) 490 { 491 int ret; 492 bool zap = !!(prz->flags & PRZ_FLAG_ZAP_OLD); 493 494 ret = persistent_ram_init_ecc(prz, ecc_info); 495 if (ret) { 496 pr_warn("ECC failed %s\n", prz->label); 497 return ret; 498 } 499 500 sig ^= PERSISTENT_RAM_SIG; 501 502 if (prz->buffer->sig == sig) { 503 if (buffer_size(prz) == 0) { 504 pr_debug("found existing empty buffer\n"); 505 return 0; 506 } 507 508 if (buffer_size(prz) > prz->buffer_size || 509 buffer_start(prz) > buffer_size(prz)) { 510 pr_info("found existing invalid buffer, size %zu, start %zu\n", 511 buffer_size(prz), buffer_start(prz)); 512 zap = true; 513 } else { 514 pr_debug("found existing buffer, size %zu, start %zu\n", 515 buffer_size(prz), buffer_start(prz)); 516 persistent_ram_save_old(prz); 517 } 518 } else { 519 pr_debug("no valid data in buffer (sig = 0x%08x)\n", 520 prz->buffer->sig); 521 prz->buffer->sig = sig; 522 zap = true; 523 } 524 525 /* Reset missing, invalid, or single-use memory area. */ 526 if (zap) 527 persistent_ram_zap(prz); 528 529 return 0; 530 } 531 532 void persistent_ram_free(struct persistent_ram_zone *prz) 533 { 534 if (!prz) 535 return; 536 537 if (prz->vaddr) { 538 if (pfn_valid(prz->paddr >> PAGE_SHIFT)) { 539 /* We must vunmap() at page-granularity. */ 540 vunmap(prz->vaddr - offset_in_page(prz->paddr)); 541 } else { 542 iounmap(prz->vaddr); 543 release_mem_region(prz->paddr, prz->size); 544 } 545 prz->vaddr = NULL; 546 } 547 if (prz->rs_decoder) { 548 free_rs(prz->rs_decoder); 549 prz->rs_decoder = NULL; 550 } 551 kfree(prz->ecc_info.par); 552 prz->ecc_info.par = NULL; 553 554 persistent_ram_free_old(prz); 555 kfree(prz->label); 556 kfree(prz); 557 } 558 559 struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size, 560 u32 sig, struct persistent_ram_ecc_info *ecc_info, 561 unsigned int memtype, u32 flags, char *label) 562 { 563 struct persistent_ram_zone *prz; 564 int ret = -ENOMEM; 565 566 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL); 567 if (!prz) { 568 pr_err("failed to allocate persistent ram zone\n"); 569 goto err; 570 } 571 572 /* Initialize general buffer state. */ 573 raw_spin_lock_init(&prz->buffer_lock); 574 prz->flags = flags; 575 prz->label = kstrdup(label, GFP_KERNEL); 576 577 ret = persistent_ram_buffer_map(start, size, prz, memtype); 578 if (ret) 579 goto err; 580 581 ret = persistent_ram_post_init(prz, sig, ecc_info); 582 if (ret) 583 goto err; 584 585 pr_debug("attached %s 0x%zx@0x%llx: %zu header, %zu data, %zu ecc (%d/%d)\n", 586 prz->label, prz->size, (unsigned long long)prz->paddr, 587 sizeof(*prz->buffer), prz->buffer_size, 588 prz->size - sizeof(*prz->buffer) - prz->buffer_size, 589 prz->ecc_info.ecc_size, prz->ecc_info.block_size); 590 591 return prz; 592 err: 593 persistent_ram_free(prz); 594 return ERR_PTR(ret); 595 } 596