1 /* 2 * (C) Copyright 2013 3 * David Feng <fenghua@phytium.com.cn> 4 * 5 * (C) Copyright 2016 6 * Alexander Graf <agraf@suse.de> 7 * 8 * SPDX-License-Identifier: GPL-2.0+ 9 */ 10 11 #include <common.h> 12 #include <asm/system.h> 13 #include <asm/armv8/mmu.h> 14 15 DECLARE_GLOBAL_DATA_PTR; 16 17 #ifndef CONFIG_SYS_DCACHE_OFF 18 19 /* 20 * With 4k page granule, a virtual address is split into 4 lookup parts 21 * spanning 9 bits each: 22 * 23 * _______________________________________________ 24 * | | | | | | | 25 * | 0 | Lv0 | Lv1 | Lv2 | Lv3 | off | 26 * |_______|_______|_______|_______|_______|_______| 27 * 63-48 47-39 38-30 29-21 20-12 11-00 28 * 29 * mask page size 30 * 31 * Lv0: FF8000000000 -- 32 * Lv1: 7FC0000000 1G 33 * Lv2: 3FE00000 2M 34 * Lv3: 1FF000 4K 35 * off: FFF 36 */ 37 38 #ifdef CONFIG_SYS_FULL_VA 39 static struct mm_region mem_map[] = CONFIG_SYS_MEM_MAP; 40 41 static u64 get_tcr(int el, u64 *pips, u64 *pva_bits) 42 { 43 u64 max_addr = 0; 44 u64 ips, va_bits; 45 u64 tcr; 46 int i; 47 48 /* Find the largest address we need to support */ 49 for (i = 0; i < ARRAY_SIZE(mem_map); i++) 50 max_addr = max(max_addr, mem_map[i].base + mem_map[i].size); 51 52 /* Calculate the maximum physical (and thus virtual) address */ 53 if (max_addr > (1ULL << 44)) { 54 ips = 5; 55 va_bits = 48; 56 } else if (max_addr > (1ULL << 42)) { 57 ips = 4; 58 va_bits = 44; 59 } else if (max_addr > (1ULL << 40)) { 60 ips = 3; 61 va_bits = 42; 62 } else if (max_addr > (1ULL << 36)) { 63 ips = 2; 64 va_bits = 40; 65 } else if (max_addr > (1ULL << 32)) { 66 ips = 1; 67 va_bits = 36; 68 } else { 69 ips = 0; 70 va_bits = 32; 71 } 72 73 if (el == 1) { 74 tcr = TCR_EL1_RSVD | (ips << 32) | TCR_EPD1_DISABLE; 75 } else if (el == 2) { 76 tcr = TCR_EL2_RSVD | (ips << 16); 77 } else { 78 tcr = TCR_EL3_RSVD | (ips << 16); 79 } 80 81 /* PTWs cacheable, inner/outer WBWA and inner shareable */ 82 tcr |= TCR_TG0_4K | TCR_SHARED_INNER | TCR_ORGN_WBWA | TCR_IRGN_WBWA; 83 tcr |= TCR_T0SZ(va_bits); 84 85 if (pips) 86 *pips = ips; 87 if (pva_bits) 88 *pva_bits = va_bits; 89 90 return tcr; 91 } 92 93 #define MAX_PTE_ENTRIES 512 94 95 static int pte_type(u64 *pte) 96 { 97 return *pte & PTE_TYPE_MASK; 98 } 99 100 /* Returns the LSB number for a PTE on level <level> */ 101 static int level2shift(int level) 102 { 103 /* Page is 12 bits wide, every level translates 9 bits */ 104 return (12 + 9 * (3 - level)); 105 } 106 107 static u64 *find_pte(u64 addr, int level) 108 { 109 int start_level = 0; 110 u64 *pte; 111 u64 idx; 112 u64 va_bits; 113 int i; 114 115 debug("addr=%llx level=%d\n", addr, level); 116 117 get_tcr(0, NULL, &va_bits); 118 if (va_bits < 39) 119 start_level = 1; 120 121 if (level < start_level) 122 return NULL; 123 124 /* Walk through all page table levels to find our PTE */ 125 pte = (u64*)gd->arch.tlb_addr; 126 for (i = start_level; i < 4; i++) { 127 idx = (addr >> level2shift(i)) & 0x1FF; 128 pte += idx; 129 debug("idx=%llx PTE %p at level %d: %llx\n", idx, pte, i, *pte); 130 131 /* Found it */ 132 if (i == level) 133 return pte; 134 /* PTE is no table (either invalid or block), can't traverse */ 135 if (pte_type(pte) != PTE_TYPE_TABLE) 136 return NULL; 137 /* Off to the next level */ 138 pte = (u64*)(*pte & 0x0000fffffffff000ULL); 139 } 140 141 /* Should never reach here */ 142 return NULL; 143 } 144 145 /* Returns and creates a new full table (512 entries) */ 146 static u64 *create_table(void) 147 { 148 u64 *new_table = (u64*)gd->arch.tlb_fillptr; 149 u64 pt_len = MAX_PTE_ENTRIES * sizeof(u64); 150 151 /* Allocate MAX_PTE_ENTRIES pte entries */ 152 gd->arch.tlb_fillptr += pt_len; 153 154 if (gd->arch.tlb_fillptr - gd->arch.tlb_addr > gd->arch.tlb_size) 155 panic("Insufficient RAM for page table: 0x%lx > 0x%lx. " 156 "Please increase the size in get_page_table_size()", 157 gd->arch.tlb_fillptr - gd->arch.tlb_addr, 158 gd->arch.tlb_size); 159 160 /* Mark all entries as invalid */ 161 memset(new_table, 0, pt_len); 162 163 return new_table; 164 } 165 166 static void set_pte_table(u64 *pte, u64 *table) 167 { 168 /* Point *pte to the new table */ 169 debug("Setting %p to addr=%p\n", pte, table); 170 *pte = PTE_TYPE_TABLE | (ulong)table; 171 } 172 173 /* Add one mm_region map entry to the page tables */ 174 static void add_map(struct mm_region *map) 175 { 176 u64 *pte; 177 u64 addr = map->base; 178 u64 size = map->size; 179 u64 attrs = map->attrs | PTE_TYPE_BLOCK | PTE_BLOCK_AF; 180 u64 blocksize; 181 int level; 182 u64 *new_table; 183 184 while (size) { 185 pte = find_pte(addr, 0); 186 if (pte && (pte_type(pte) == PTE_TYPE_FAULT)) { 187 debug("Creating table for addr 0x%llx\n", addr); 188 new_table = create_table(); 189 set_pte_table(pte, new_table); 190 } 191 192 for (level = 1; level < 4; level++) { 193 pte = find_pte(addr, level); 194 blocksize = 1ULL << level2shift(level); 195 debug("Checking if pte fits for addr=%llx size=%llx " 196 "blocksize=%llx\n", addr, size, blocksize); 197 if (size >= blocksize && !(addr & (blocksize - 1))) { 198 /* Page fits, create block PTE */ 199 debug("Setting PTE %p to block addr=%llx\n", 200 pte, addr); 201 *pte = addr | attrs; 202 addr += blocksize; 203 size -= blocksize; 204 break; 205 } else if ((pte_type(pte) == PTE_TYPE_FAULT)) { 206 /* Page doesn't fit, create subpages */ 207 debug("Creating subtable for addr 0x%llx " 208 "blksize=%llx\n", addr, blocksize); 209 new_table = create_table(); 210 set_pte_table(pte, new_table); 211 } 212 } 213 } 214 } 215 216 /* Splits a block PTE into table with subpages spanning the old block */ 217 static void split_block(u64 *pte, int level) 218 { 219 u64 old_pte = *pte; 220 u64 *new_table; 221 u64 i = 0; 222 /* level describes the parent level, we need the child ones */ 223 int levelshift = level2shift(level + 1); 224 225 if (pte_type(pte) != PTE_TYPE_BLOCK) 226 panic("PTE %p (%llx) is not a block. Some driver code wants to " 227 "modify dcache settings for an range not covered in " 228 "mem_map.", pte, old_pte); 229 230 new_table = create_table(); 231 debug("Splitting pte %p (%llx) into %p\n", pte, old_pte, new_table); 232 233 for (i = 0; i < MAX_PTE_ENTRIES; i++) { 234 new_table[i] = old_pte | (i << levelshift); 235 236 /* Level 3 block PTEs have the table type */ 237 if ((level + 1) == 3) 238 new_table[i] |= PTE_TYPE_TABLE; 239 240 debug("Setting new_table[%lld] = %llx\n", i, new_table[i]); 241 } 242 243 /* Set the new table into effect */ 244 set_pte_table(pte, new_table); 245 } 246 247 enum pte_type { 248 PTE_INVAL, 249 PTE_BLOCK, 250 PTE_LEVEL, 251 }; 252 253 /* 254 * This is a recursively called function to count the number of 255 * page tables we need to cover a particular PTE range. If you 256 * call this with level = -1 you basically get the full 48 bit 257 * coverage. 258 */ 259 static int count_required_pts(u64 addr, int level, u64 maxaddr) 260 { 261 int levelshift = level2shift(level); 262 u64 levelsize = 1ULL << levelshift; 263 u64 levelmask = levelsize - 1; 264 u64 levelend = addr + levelsize; 265 int r = 0; 266 int i; 267 enum pte_type pte_type = PTE_INVAL; 268 269 for (i = 0; i < ARRAY_SIZE(mem_map); i++) { 270 struct mm_region *map = &mem_map[i]; 271 u64 start = map->base; 272 u64 end = start + map->size; 273 274 /* Check if the PTE would overlap with the map */ 275 if (max(addr, start) <= min(levelend, end)) { 276 start = max(addr, start); 277 end = min(levelend, end); 278 279 /* We need a sub-pt for this level */ 280 if ((start & levelmask) || (end & levelmask)) { 281 pte_type = PTE_LEVEL; 282 break; 283 } 284 285 /* Lv0 can not do block PTEs, so do levels here too */ 286 if (level <= 0) { 287 pte_type = PTE_LEVEL; 288 break; 289 } 290 291 /* PTE is active, but fits into a block */ 292 pte_type = PTE_BLOCK; 293 } 294 } 295 296 /* 297 * Block PTEs at this level are already covered by the parent page 298 * table, so we only need to count sub page tables. 299 */ 300 if (pte_type == PTE_LEVEL) { 301 int sublevel = level + 1; 302 u64 sublevelsize = 1ULL << level2shift(sublevel); 303 304 /* Account for the new sub page table ... */ 305 r = 1; 306 307 /* ... and for all child page tables that one might have */ 308 for (i = 0; i < MAX_PTE_ENTRIES; i++) { 309 r += count_required_pts(addr, sublevel, maxaddr); 310 addr += sublevelsize; 311 312 if (addr >= maxaddr) { 313 /* 314 * We reached the end of address space, no need 315 * to look any further. 316 */ 317 break; 318 } 319 } 320 } 321 322 return r; 323 } 324 325 /* Returns the estimated required size of all page tables */ 326 u64 get_page_table_size(void) 327 { 328 u64 one_pt = MAX_PTE_ENTRIES * sizeof(u64); 329 u64 size = 0; 330 u64 va_bits; 331 int start_level = 0; 332 333 get_tcr(0, NULL, &va_bits); 334 if (va_bits < 39) 335 start_level = 1; 336 337 /* Account for all page tables we would need to cover our memory map */ 338 size = one_pt * count_required_pts(0, start_level - 1, 1ULL << va_bits); 339 340 /* 341 * We need to duplicate our page table once to have an emergency pt to 342 * resort to when splitting page tables later on 343 */ 344 size *= 2; 345 346 /* 347 * We may need to split page tables later on if dcache settings change, 348 * so reserve up to 4 (random pick) page tables for that. 349 */ 350 size += one_pt * 4; 351 352 return size; 353 } 354 355 static void setup_pgtables(void) 356 { 357 int i; 358 359 /* 360 * Allocate the first level we're on with invalidate entries. 361 * If the starting level is 0 (va_bits >= 39), then this is our 362 * Lv0 page table, otherwise it's the entry Lv1 page table. 363 */ 364 create_table(); 365 366 /* Now add all MMU table entries one after another to the table */ 367 for (i = 0; i < ARRAY_SIZE(mem_map); i++) 368 add_map(&mem_map[i]); 369 370 /* Create the same thing once more for our emergency page table */ 371 create_table(); 372 } 373 374 static void setup_all_pgtables(void) 375 { 376 u64 tlb_addr = gd->arch.tlb_addr; 377 378 /* Reset the fill ptr */ 379 gd->arch.tlb_fillptr = tlb_addr; 380 381 /* Create normal system page tables */ 382 setup_pgtables(); 383 384 /* Create emergency page tables */ 385 gd->arch.tlb_addr = gd->arch.tlb_fillptr; 386 setup_pgtables(); 387 gd->arch.tlb_emerg = gd->arch.tlb_addr; 388 gd->arch.tlb_addr = tlb_addr; 389 } 390 391 #else 392 393 inline void set_pgtable_section(u64 *page_table, u64 index, u64 section, 394 u64 memory_type, u64 attribute) 395 { 396 u64 value; 397 398 value = section | PMD_TYPE_SECT | PMD_SECT_AF; 399 value |= PMD_ATTRINDX(memory_type); 400 value |= attribute; 401 page_table[index] = value; 402 } 403 404 inline void set_pgtable_table(u64 *page_table, u64 index, u64 *table_addr) 405 { 406 u64 value; 407 408 value = (u64)table_addr | PMD_TYPE_TABLE; 409 page_table[index] = value; 410 } 411 #endif 412 413 /* to activate the MMU we need to set up virtual memory */ 414 __weak void mmu_setup(void) 415 { 416 #ifndef CONFIG_SYS_FULL_VA 417 bd_t *bd = gd->bd; 418 u64 *page_table = (u64 *)gd->arch.tlb_addr, i, j; 419 #endif 420 int el; 421 422 #ifdef CONFIG_SYS_FULL_VA 423 /* Set up page tables only once */ 424 if (!gd->arch.tlb_fillptr) 425 setup_all_pgtables(); 426 427 el = current_el(); 428 set_ttbr_tcr_mair(el, gd->arch.tlb_addr, get_tcr(el, NULL, NULL), 429 MEMORY_ATTRIBUTES); 430 #else 431 /* Setup an identity-mapping for all spaces */ 432 for (i = 0; i < (PGTABLE_SIZE >> 3); i++) { 433 set_pgtable_section(page_table, i, i << SECTION_SHIFT, 434 MT_DEVICE_NGNRNE, PMD_SECT_NON_SHARE); 435 } 436 437 /* Setup an identity-mapping for all RAM space */ 438 for (i = 0; i < CONFIG_NR_DRAM_BANKS; i++) { 439 ulong start = bd->bi_dram[i].start; 440 ulong end = bd->bi_dram[i].start + bd->bi_dram[i].size; 441 for (j = start >> SECTION_SHIFT; 442 j < end >> SECTION_SHIFT; j++) { 443 set_pgtable_section(page_table, j, j << SECTION_SHIFT, 444 MT_NORMAL, PMD_SECT_NON_SHARE); 445 } 446 } 447 448 /* load TTBR0 */ 449 el = current_el(); 450 if (el == 1) { 451 set_ttbr_tcr_mair(el, gd->arch.tlb_addr, 452 TCR_EL1_RSVD | TCR_FLAGS | TCR_EL1_IPS_BITS, 453 MEMORY_ATTRIBUTES); 454 } else if (el == 2) { 455 set_ttbr_tcr_mair(el, gd->arch.tlb_addr, 456 TCR_EL2_RSVD | TCR_FLAGS | TCR_EL2_IPS_BITS, 457 MEMORY_ATTRIBUTES); 458 } else { 459 set_ttbr_tcr_mair(el, gd->arch.tlb_addr, 460 TCR_EL3_RSVD | TCR_FLAGS | TCR_EL3_IPS_BITS, 461 MEMORY_ATTRIBUTES); 462 } 463 #endif 464 465 /* enable the mmu */ 466 set_sctlr(get_sctlr() | CR_M); 467 } 468 469 /* 470 * Performs a invalidation of the entire data cache at all levels 471 */ 472 void invalidate_dcache_all(void) 473 { 474 __asm_invalidate_dcache_all(); 475 } 476 477 /* 478 * Performs a clean & invalidation of the entire data cache at all levels. 479 * This function needs to be inline to avoid using stack. 480 * __asm_flush_l3_cache return status of timeout 481 */ 482 inline void flush_dcache_all(void) 483 { 484 int ret; 485 486 __asm_flush_dcache_all(); 487 ret = __asm_flush_l3_cache(); 488 if (ret) 489 debug("flushing dcache returns 0x%x\n", ret); 490 else 491 debug("flushing dcache successfully.\n"); 492 } 493 494 /* 495 * Invalidates range in all levels of D-cache/unified cache 496 */ 497 void invalidate_dcache_range(unsigned long start, unsigned long stop) 498 { 499 __asm_flush_dcache_range(start, stop); 500 } 501 502 /* 503 * Flush range(clean & invalidate) from all levels of D-cache/unified cache 504 */ 505 void flush_dcache_range(unsigned long start, unsigned long stop) 506 { 507 __asm_flush_dcache_range(start, stop); 508 } 509 510 void dcache_enable(void) 511 { 512 /* The data cache is not active unless the mmu is enabled */ 513 if (!(get_sctlr() & CR_M)) { 514 invalidate_dcache_all(); 515 __asm_invalidate_tlb_all(); 516 mmu_setup(); 517 } 518 519 set_sctlr(get_sctlr() | CR_C); 520 } 521 522 void dcache_disable(void) 523 { 524 uint32_t sctlr; 525 526 sctlr = get_sctlr(); 527 528 /* if cache isn't enabled no need to disable */ 529 if (!(sctlr & CR_C)) 530 return; 531 532 set_sctlr(sctlr & ~(CR_C|CR_M)); 533 534 flush_dcache_all(); 535 __asm_invalidate_tlb_all(); 536 } 537 538 int dcache_status(void) 539 { 540 return (get_sctlr() & CR_C) != 0; 541 } 542 543 u64 *__weak arch_get_page_table(void) { 544 puts("No page table offset defined\n"); 545 546 return NULL; 547 } 548 549 #ifndef CONFIG_SYS_FULL_VA 550 void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size, 551 enum dcache_option option) 552 { 553 u64 *page_table = arch_get_page_table(); 554 u64 upto, end; 555 556 if (page_table == NULL) 557 return; 558 559 end = ALIGN(start + size, (1 << MMU_SECTION_SHIFT)) >> 560 MMU_SECTION_SHIFT; 561 start = start >> MMU_SECTION_SHIFT; 562 for (upto = start; upto < end; upto++) { 563 page_table[upto] &= ~PMD_ATTRINDX_MASK; 564 page_table[upto] |= PMD_ATTRINDX(option); 565 } 566 asm volatile("dsb sy"); 567 __asm_invalidate_tlb_all(); 568 asm volatile("dsb sy"); 569 asm volatile("isb"); 570 start = start << MMU_SECTION_SHIFT; 571 end = end << MMU_SECTION_SHIFT; 572 flush_dcache_range(start, end); 573 asm volatile("dsb sy"); 574 } 575 #else 576 static bool is_aligned(u64 addr, u64 size, u64 align) 577 { 578 return !(addr & (align - 1)) && !(size & (align - 1)); 579 } 580 581 static u64 set_one_region(u64 start, u64 size, u64 attrs, int level) 582 { 583 int levelshift = level2shift(level); 584 u64 levelsize = 1ULL << levelshift; 585 u64 *pte = find_pte(start, level); 586 587 /* Can we can just modify the current level block PTE? */ 588 if (is_aligned(start, size, levelsize)) { 589 *pte &= ~PMD_ATTRINDX_MASK; 590 *pte |= attrs; 591 debug("Set attrs=%llx pte=%p level=%d\n", attrs, pte, level); 592 593 return levelsize; 594 } 595 596 /* Unaligned or doesn't fit, maybe split block into table */ 597 debug("addr=%llx level=%d pte=%p (%llx)\n", start, level, pte, *pte); 598 599 /* Maybe we need to split the block into a table */ 600 if (pte_type(pte) == PTE_TYPE_BLOCK) 601 split_block(pte, level); 602 603 /* And then double-check it became a table or already is one */ 604 if (pte_type(pte) != PTE_TYPE_TABLE) 605 panic("PTE %p (%llx) for addr=%llx should be a table", 606 pte, *pte, start); 607 608 /* Roll on to the next page table level */ 609 return 0; 610 } 611 612 void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size, 613 enum dcache_option option) 614 { 615 u64 attrs = PMD_ATTRINDX(option); 616 u64 real_start = start; 617 u64 real_size = size; 618 619 debug("start=%lx size=%lx\n", (ulong)start, (ulong)size); 620 621 /* 622 * We can not modify page tables that we're currently running on, 623 * so we first need to switch to the "emergency" page tables where 624 * we can safely modify our primary page tables and then switch back 625 */ 626 __asm_switch_ttbr(gd->arch.tlb_emerg); 627 628 /* 629 * Loop through the address range until we find a page granule that fits 630 * our alignment constraints, then set it to the new cache attributes 631 */ 632 while (size > 0) { 633 int level; 634 u64 r; 635 636 for (level = 1; level < 4; level++) { 637 r = set_one_region(start, size, attrs, level); 638 if (r) { 639 /* PTE successfully replaced */ 640 size -= r; 641 start += r; 642 break; 643 } 644 } 645 646 } 647 648 /* We're done modifying page tables, switch back to our primary ones */ 649 __asm_switch_ttbr(gd->arch.tlb_addr); 650 651 /* 652 * Make sure there's nothing stale in dcache for a region that might 653 * have caches off now 654 */ 655 flush_dcache_range(real_start, real_start + real_size); 656 } 657 #endif 658 659 #else /* CONFIG_SYS_DCACHE_OFF */ 660 661 void invalidate_dcache_all(void) 662 { 663 } 664 665 void flush_dcache_all(void) 666 { 667 } 668 669 void dcache_enable(void) 670 { 671 } 672 673 void dcache_disable(void) 674 { 675 } 676 677 int dcache_status(void) 678 { 679 return 0; 680 } 681 682 void mmu_set_region_dcache_behaviour(phys_addr_t start, size_t size, 683 enum dcache_option option) 684 { 685 } 686 687 #endif /* CONFIG_SYS_DCACHE_OFF */ 688 689 #ifndef CONFIG_SYS_ICACHE_OFF 690 691 void icache_enable(void) 692 { 693 __asm_invalidate_icache_all(); 694 set_sctlr(get_sctlr() | CR_I); 695 } 696 697 void icache_disable(void) 698 { 699 set_sctlr(get_sctlr() & ~CR_I); 700 } 701 702 int icache_status(void) 703 { 704 return (get_sctlr() & CR_I) != 0; 705 } 706 707 void invalidate_icache_all(void) 708 { 709 __asm_invalidate_icache_all(); 710 } 711 712 #else /* CONFIG_SYS_ICACHE_OFF */ 713 714 void icache_enable(void) 715 { 716 } 717 718 void icache_disable(void) 719 { 720 } 721 722 int icache_status(void) 723 { 724 return 0; 725 } 726 727 void invalidate_icache_all(void) 728 { 729 } 730 731 #endif /* CONFIG_SYS_ICACHE_OFF */ 732 733 /* 734 * Enable dCache & iCache, whether cache is actually enabled 735 * depend on CONFIG_SYS_DCACHE_OFF and CONFIG_SYS_ICACHE_OFF 736 */ 737 void __weak enable_caches(void) 738 { 739 icache_enable(); 740 dcache_enable(); 741 } 742