1c1cc1552SCatalin Marinas /* 2c1cc1552SCatalin Marinas * Based on arch/arm/mm/mmu.c 3c1cc1552SCatalin Marinas * 4c1cc1552SCatalin Marinas * Copyright (C) 1995-2005 Russell King 5c1cc1552SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 6c1cc1552SCatalin Marinas * 7c1cc1552SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8c1cc1552SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9c1cc1552SCatalin Marinas * published by the Free Software Foundation. 10c1cc1552SCatalin Marinas * 11c1cc1552SCatalin Marinas * This program is distributed in the hope that it will be useful, 12c1cc1552SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 13c1cc1552SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14c1cc1552SCatalin Marinas * GNU General Public License for more details. 15c1cc1552SCatalin Marinas * 16c1cc1552SCatalin Marinas * You should have received a copy of the GNU General Public License 17c1cc1552SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 18c1cc1552SCatalin Marinas */ 19c1cc1552SCatalin Marinas 20c1cc1552SCatalin Marinas #include <linux/export.h> 21c1cc1552SCatalin Marinas #include <linux/kernel.h> 22c1cc1552SCatalin Marinas #include <linux/errno.h> 23c1cc1552SCatalin Marinas #include <linux/init.h> 2461bd93ceSArd Biesheuvel #include <linux/libfdt.h> 25c1cc1552SCatalin Marinas #include <linux/mman.h> 26c1cc1552SCatalin Marinas #include <linux/nodemask.h> 27c1cc1552SCatalin Marinas #include <linux/memblock.h> 28c1cc1552SCatalin Marinas #include <linux/fs.h> 292475ff9dSCatalin Marinas #include <linux/io.h> 3041089357SCatalin Marinas #include <linux/slab.h> 31da141706SLaura Abbott #include <linux/stop_machine.h> 32c1cc1552SCatalin Marinas 33c1cc1552SCatalin Marinas #include <asm/cputype.h> 34af86e597SLaura Abbott #include <asm/fixmap.h> 35c1cc1552SCatalin Marinas #include <asm/sections.h> 36c1cc1552SCatalin Marinas #include <asm/setup.h> 37c1cc1552SCatalin Marinas #include <asm/sizes.h> 38c1cc1552SCatalin Marinas #include <asm/tlb.h> 39c79b954bSJungseok Lee #include <asm/memblock.h> 40c1cc1552SCatalin Marinas #include <asm/mmu_context.h> 41c1cc1552SCatalin Marinas 42c1cc1552SCatalin Marinas #include "mm.h" 43c1cc1552SCatalin Marinas 44dd006da2SArd Biesheuvel u64 idmap_t0sz = TCR_T0SZ(VA_BITS); 45dd006da2SArd Biesheuvel 46c1cc1552SCatalin Marinas /* 47c1cc1552SCatalin Marinas * Empty_zero_page is a special page that is used for zero-initialized data 48c1cc1552SCatalin Marinas * and COW. 49c1cc1552SCatalin Marinas */ 50c1cc1552SCatalin Marinas struct page *empty_zero_page; 51c1cc1552SCatalin Marinas EXPORT_SYMBOL(empty_zero_page); 52c1cc1552SCatalin Marinas 53c1cc1552SCatalin Marinas pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 54c1cc1552SCatalin Marinas unsigned long size, pgprot_t vma_prot) 55c1cc1552SCatalin Marinas { 56c1cc1552SCatalin Marinas if (!pfn_valid(pfn)) 57c1cc1552SCatalin Marinas return pgprot_noncached(vma_prot); 58c1cc1552SCatalin Marinas else if (file->f_flags & O_SYNC) 59c1cc1552SCatalin Marinas return pgprot_writecombine(vma_prot); 60c1cc1552SCatalin Marinas return vma_prot; 61c1cc1552SCatalin Marinas } 62c1cc1552SCatalin Marinas EXPORT_SYMBOL(phys_mem_access_prot); 63c1cc1552SCatalin Marinas 64c1cc1552SCatalin Marinas static void __init *early_alloc(unsigned long sz) 65c1cc1552SCatalin Marinas { 66c1cc1552SCatalin Marinas void *ptr = __va(memblock_alloc(sz, sz)); 67da141706SLaura Abbott BUG_ON(!ptr); 68c1cc1552SCatalin Marinas memset(ptr, 0, sz); 69c1cc1552SCatalin Marinas return ptr; 70c1cc1552SCatalin Marinas } 71c1cc1552SCatalin Marinas 72da141706SLaura Abbott /* 73da141706SLaura Abbott * remap a PMD into pages 74da141706SLaura Abbott */ 75da141706SLaura Abbott static void split_pmd(pmd_t *pmd, pte_t *pte) 76da141706SLaura Abbott { 77da141706SLaura Abbott unsigned long pfn = pmd_pfn(*pmd); 78da141706SLaura Abbott int i = 0; 79da141706SLaura Abbott 80da141706SLaura Abbott do { 81da141706SLaura Abbott /* 82da141706SLaura Abbott * Need to have the least restrictive permissions available 83348a65cdSJeremy Linton * permissions will be fixed up later. Default the new page 84348a65cdSJeremy Linton * range as contiguous ptes. 85da141706SLaura Abbott */ 86348a65cdSJeremy Linton set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC_CONT)); 87da141706SLaura Abbott pfn++; 88da141706SLaura Abbott } while (pte++, i++, i < PTRS_PER_PTE); 89da141706SLaura Abbott } 90da141706SLaura Abbott 91348a65cdSJeremy Linton /* 92348a65cdSJeremy Linton * Given a PTE with the CONT bit set, determine where the CONT range 93348a65cdSJeremy Linton * starts, and clear the entire range of PTE CONT bits. 94348a65cdSJeremy Linton */ 95348a65cdSJeremy Linton static void clear_cont_pte_range(pte_t *pte, unsigned long addr) 96348a65cdSJeremy Linton { 97348a65cdSJeremy Linton int i; 98348a65cdSJeremy Linton 99348a65cdSJeremy Linton pte -= CONT_RANGE_OFFSET(addr); 100348a65cdSJeremy Linton for (i = 0; i < CONT_PTES; i++) { 101348a65cdSJeremy Linton set_pte(pte, pte_mknoncont(*pte)); 102348a65cdSJeremy Linton pte++; 103348a65cdSJeremy Linton } 104348a65cdSJeremy Linton flush_tlb_all(); 105348a65cdSJeremy Linton } 106348a65cdSJeremy Linton 107348a65cdSJeremy Linton /* 108348a65cdSJeremy Linton * Given a range of PTEs set the pfn and provided page protection flags 109348a65cdSJeremy Linton */ 110348a65cdSJeremy Linton static void __populate_init_pte(pte_t *pte, unsigned long addr, 111348a65cdSJeremy Linton unsigned long end, phys_addr_t phys, 112348a65cdSJeremy Linton pgprot_t prot) 113348a65cdSJeremy Linton { 114348a65cdSJeremy Linton unsigned long pfn = __phys_to_pfn(phys); 115348a65cdSJeremy Linton 116348a65cdSJeremy Linton do { 117348a65cdSJeremy Linton /* clear all the bits except the pfn, then apply the prot */ 118348a65cdSJeremy Linton set_pte(pte, pfn_pte(pfn, prot)); 119348a65cdSJeremy Linton pte++; 120348a65cdSJeremy Linton pfn++; 121348a65cdSJeremy Linton addr += PAGE_SIZE; 122348a65cdSJeremy Linton } while (addr != end); 123348a65cdSJeremy Linton } 124348a65cdSJeremy Linton 125da141706SLaura Abbott static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 126348a65cdSJeremy Linton unsigned long end, phys_addr_t phys, 127da141706SLaura Abbott pgprot_t prot, 128da141706SLaura Abbott void *(*alloc)(unsigned long size)) 129c1cc1552SCatalin Marinas { 130c1cc1552SCatalin Marinas pte_t *pte; 131348a65cdSJeremy Linton unsigned long next; 132c1cc1552SCatalin Marinas 133a1c76574SMark Rutland if (pmd_none(*pmd) || pmd_sect(*pmd)) { 134da141706SLaura Abbott pte = alloc(PTRS_PER_PTE * sizeof(pte_t)); 135da141706SLaura Abbott if (pmd_sect(*pmd)) 136da141706SLaura Abbott split_pmd(pmd, pte); 137c1cc1552SCatalin Marinas __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE); 138da141706SLaura Abbott flush_tlb_all(); 139c1cc1552SCatalin Marinas } 140a1c76574SMark Rutland BUG_ON(pmd_bad(*pmd)); 141c1cc1552SCatalin Marinas 142c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 143c1cc1552SCatalin Marinas do { 144348a65cdSJeremy Linton next = min(end, (addr + CONT_SIZE) & CONT_MASK); 145348a65cdSJeremy Linton if (((addr | next | phys) & ~CONT_MASK) == 0) { 146348a65cdSJeremy Linton /* a block of CONT_PTES */ 147348a65cdSJeremy Linton __populate_init_pte(pte, addr, next, phys, 148348a65cdSJeremy Linton prot | __pgprot(PTE_CONT)); 149348a65cdSJeremy Linton } else { 150348a65cdSJeremy Linton /* 151348a65cdSJeremy Linton * If the range being split is already inside of a 152348a65cdSJeremy Linton * contiguous range but this PTE isn't going to be 153348a65cdSJeremy Linton * contiguous, then we want to unmark the adjacent 154348a65cdSJeremy Linton * ranges, then update the portion of the range we 155348a65cdSJeremy Linton * are interrested in. 156348a65cdSJeremy Linton */ 157348a65cdSJeremy Linton clear_cont_pte_range(pte, addr); 158348a65cdSJeremy Linton __populate_init_pte(pte, addr, next, phys, prot); 159348a65cdSJeremy Linton } 160348a65cdSJeremy Linton 161348a65cdSJeremy Linton pte += (next - addr) >> PAGE_SHIFT; 162348a65cdSJeremy Linton phys += next - addr; 163348a65cdSJeremy Linton addr = next; 164348a65cdSJeremy Linton } while (addr != end); 165c1cc1552SCatalin Marinas } 166c1cc1552SCatalin Marinas 167da141706SLaura Abbott void split_pud(pud_t *old_pud, pmd_t *pmd) 168da141706SLaura Abbott { 169da141706SLaura Abbott unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; 170da141706SLaura Abbott pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); 171da141706SLaura Abbott int i = 0; 172da141706SLaura Abbott 173da141706SLaura Abbott do { 1741e43ba9cSArd Biesheuvel set_pmd(pmd, __pmd(addr | pgprot_val(prot))); 175da141706SLaura Abbott addr += PMD_SIZE; 176da141706SLaura Abbott } while (pmd++, i++, i < PTRS_PER_PMD); 177da141706SLaura Abbott } 178da141706SLaura Abbott 179da141706SLaura Abbott static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, 180e1e1fddaSArd Biesheuvel unsigned long addr, unsigned long end, 181da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 182da141706SLaura Abbott void *(*alloc)(unsigned long size)) 183c1cc1552SCatalin Marinas { 184c1cc1552SCatalin Marinas pmd_t *pmd; 185c1cc1552SCatalin Marinas unsigned long next; 186c1cc1552SCatalin Marinas 187c1cc1552SCatalin Marinas /* 188c1cc1552SCatalin Marinas * Check for initial section mappings in the pgd/pud and remove them. 189c1cc1552SCatalin Marinas */ 190a1c76574SMark Rutland if (pud_none(*pud) || pud_sect(*pud)) { 191da141706SLaura Abbott pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t)); 192da141706SLaura Abbott if (pud_sect(*pud)) { 193da141706SLaura Abbott /* 194da141706SLaura Abbott * need to have the 1G of mappings continue to be 195da141706SLaura Abbott * present 196da141706SLaura Abbott */ 197da141706SLaura Abbott split_pud(pud, pmd); 198da141706SLaura Abbott } 199e1e1fddaSArd Biesheuvel pud_populate(mm, pud, pmd); 200da141706SLaura Abbott flush_tlb_all(); 201c1cc1552SCatalin Marinas } 202a1c76574SMark Rutland BUG_ON(pud_bad(*pud)); 203c1cc1552SCatalin Marinas 204c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 205c1cc1552SCatalin Marinas do { 206c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 207c1cc1552SCatalin Marinas /* try section mapping first */ 208a55f9929SCatalin Marinas if (((addr | next | phys) & ~SECTION_MASK) == 0) { 209a55f9929SCatalin Marinas pmd_t old_pmd =*pmd; 2108ce837ceSArd Biesheuvel set_pmd(pmd, __pmd(phys | 2118ce837ceSArd Biesheuvel pgprot_val(mk_sect_prot(prot)))); 212a55f9929SCatalin Marinas /* 213a55f9929SCatalin Marinas * Check for previous table entries created during 214a55f9929SCatalin Marinas * boot (__create_page_tables) and flush them. 215a55f9929SCatalin Marinas */ 216523d6e9fSzhichang.yuan if (!pmd_none(old_pmd)) { 217a55f9929SCatalin Marinas flush_tlb_all(); 218523d6e9fSzhichang.yuan if (pmd_table(old_pmd)) { 219523d6e9fSzhichang.yuan phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0)); 22041089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 221523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 222523d6e9fSzhichang.yuan } 223523d6e9fSzhichang.yuan } 224a55f9929SCatalin Marinas } else { 225348a65cdSJeremy Linton alloc_init_pte(pmd, addr, next, phys, prot, alloc); 226a55f9929SCatalin Marinas } 227c1cc1552SCatalin Marinas phys += next - addr; 228c1cc1552SCatalin Marinas } while (pmd++, addr = next, addr != end); 229c1cc1552SCatalin Marinas } 230c1cc1552SCatalin Marinas 231da141706SLaura Abbott static inline bool use_1G_block(unsigned long addr, unsigned long next, 232da141706SLaura Abbott unsigned long phys) 233da141706SLaura Abbott { 234da141706SLaura Abbott if (PAGE_SHIFT != 12) 235da141706SLaura Abbott return false; 236da141706SLaura Abbott 237da141706SLaura Abbott if (((addr | next | phys) & ~PUD_MASK) != 0) 238da141706SLaura Abbott return false; 239da141706SLaura Abbott 240da141706SLaura Abbott return true; 241da141706SLaura Abbott } 242da141706SLaura Abbott 243da141706SLaura Abbott static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, 244e1e1fddaSArd Biesheuvel unsigned long addr, unsigned long end, 245da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 246da141706SLaura Abbott void *(*alloc)(unsigned long size)) 247c1cc1552SCatalin Marinas { 248c79b954bSJungseok Lee pud_t *pud; 249c1cc1552SCatalin Marinas unsigned long next; 250c1cc1552SCatalin Marinas 251c79b954bSJungseok Lee if (pgd_none(*pgd)) { 252da141706SLaura Abbott pud = alloc(PTRS_PER_PUD * sizeof(pud_t)); 253e1e1fddaSArd Biesheuvel pgd_populate(mm, pgd, pud); 254c79b954bSJungseok Lee } 255c79b954bSJungseok Lee BUG_ON(pgd_bad(*pgd)); 256c79b954bSJungseok Lee 257c79b954bSJungseok Lee pud = pud_offset(pgd, addr); 258c1cc1552SCatalin Marinas do { 259c1cc1552SCatalin Marinas next = pud_addr_end(addr, end); 260206a2a73SSteve Capper 261206a2a73SSteve Capper /* 262206a2a73SSteve Capper * For 4K granule only, attempt to put down a 1GB block 263206a2a73SSteve Capper */ 264da141706SLaura Abbott if (use_1G_block(addr, next, phys)) { 265206a2a73SSteve Capper pud_t old_pud = *pud; 2668ce837ceSArd Biesheuvel set_pud(pud, __pud(phys | 2678ce837ceSArd Biesheuvel pgprot_val(mk_sect_prot(prot)))); 268206a2a73SSteve Capper 269206a2a73SSteve Capper /* 270206a2a73SSteve Capper * If we have an old value for a pud, it will 271206a2a73SSteve Capper * be pointing to a pmd table that we no longer 272206a2a73SSteve Capper * need (from swapper_pg_dir). 273206a2a73SSteve Capper * 274206a2a73SSteve Capper * Look up the old pmd table and free it. 275206a2a73SSteve Capper */ 276206a2a73SSteve Capper if (!pud_none(old_pud)) { 277206a2a73SSteve Capper flush_tlb_all(); 278523d6e9fSzhichang.yuan if (pud_table(old_pud)) { 279523d6e9fSzhichang.yuan phys_addr_t table = __pa(pmd_offset(&old_pud, 0)); 28041089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 281523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 282523d6e9fSzhichang.yuan } 283206a2a73SSteve Capper } 284206a2a73SSteve Capper } else { 285da141706SLaura Abbott alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc); 286206a2a73SSteve Capper } 287c1cc1552SCatalin Marinas phys += next - addr; 288c1cc1552SCatalin Marinas } while (pud++, addr = next, addr != end); 289c1cc1552SCatalin Marinas } 290c1cc1552SCatalin Marinas 291c1cc1552SCatalin Marinas /* 292c1cc1552SCatalin Marinas * Create the page directory entries and any necessary page tables for the 293c1cc1552SCatalin Marinas * mapping specified by 'md'. 294c1cc1552SCatalin Marinas */ 295da141706SLaura Abbott static void __create_mapping(struct mm_struct *mm, pgd_t *pgd, 296e1e1fddaSArd Biesheuvel phys_addr_t phys, unsigned long virt, 297da141706SLaura Abbott phys_addr_t size, pgprot_t prot, 298da141706SLaura Abbott void *(*alloc)(unsigned long size)) 299c1cc1552SCatalin Marinas { 300c1cc1552SCatalin Marinas unsigned long addr, length, end, next; 301c1cc1552SCatalin Marinas 302c1cc1552SCatalin Marinas addr = virt & PAGE_MASK; 303c1cc1552SCatalin Marinas length = PAGE_ALIGN(size + (virt & ~PAGE_MASK)); 304c1cc1552SCatalin Marinas 305c1cc1552SCatalin Marinas end = addr + length; 306c1cc1552SCatalin Marinas do { 307c1cc1552SCatalin Marinas next = pgd_addr_end(addr, end); 308da141706SLaura Abbott alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc); 309c1cc1552SCatalin Marinas phys += next - addr; 310c1cc1552SCatalin Marinas } while (pgd++, addr = next, addr != end); 311c1cc1552SCatalin Marinas } 312c1cc1552SCatalin Marinas 313da141706SLaura Abbott static void *late_alloc(unsigned long size) 314da141706SLaura Abbott { 315da141706SLaura Abbott void *ptr; 316da141706SLaura Abbott 317da141706SLaura Abbott BUG_ON(size > PAGE_SIZE); 318da141706SLaura Abbott ptr = (void *)__get_free_page(PGALLOC_GFP); 319da141706SLaura Abbott BUG_ON(!ptr); 320da141706SLaura Abbott return ptr; 321da141706SLaura Abbott } 322da141706SLaura Abbott 323c53e0baaSMark Rutland static void __init create_mapping(phys_addr_t phys, unsigned long virt, 324da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 325d7ecbddfSMark Salter { 326d7ecbddfSMark Salter if (virt < VMALLOC_START) { 327d7ecbddfSMark Salter pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 328d7ecbddfSMark Salter &phys, virt); 329d7ecbddfSMark Salter return; 330d7ecbddfSMark Salter } 331e1e1fddaSArd Biesheuvel __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt, 332da141706SLaura Abbott size, prot, early_alloc); 333d7ecbddfSMark Salter } 334d7ecbddfSMark Salter 3358ce837ceSArd Biesheuvel void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 3368ce837ceSArd Biesheuvel unsigned long virt, phys_addr_t size, 3378ce837ceSArd Biesheuvel pgprot_t prot) 3388ce837ceSArd Biesheuvel { 339da141706SLaura Abbott __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot, 34060305db9SArd Biesheuvel late_alloc); 341d7ecbddfSMark Salter } 342d7ecbddfSMark Salter 343da141706SLaura Abbott static void create_mapping_late(phys_addr_t phys, unsigned long virt, 344da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 345da141706SLaura Abbott { 346da141706SLaura Abbott if (virt < VMALLOC_START) { 347da141706SLaura Abbott pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 348da141706SLaura Abbott &phys, virt); 349da141706SLaura Abbott return; 350da141706SLaura Abbott } 351da141706SLaura Abbott 352da141706SLaura Abbott return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), 353da141706SLaura Abbott phys, virt, size, prot, late_alloc); 354da141706SLaura Abbott } 355da141706SLaura Abbott 356da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 357da141706SLaura Abbott static void __init __map_memblock(phys_addr_t start, phys_addr_t end) 358da141706SLaura Abbott { 359da141706SLaura Abbott /* 360da141706SLaura Abbott * Set up the executable regions using the existing section mappings 361da141706SLaura Abbott * for now. This will get more fine grained later once all memory 362da141706SLaura Abbott * is mapped 363da141706SLaura Abbott */ 364da141706SLaura Abbott unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE); 365da141706SLaura Abbott unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); 366da141706SLaura Abbott 367da141706SLaura Abbott if (end < kernel_x_start) { 368da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 369da141706SLaura Abbott end - start, PAGE_KERNEL); 370da141706SLaura Abbott } else if (start >= kernel_x_end) { 371da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 372da141706SLaura Abbott end - start, PAGE_KERNEL); 373da141706SLaura Abbott } else { 374da141706SLaura Abbott if (start < kernel_x_start) 375da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 376da141706SLaura Abbott kernel_x_start - start, 377da141706SLaura Abbott PAGE_KERNEL); 378da141706SLaura Abbott create_mapping(kernel_x_start, 379da141706SLaura Abbott __phys_to_virt(kernel_x_start), 380da141706SLaura Abbott kernel_x_end - kernel_x_start, 381da141706SLaura Abbott PAGE_KERNEL_EXEC); 382da141706SLaura Abbott if (kernel_x_end < end) 383da141706SLaura Abbott create_mapping(kernel_x_end, 384da141706SLaura Abbott __phys_to_virt(kernel_x_end), 385da141706SLaura Abbott end - kernel_x_end, 386da141706SLaura Abbott PAGE_KERNEL); 387da141706SLaura Abbott } 388da141706SLaura Abbott 389da141706SLaura Abbott } 390da141706SLaura Abbott #else 391da141706SLaura Abbott static void __init __map_memblock(phys_addr_t start, phys_addr_t end) 392da141706SLaura Abbott { 393da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), end - start, 394da141706SLaura Abbott PAGE_KERNEL_EXEC); 395da141706SLaura Abbott } 396da141706SLaura Abbott #endif 397da141706SLaura Abbott 398c1cc1552SCatalin Marinas static void __init map_mem(void) 399c1cc1552SCatalin Marinas { 400c1cc1552SCatalin Marinas struct memblock_region *reg; 401e25208f7SCatalin Marinas phys_addr_t limit; 402c1cc1552SCatalin Marinas 403f6bc87c3SSteve Capper /* 404f6bc87c3SSteve Capper * Temporarily limit the memblock range. We need to do this as 405f6bc87c3SSteve Capper * create_mapping requires puds, pmds and ptes to be allocated from 406f6bc87c3SSteve Capper * memory addressable from the initial direct kernel mapping. 407f6bc87c3SSteve Capper * 4083dec0fe4SCatalin Marinas * The initial direct kernel mapping, located at swapper_pg_dir, gives 4093dec0fe4SCatalin Marinas * us PUD_SIZE (4K pages) or PMD_SIZE (64K pages) memory starting from 4103dec0fe4SCatalin Marinas * PHYS_OFFSET (which must be aligned to 2MB as per 4113dec0fe4SCatalin Marinas * Documentation/arm64/booting.txt). 412f6bc87c3SSteve Capper */ 4133dec0fe4SCatalin Marinas if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) 4143dec0fe4SCatalin Marinas limit = PHYS_OFFSET + PMD_SIZE; 4153dec0fe4SCatalin Marinas else 416c79b954bSJungseok Lee limit = PHYS_OFFSET + PUD_SIZE; 417e25208f7SCatalin Marinas memblock_set_current_limit(limit); 418f6bc87c3SSteve Capper 419c1cc1552SCatalin Marinas /* map all the memory banks */ 420c1cc1552SCatalin Marinas for_each_memblock(memory, reg) { 421c1cc1552SCatalin Marinas phys_addr_t start = reg->base; 422c1cc1552SCatalin Marinas phys_addr_t end = start + reg->size; 423c1cc1552SCatalin Marinas 424c1cc1552SCatalin Marinas if (start >= end) 425c1cc1552SCatalin Marinas break; 426c1cc1552SCatalin Marinas 427e25208f7SCatalin Marinas #ifndef CONFIG_ARM64_64K_PAGES 428e25208f7SCatalin Marinas /* 429e25208f7SCatalin Marinas * For the first memory bank align the start address and 430e25208f7SCatalin Marinas * current memblock limit to prevent create_mapping() from 431e25208f7SCatalin Marinas * allocating pte page tables from unmapped memory. 432e25208f7SCatalin Marinas * When 64K pages are enabled, the pte page table for the 433e25208f7SCatalin Marinas * first PGDIR_SIZE is already present in swapper_pg_dir. 434e25208f7SCatalin Marinas */ 435e25208f7SCatalin Marinas if (start < limit) 436e25208f7SCatalin Marinas start = ALIGN(start, PMD_SIZE); 437e25208f7SCatalin Marinas if (end < limit) { 438e25208f7SCatalin Marinas limit = end & PMD_MASK; 439e25208f7SCatalin Marinas memblock_set_current_limit(limit); 440e25208f7SCatalin Marinas } 441e25208f7SCatalin Marinas #endif 442da141706SLaura Abbott __map_memblock(start, end); 443c1cc1552SCatalin Marinas } 444f6bc87c3SSteve Capper 445f6bc87c3SSteve Capper /* Limit no longer required. */ 446f6bc87c3SSteve Capper memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); 447c1cc1552SCatalin Marinas } 448c1cc1552SCatalin Marinas 449da141706SLaura Abbott void __init fixup_executable(void) 450da141706SLaura Abbott { 451da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 452da141706SLaura Abbott /* now that we are actually fully mapped, make the start/end more fine grained */ 453da141706SLaura Abbott if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) { 454da141706SLaura Abbott unsigned long aligned_start = round_down(__pa(_stext), 455da141706SLaura Abbott SECTION_SIZE); 456da141706SLaura Abbott 457da141706SLaura Abbott create_mapping(aligned_start, __phys_to_virt(aligned_start), 458da141706SLaura Abbott __pa(_stext) - aligned_start, 459da141706SLaura Abbott PAGE_KERNEL); 460da141706SLaura Abbott } 461da141706SLaura Abbott 462da141706SLaura Abbott if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) { 463da141706SLaura Abbott unsigned long aligned_end = round_up(__pa(__init_end), 464da141706SLaura Abbott SECTION_SIZE); 465da141706SLaura Abbott create_mapping(__pa(__init_end), (unsigned long)__init_end, 466da141706SLaura Abbott aligned_end - __pa(__init_end), 467da141706SLaura Abbott PAGE_KERNEL); 468da141706SLaura Abbott } 469da141706SLaura Abbott #endif 470da141706SLaura Abbott } 471da141706SLaura Abbott 472da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 473da141706SLaura Abbott void mark_rodata_ro(void) 474da141706SLaura Abbott { 475da141706SLaura Abbott create_mapping_late(__pa(_stext), (unsigned long)_stext, 476da141706SLaura Abbott (unsigned long)_etext - (unsigned long)_stext, 477da141706SLaura Abbott PAGE_KERNEL_EXEC | PTE_RDONLY); 478da141706SLaura Abbott 479da141706SLaura Abbott } 480da141706SLaura Abbott #endif 481da141706SLaura Abbott 482da141706SLaura Abbott void fixup_init(void) 483da141706SLaura Abbott { 484da141706SLaura Abbott create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin, 485da141706SLaura Abbott (unsigned long)__init_end - (unsigned long)__init_begin, 486da141706SLaura Abbott PAGE_KERNEL); 487da141706SLaura Abbott } 488da141706SLaura Abbott 489c1cc1552SCatalin Marinas /* 490c1cc1552SCatalin Marinas * paging_init() sets up the page tables, initialises the zone memory 491c1cc1552SCatalin Marinas * maps and sets up the zero page. 492c1cc1552SCatalin Marinas */ 493c1cc1552SCatalin Marinas void __init paging_init(void) 494c1cc1552SCatalin Marinas { 495c1cc1552SCatalin Marinas void *zero_page; 496c1cc1552SCatalin Marinas 497c1cc1552SCatalin Marinas map_mem(); 498da141706SLaura Abbott fixup_executable(); 499c1cc1552SCatalin Marinas 500c1cc1552SCatalin Marinas /* allocate the zero page. */ 501c1cc1552SCatalin Marinas zero_page = early_alloc(PAGE_SIZE); 502c1cc1552SCatalin Marinas 503c1cc1552SCatalin Marinas bootmem_init(); 504c1cc1552SCatalin Marinas 505c1cc1552SCatalin Marinas empty_zero_page = virt_to_page(zero_page); 506c1cc1552SCatalin Marinas 507c1cc1552SCatalin Marinas /* 508c1cc1552SCatalin Marinas * TTBR0 is only used for the identity mapping at this stage. Make it 509c1cc1552SCatalin Marinas * point to zero page to avoid speculatively fetching new entries. 510c1cc1552SCatalin Marinas */ 511c1cc1552SCatalin Marinas cpu_set_reserved_ttbr0(); 5128e63d388SWill Deacon local_flush_tlb_all(); 513dd006da2SArd Biesheuvel cpu_set_default_tcr_t0sz(); 514c1cc1552SCatalin Marinas } 515c1cc1552SCatalin Marinas 516c1cc1552SCatalin Marinas /* 517c1cc1552SCatalin Marinas * Check whether a kernel address is valid (derived from arch/x86/). 518c1cc1552SCatalin Marinas */ 519c1cc1552SCatalin Marinas int kern_addr_valid(unsigned long addr) 520c1cc1552SCatalin Marinas { 521c1cc1552SCatalin Marinas pgd_t *pgd; 522c1cc1552SCatalin Marinas pud_t *pud; 523c1cc1552SCatalin Marinas pmd_t *pmd; 524c1cc1552SCatalin Marinas pte_t *pte; 525c1cc1552SCatalin Marinas 526c1cc1552SCatalin Marinas if ((((long)addr) >> VA_BITS) != -1UL) 527c1cc1552SCatalin Marinas return 0; 528c1cc1552SCatalin Marinas 529c1cc1552SCatalin Marinas pgd = pgd_offset_k(addr); 530c1cc1552SCatalin Marinas if (pgd_none(*pgd)) 531c1cc1552SCatalin Marinas return 0; 532c1cc1552SCatalin Marinas 533c1cc1552SCatalin Marinas pud = pud_offset(pgd, addr); 534c1cc1552SCatalin Marinas if (pud_none(*pud)) 535c1cc1552SCatalin Marinas return 0; 536c1cc1552SCatalin Marinas 537206a2a73SSteve Capper if (pud_sect(*pud)) 538206a2a73SSteve Capper return pfn_valid(pud_pfn(*pud)); 539206a2a73SSteve Capper 540c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 541c1cc1552SCatalin Marinas if (pmd_none(*pmd)) 542c1cc1552SCatalin Marinas return 0; 543c1cc1552SCatalin Marinas 544da6e4cb6SDave Anderson if (pmd_sect(*pmd)) 545da6e4cb6SDave Anderson return pfn_valid(pmd_pfn(*pmd)); 546da6e4cb6SDave Anderson 547c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 548c1cc1552SCatalin Marinas if (pte_none(*pte)) 549c1cc1552SCatalin Marinas return 0; 550c1cc1552SCatalin Marinas 551c1cc1552SCatalin Marinas return pfn_valid(pte_pfn(*pte)); 552c1cc1552SCatalin Marinas } 553c1cc1552SCatalin Marinas #ifdef CONFIG_SPARSEMEM_VMEMMAP 554c1cc1552SCatalin Marinas #ifdef CONFIG_ARM64_64K_PAGES 5550aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 556c1cc1552SCatalin Marinas { 5570aad818bSJohannes Weiner return vmemmap_populate_basepages(start, end, node); 558c1cc1552SCatalin Marinas } 559c1cc1552SCatalin Marinas #else /* !CONFIG_ARM64_64K_PAGES */ 5600aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 561c1cc1552SCatalin Marinas { 5620aad818bSJohannes Weiner unsigned long addr = start; 563c1cc1552SCatalin Marinas unsigned long next; 564c1cc1552SCatalin Marinas pgd_t *pgd; 565c1cc1552SCatalin Marinas pud_t *pud; 566c1cc1552SCatalin Marinas pmd_t *pmd; 567c1cc1552SCatalin Marinas 568c1cc1552SCatalin Marinas do { 569c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 570c1cc1552SCatalin Marinas 571c1cc1552SCatalin Marinas pgd = vmemmap_pgd_populate(addr, node); 572c1cc1552SCatalin Marinas if (!pgd) 573c1cc1552SCatalin Marinas return -ENOMEM; 574c1cc1552SCatalin Marinas 575c1cc1552SCatalin Marinas pud = vmemmap_pud_populate(pgd, addr, node); 576c1cc1552SCatalin Marinas if (!pud) 577c1cc1552SCatalin Marinas return -ENOMEM; 578c1cc1552SCatalin Marinas 579c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 580c1cc1552SCatalin Marinas if (pmd_none(*pmd)) { 581c1cc1552SCatalin Marinas void *p = NULL; 582c1cc1552SCatalin Marinas 583c1cc1552SCatalin Marinas p = vmemmap_alloc_block_buf(PMD_SIZE, node); 584c1cc1552SCatalin Marinas if (!p) 585c1cc1552SCatalin Marinas return -ENOMEM; 586c1cc1552SCatalin Marinas 587a501e324SCatalin Marinas set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL)); 588c1cc1552SCatalin Marinas } else 589c1cc1552SCatalin Marinas vmemmap_verify((pte_t *)pmd, node, addr, next); 590c1cc1552SCatalin Marinas } while (addr = next, addr != end); 591c1cc1552SCatalin Marinas 592c1cc1552SCatalin Marinas return 0; 593c1cc1552SCatalin Marinas } 594c1cc1552SCatalin Marinas #endif /* CONFIG_ARM64_64K_PAGES */ 5950aad818bSJohannes Weiner void vmemmap_free(unsigned long start, unsigned long end) 5960197518cSTang Chen { 5970197518cSTang Chen } 598c1cc1552SCatalin Marinas #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 599af86e597SLaura Abbott 600af86e597SLaura Abbott static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 6019f25e6adSKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS > 2 602af86e597SLaura Abbott static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss; 603af86e597SLaura Abbott #endif 6049f25e6adSKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS > 3 605af86e597SLaura Abbott static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss; 606af86e597SLaura Abbott #endif 607af86e597SLaura Abbott 608af86e597SLaura Abbott static inline pud_t * fixmap_pud(unsigned long addr) 609af86e597SLaura Abbott { 610af86e597SLaura Abbott pgd_t *pgd = pgd_offset_k(addr); 611af86e597SLaura Abbott 612af86e597SLaura Abbott BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); 613af86e597SLaura Abbott 614af86e597SLaura Abbott return pud_offset(pgd, addr); 615af86e597SLaura Abbott } 616af86e597SLaura Abbott 617af86e597SLaura Abbott static inline pmd_t * fixmap_pmd(unsigned long addr) 618af86e597SLaura Abbott { 619af86e597SLaura Abbott pud_t *pud = fixmap_pud(addr); 620af86e597SLaura Abbott 621af86e597SLaura Abbott BUG_ON(pud_none(*pud) || pud_bad(*pud)); 622af86e597SLaura Abbott 623af86e597SLaura Abbott return pmd_offset(pud, addr); 624af86e597SLaura Abbott } 625af86e597SLaura Abbott 626af86e597SLaura Abbott static inline pte_t * fixmap_pte(unsigned long addr) 627af86e597SLaura Abbott { 628af86e597SLaura Abbott pmd_t *pmd = fixmap_pmd(addr); 629af86e597SLaura Abbott 630af86e597SLaura Abbott BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd)); 631af86e597SLaura Abbott 632af86e597SLaura Abbott return pte_offset_kernel(pmd, addr); 633af86e597SLaura Abbott } 634af86e597SLaura Abbott 635af86e597SLaura Abbott void __init early_fixmap_init(void) 636af86e597SLaura Abbott { 637af86e597SLaura Abbott pgd_t *pgd; 638af86e597SLaura Abbott pud_t *pud; 639af86e597SLaura Abbott pmd_t *pmd; 640af86e597SLaura Abbott unsigned long addr = FIXADDR_START; 641af86e597SLaura Abbott 642af86e597SLaura Abbott pgd = pgd_offset_k(addr); 643af86e597SLaura Abbott pgd_populate(&init_mm, pgd, bm_pud); 644af86e597SLaura Abbott pud = pud_offset(pgd, addr); 645af86e597SLaura Abbott pud_populate(&init_mm, pud, bm_pmd); 646af86e597SLaura Abbott pmd = pmd_offset(pud, addr); 647af86e597SLaura Abbott pmd_populate_kernel(&init_mm, pmd, bm_pte); 648af86e597SLaura Abbott 649af86e597SLaura Abbott /* 650af86e597SLaura Abbott * The boot-ioremap range spans multiple pmds, for which 651af86e597SLaura Abbott * we are not preparted: 652af86e597SLaura Abbott */ 653af86e597SLaura Abbott BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 654af86e597SLaura Abbott != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 655af86e597SLaura Abbott 656af86e597SLaura Abbott if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 657af86e597SLaura Abbott || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 658af86e597SLaura Abbott WARN_ON(1); 659af86e597SLaura Abbott pr_warn("pmd %p != %p, %p\n", 660af86e597SLaura Abbott pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 661af86e597SLaura Abbott fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 662af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 663af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_BEGIN)); 664af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 665af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_END)); 666af86e597SLaura Abbott 667af86e597SLaura Abbott pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 668af86e597SLaura Abbott pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 669af86e597SLaura Abbott } 670af86e597SLaura Abbott } 671af86e597SLaura Abbott 672af86e597SLaura Abbott void __set_fixmap(enum fixed_addresses idx, 673af86e597SLaura Abbott phys_addr_t phys, pgprot_t flags) 674af86e597SLaura Abbott { 675af86e597SLaura Abbott unsigned long addr = __fix_to_virt(idx); 676af86e597SLaura Abbott pte_t *pte; 677af86e597SLaura Abbott 678b63dbef9SMark Rutland BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 679af86e597SLaura Abbott 680af86e597SLaura Abbott pte = fixmap_pte(addr); 681af86e597SLaura Abbott 682af86e597SLaura Abbott if (pgprot_val(flags)) { 683af86e597SLaura Abbott set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 684af86e597SLaura Abbott } else { 685af86e597SLaura Abbott pte_clear(&init_mm, addr, pte); 686af86e597SLaura Abbott flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 687af86e597SLaura Abbott } 688af86e597SLaura Abbott } 68961bd93ceSArd Biesheuvel 69061bd93ceSArd Biesheuvel void *__init fixmap_remap_fdt(phys_addr_t dt_phys) 69161bd93ceSArd Biesheuvel { 69261bd93ceSArd Biesheuvel const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 69361bd93ceSArd Biesheuvel pgprot_t prot = PAGE_KERNEL | PTE_RDONLY; 69461bd93ceSArd Biesheuvel int granularity, size, offset; 69561bd93ceSArd Biesheuvel void *dt_virt; 69661bd93ceSArd Biesheuvel 69761bd93ceSArd Biesheuvel /* 69861bd93ceSArd Biesheuvel * Check whether the physical FDT address is set and meets the minimum 69961bd93ceSArd Biesheuvel * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 70061bd93ceSArd Biesheuvel * at least 8 bytes so that we can always access the size field of the 70161bd93ceSArd Biesheuvel * FDT header after mapping the first chunk, double check here if that 70261bd93ceSArd Biesheuvel * is indeed the case. 70361bd93ceSArd Biesheuvel */ 70461bd93ceSArd Biesheuvel BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 70561bd93ceSArd Biesheuvel if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 70661bd93ceSArd Biesheuvel return NULL; 70761bd93ceSArd Biesheuvel 70861bd93ceSArd Biesheuvel /* 70961bd93ceSArd Biesheuvel * Make sure that the FDT region can be mapped without the need to 71061bd93ceSArd Biesheuvel * allocate additional translation table pages, so that it is safe 71161bd93ceSArd Biesheuvel * to call create_mapping() this early. 71261bd93ceSArd Biesheuvel * 71361bd93ceSArd Biesheuvel * On 64k pages, the FDT will be mapped using PTEs, so we need to 71461bd93ceSArd Biesheuvel * be in the same PMD as the rest of the fixmap. 71561bd93ceSArd Biesheuvel * On 4k pages, we'll use section mappings for the FDT so we only 71661bd93ceSArd Biesheuvel * have to be in the same PUD. 71761bd93ceSArd Biesheuvel */ 71861bd93ceSArd Biesheuvel BUILD_BUG_ON(dt_virt_base % SZ_2M); 71961bd93ceSArd Biesheuvel 72061bd93ceSArd Biesheuvel if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) { 72161bd93ceSArd Biesheuvel BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> PMD_SHIFT != 72261bd93ceSArd Biesheuvel __fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT); 72361bd93ceSArd Biesheuvel 72461bd93ceSArd Biesheuvel granularity = PAGE_SIZE; 72561bd93ceSArd Biesheuvel } else { 72661bd93ceSArd Biesheuvel BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> PUD_SHIFT != 72761bd93ceSArd Biesheuvel __fix_to_virt(FIX_BTMAP_BEGIN) >> PUD_SHIFT); 72861bd93ceSArd Biesheuvel 72961bd93ceSArd Biesheuvel granularity = PMD_SIZE; 73061bd93ceSArd Biesheuvel } 73161bd93ceSArd Biesheuvel 73261bd93ceSArd Biesheuvel offset = dt_phys % granularity; 73361bd93ceSArd Biesheuvel dt_virt = (void *)dt_virt_base + offset; 73461bd93ceSArd Biesheuvel 73561bd93ceSArd Biesheuvel /* map the first chunk so we can read the size from the header */ 73661bd93ceSArd Biesheuvel create_mapping(round_down(dt_phys, granularity), dt_virt_base, 73761bd93ceSArd Biesheuvel granularity, prot); 73861bd93ceSArd Biesheuvel 73961bd93ceSArd Biesheuvel if (fdt_check_header(dt_virt) != 0) 74061bd93ceSArd Biesheuvel return NULL; 74161bd93ceSArd Biesheuvel 74261bd93ceSArd Biesheuvel size = fdt_totalsize(dt_virt); 74361bd93ceSArd Biesheuvel if (size > MAX_FDT_SIZE) 74461bd93ceSArd Biesheuvel return NULL; 74561bd93ceSArd Biesheuvel 74661bd93ceSArd Biesheuvel if (offset + size > granularity) 74761bd93ceSArd Biesheuvel create_mapping(round_down(dt_phys, granularity), dt_virt_base, 74861bd93ceSArd Biesheuvel round_up(offset + size, granularity), prot); 74961bd93ceSArd Biesheuvel 75061bd93ceSArd Biesheuvel memblock_reserve(dt_phys, size); 75161bd93ceSArd Biesheuvel 75261bd93ceSArd Biesheuvel return dt_virt; 75361bd93ceSArd Biesheuvel } 754