1c1cc1552SCatalin Marinas /* 2c1cc1552SCatalin Marinas * Based on arch/arm/mm/mmu.c 3c1cc1552SCatalin Marinas * 4c1cc1552SCatalin Marinas * Copyright (C) 1995-2005 Russell King 5c1cc1552SCatalin Marinas * Copyright (C) 2012 ARM Ltd. 6c1cc1552SCatalin Marinas * 7c1cc1552SCatalin Marinas * This program is free software; you can redistribute it and/or modify 8c1cc1552SCatalin Marinas * it under the terms of the GNU General Public License version 2 as 9c1cc1552SCatalin Marinas * published by the Free Software Foundation. 10c1cc1552SCatalin Marinas * 11c1cc1552SCatalin Marinas * This program is distributed in the hope that it will be useful, 12c1cc1552SCatalin Marinas * but WITHOUT ANY WARRANTY; without even the implied warranty of 13c1cc1552SCatalin Marinas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14c1cc1552SCatalin Marinas * GNU General Public License for more details. 15c1cc1552SCatalin Marinas * 16c1cc1552SCatalin Marinas * You should have received a copy of the GNU General Public License 17c1cc1552SCatalin Marinas * along with this program. If not, see <http://www.gnu.org/licenses/>. 18c1cc1552SCatalin Marinas */ 19c1cc1552SCatalin Marinas 20c1cc1552SCatalin Marinas #include <linux/export.h> 21c1cc1552SCatalin Marinas #include <linux/kernel.h> 22c1cc1552SCatalin Marinas #include <linux/errno.h> 23c1cc1552SCatalin Marinas #include <linux/init.h> 2461bd93ceSArd Biesheuvel #include <linux/libfdt.h> 25c1cc1552SCatalin Marinas #include <linux/mman.h> 26c1cc1552SCatalin Marinas #include <linux/nodemask.h> 27c1cc1552SCatalin Marinas #include <linux/memblock.h> 28c1cc1552SCatalin Marinas #include <linux/fs.h> 292475ff9dSCatalin Marinas #include <linux/io.h> 3041089357SCatalin Marinas #include <linux/slab.h> 31da141706SLaura Abbott #include <linux/stop_machine.h> 32c1cc1552SCatalin Marinas 33c1cc1552SCatalin Marinas #include <asm/cputype.h> 34af86e597SLaura Abbott #include <asm/fixmap.h> 35b433dce0SSuzuki K. Poulose #include <asm/kernel-pgtable.h> 36c1cc1552SCatalin Marinas #include <asm/sections.h> 37c1cc1552SCatalin Marinas #include <asm/setup.h> 38c1cc1552SCatalin Marinas #include <asm/sizes.h> 39c1cc1552SCatalin Marinas #include <asm/tlb.h> 40c79b954bSJungseok Lee #include <asm/memblock.h> 41c1cc1552SCatalin Marinas #include <asm/mmu_context.h> 42c1cc1552SCatalin Marinas 43c1cc1552SCatalin Marinas #include "mm.h" 44c1cc1552SCatalin Marinas 45dd006da2SArd Biesheuvel u64 idmap_t0sz = TCR_T0SZ(VA_BITS); 46dd006da2SArd Biesheuvel 47c1cc1552SCatalin Marinas /* 48c1cc1552SCatalin Marinas * Empty_zero_page is a special page that is used for zero-initialized data 49c1cc1552SCatalin Marinas * and COW. 50c1cc1552SCatalin Marinas */ 51c1cc1552SCatalin Marinas struct page *empty_zero_page; 52c1cc1552SCatalin Marinas EXPORT_SYMBOL(empty_zero_page); 53c1cc1552SCatalin Marinas 54c1cc1552SCatalin Marinas pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, 55c1cc1552SCatalin Marinas unsigned long size, pgprot_t vma_prot) 56c1cc1552SCatalin Marinas { 57c1cc1552SCatalin Marinas if (!pfn_valid(pfn)) 58c1cc1552SCatalin Marinas return pgprot_noncached(vma_prot); 59c1cc1552SCatalin Marinas else if (file->f_flags & O_SYNC) 60c1cc1552SCatalin Marinas return pgprot_writecombine(vma_prot); 61c1cc1552SCatalin Marinas return vma_prot; 62c1cc1552SCatalin Marinas } 63c1cc1552SCatalin Marinas EXPORT_SYMBOL(phys_mem_access_prot); 64c1cc1552SCatalin Marinas 65c1cc1552SCatalin Marinas static void __init *early_alloc(unsigned long sz) 66c1cc1552SCatalin Marinas { 67c1cc1552SCatalin Marinas void *ptr = __va(memblock_alloc(sz, sz)); 68da141706SLaura Abbott BUG_ON(!ptr); 69c1cc1552SCatalin Marinas memset(ptr, 0, sz); 70c1cc1552SCatalin Marinas return ptr; 71c1cc1552SCatalin Marinas } 72c1cc1552SCatalin Marinas 73da141706SLaura Abbott /* 74da141706SLaura Abbott * remap a PMD into pages 75da141706SLaura Abbott */ 76da141706SLaura Abbott static void split_pmd(pmd_t *pmd, pte_t *pte) 77da141706SLaura Abbott { 78da141706SLaura Abbott unsigned long pfn = pmd_pfn(*pmd); 79da141706SLaura Abbott int i = 0; 80da141706SLaura Abbott 81da141706SLaura Abbott do { 82da141706SLaura Abbott /* 83da141706SLaura Abbott * Need to have the least restrictive permissions available 84348a65cdSJeremy Linton * permissions will be fixed up later. Default the new page 85348a65cdSJeremy Linton * range as contiguous ptes. 86da141706SLaura Abbott */ 87348a65cdSJeremy Linton set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC_CONT)); 88da141706SLaura Abbott pfn++; 89da141706SLaura Abbott } while (pte++, i++, i < PTRS_PER_PTE); 90da141706SLaura Abbott } 91da141706SLaura Abbott 92348a65cdSJeremy Linton /* 93348a65cdSJeremy Linton * Given a PTE with the CONT bit set, determine where the CONT range 94348a65cdSJeremy Linton * starts, and clear the entire range of PTE CONT bits. 95348a65cdSJeremy Linton */ 96348a65cdSJeremy Linton static void clear_cont_pte_range(pte_t *pte, unsigned long addr) 97348a65cdSJeremy Linton { 98348a65cdSJeremy Linton int i; 99348a65cdSJeremy Linton 100348a65cdSJeremy Linton pte -= CONT_RANGE_OFFSET(addr); 101348a65cdSJeremy Linton for (i = 0; i < CONT_PTES; i++) { 102348a65cdSJeremy Linton set_pte(pte, pte_mknoncont(*pte)); 103348a65cdSJeremy Linton pte++; 104348a65cdSJeremy Linton } 105348a65cdSJeremy Linton flush_tlb_all(); 106348a65cdSJeremy Linton } 107348a65cdSJeremy Linton 108348a65cdSJeremy Linton /* 109348a65cdSJeremy Linton * Given a range of PTEs set the pfn and provided page protection flags 110348a65cdSJeremy Linton */ 111348a65cdSJeremy Linton static void __populate_init_pte(pte_t *pte, unsigned long addr, 112348a65cdSJeremy Linton unsigned long end, phys_addr_t phys, 113348a65cdSJeremy Linton pgprot_t prot) 114348a65cdSJeremy Linton { 115348a65cdSJeremy Linton unsigned long pfn = __phys_to_pfn(phys); 116348a65cdSJeremy Linton 117348a65cdSJeremy Linton do { 118348a65cdSJeremy Linton /* clear all the bits except the pfn, then apply the prot */ 119348a65cdSJeremy Linton set_pte(pte, pfn_pte(pfn, prot)); 120348a65cdSJeremy Linton pte++; 121348a65cdSJeremy Linton pfn++; 122348a65cdSJeremy Linton addr += PAGE_SIZE; 123348a65cdSJeremy Linton } while (addr != end); 124348a65cdSJeremy Linton } 125348a65cdSJeremy Linton 126da141706SLaura Abbott static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 127348a65cdSJeremy Linton unsigned long end, phys_addr_t phys, 128da141706SLaura Abbott pgprot_t prot, 129da141706SLaura Abbott void *(*alloc)(unsigned long size)) 130c1cc1552SCatalin Marinas { 131c1cc1552SCatalin Marinas pte_t *pte; 132348a65cdSJeremy Linton unsigned long next; 133c1cc1552SCatalin Marinas 134a1c76574SMark Rutland if (pmd_none(*pmd) || pmd_sect(*pmd)) { 135da141706SLaura Abbott pte = alloc(PTRS_PER_PTE * sizeof(pte_t)); 136da141706SLaura Abbott if (pmd_sect(*pmd)) 137da141706SLaura Abbott split_pmd(pmd, pte); 138c1cc1552SCatalin Marinas __pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE); 139da141706SLaura Abbott flush_tlb_all(); 140c1cc1552SCatalin Marinas } 141a1c76574SMark Rutland BUG_ON(pmd_bad(*pmd)); 142c1cc1552SCatalin Marinas 143c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 144c1cc1552SCatalin Marinas do { 145348a65cdSJeremy Linton next = min(end, (addr + CONT_SIZE) & CONT_MASK); 146348a65cdSJeremy Linton if (((addr | next | phys) & ~CONT_MASK) == 0) { 147348a65cdSJeremy Linton /* a block of CONT_PTES */ 148348a65cdSJeremy Linton __populate_init_pte(pte, addr, next, phys, 149b219545eSArd Biesheuvel __pgprot(pgprot_val(prot) | PTE_CONT)); 150348a65cdSJeremy Linton } else { 151348a65cdSJeremy Linton /* 152348a65cdSJeremy Linton * If the range being split is already inside of a 153348a65cdSJeremy Linton * contiguous range but this PTE isn't going to be 154348a65cdSJeremy Linton * contiguous, then we want to unmark the adjacent 155348a65cdSJeremy Linton * ranges, then update the portion of the range we 156348a65cdSJeremy Linton * are interrested in. 157348a65cdSJeremy Linton */ 158348a65cdSJeremy Linton clear_cont_pte_range(pte, addr); 159348a65cdSJeremy Linton __populate_init_pte(pte, addr, next, phys, prot); 160348a65cdSJeremy Linton } 161348a65cdSJeremy Linton 162348a65cdSJeremy Linton pte += (next - addr) >> PAGE_SHIFT; 163348a65cdSJeremy Linton phys += next - addr; 164348a65cdSJeremy Linton addr = next; 165348a65cdSJeremy Linton } while (addr != end); 166c1cc1552SCatalin Marinas } 167c1cc1552SCatalin Marinas 168da141706SLaura Abbott void split_pud(pud_t *old_pud, pmd_t *pmd) 169da141706SLaura Abbott { 170da141706SLaura Abbott unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT; 171da141706SLaura Abbott pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr); 172da141706SLaura Abbott int i = 0; 173da141706SLaura Abbott 174da141706SLaura Abbott do { 1751e43ba9cSArd Biesheuvel set_pmd(pmd, __pmd(addr | pgprot_val(prot))); 176da141706SLaura Abbott addr += PMD_SIZE; 177da141706SLaura Abbott } while (pmd++, i++, i < PTRS_PER_PMD); 178da141706SLaura Abbott } 179da141706SLaura Abbott 180da141706SLaura Abbott static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud, 181e1e1fddaSArd Biesheuvel unsigned long addr, unsigned long end, 182da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 183da141706SLaura Abbott void *(*alloc)(unsigned long size)) 184c1cc1552SCatalin Marinas { 185c1cc1552SCatalin Marinas pmd_t *pmd; 186c1cc1552SCatalin Marinas unsigned long next; 187c1cc1552SCatalin Marinas 188c1cc1552SCatalin Marinas /* 189c1cc1552SCatalin Marinas * Check for initial section mappings in the pgd/pud and remove them. 190c1cc1552SCatalin Marinas */ 191a1c76574SMark Rutland if (pud_none(*pud) || pud_sect(*pud)) { 192da141706SLaura Abbott pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t)); 193da141706SLaura Abbott if (pud_sect(*pud)) { 194da141706SLaura Abbott /* 195da141706SLaura Abbott * need to have the 1G of mappings continue to be 196da141706SLaura Abbott * present 197da141706SLaura Abbott */ 198da141706SLaura Abbott split_pud(pud, pmd); 199da141706SLaura Abbott } 200e1e1fddaSArd Biesheuvel pud_populate(mm, pud, pmd); 201da141706SLaura Abbott flush_tlb_all(); 202c1cc1552SCatalin Marinas } 203a1c76574SMark Rutland BUG_ON(pud_bad(*pud)); 204c1cc1552SCatalin Marinas 205c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 206c1cc1552SCatalin Marinas do { 207c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 208c1cc1552SCatalin Marinas /* try section mapping first */ 209a55f9929SCatalin Marinas if (((addr | next | phys) & ~SECTION_MASK) == 0) { 210a55f9929SCatalin Marinas pmd_t old_pmd =*pmd; 2118ce837ceSArd Biesheuvel set_pmd(pmd, __pmd(phys | 2128ce837ceSArd Biesheuvel pgprot_val(mk_sect_prot(prot)))); 213a55f9929SCatalin Marinas /* 214a55f9929SCatalin Marinas * Check for previous table entries created during 215a55f9929SCatalin Marinas * boot (__create_page_tables) and flush them. 216a55f9929SCatalin Marinas */ 217523d6e9fSzhichang.yuan if (!pmd_none(old_pmd)) { 218a55f9929SCatalin Marinas flush_tlb_all(); 219523d6e9fSzhichang.yuan if (pmd_table(old_pmd)) { 220523d6e9fSzhichang.yuan phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0)); 22141089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 222523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 223523d6e9fSzhichang.yuan } 224523d6e9fSzhichang.yuan } 225a55f9929SCatalin Marinas } else { 226348a65cdSJeremy Linton alloc_init_pte(pmd, addr, next, phys, prot, alloc); 227a55f9929SCatalin Marinas } 228c1cc1552SCatalin Marinas phys += next - addr; 229c1cc1552SCatalin Marinas } while (pmd++, addr = next, addr != end); 230c1cc1552SCatalin Marinas } 231c1cc1552SCatalin Marinas 232da141706SLaura Abbott static inline bool use_1G_block(unsigned long addr, unsigned long next, 233da141706SLaura Abbott unsigned long phys) 234da141706SLaura Abbott { 235da141706SLaura Abbott if (PAGE_SHIFT != 12) 236da141706SLaura Abbott return false; 237da141706SLaura Abbott 238da141706SLaura Abbott if (((addr | next | phys) & ~PUD_MASK) != 0) 239da141706SLaura Abbott return false; 240da141706SLaura Abbott 241da141706SLaura Abbott return true; 242da141706SLaura Abbott } 243da141706SLaura Abbott 244da141706SLaura Abbott static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd, 245e1e1fddaSArd Biesheuvel unsigned long addr, unsigned long end, 246da141706SLaura Abbott phys_addr_t phys, pgprot_t prot, 247da141706SLaura Abbott void *(*alloc)(unsigned long size)) 248c1cc1552SCatalin Marinas { 249c79b954bSJungseok Lee pud_t *pud; 250c1cc1552SCatalin Marinas unsigned long next; 251c1cc1552SCatalin Marinas 252c79b954bSJungseok Lee if (pgd_none(*pgd)) { 253da141706SLaura Abbott pud = alloc(PTRS_PER_PUD * sizeof(pud_t)); 254e1e1fddaSArd Biesheuvel pgd_populate(mm, pgd, pud); 255c79b954bSJungseok Lee } 256c79b954bSJungseok Lee BUG_ON(pgd_bad(*pgd)); 257c79b954bSJungseok Lee 258c79b954bSJungseok Lee pud = pud_offset(pgd, addr); 259c1cc1552SCatalin Marinas do { 260c1cc1552SCatalin Marinas next = pud_addr_end(addr, end); 261206a2a73SSteve Capper 262206a2a73SSteve Capper /* 263206a2a73SSteve Capper * For 4K granule only, attempt to put down a 1GB block 264206a2a73SSteve Capper */ 265da141706SLaura Abbott if (use_1G_block(addr, next, phys)) { 266206a2a73SSteve Capper pud_t old_pud = *pud; 2678ce837ceSArd Biesheuvel set_pud(pud, __pud(phys | 2688ce837ceSArd Biesheuvel pgprot_val(mk_sect_prot(prot)))); 269206a2a73SSteve Capper 270206a2a73SSteve Capper /* 271206a2a73SSteve Capper * If we have an old value for a pud, it will 272206a2a73SSteve Capper * be pointing to a pmd table that we no longer 273206a2a73SSteve Capper * need (from swapper_pg_dir). 274206a2a73SSteve Capper * 275206a2a73SSteve Capper * Look up the old pmd table and free it. 276206a2a73SSteve Capper */ 277206a2a73SSteve Capper if (!pud_none(old_pud)) { 278206a2a73SSteve Capper flush_tlb_all(); 279523d6e9fSzhichang.yuan if (pud_table(old_pud)) { 280523d6e9fSzhichang.yuan phys_addr_t table = __pa(pmd_offset(&old_pud, 0)); 28141089357SCatalin Marinas if (!WARN_ON_ONCE(slab_is_available())) 282523d6e9fSzhichang.yuan memblock_free(table, PAGE_SIZE); 283523d6e9fSzhichang.yuan } 284206a2a73SSteve Capper } 285206a2a73SSteve Capper } else { 286da141706SLaura Abbott alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc); 287206a2a73SSteve Capper } 288c1cc1552SCatalin Marinas phys += next - addr; 289c1cc1552SCatalin Marinas } while (pud++, addr = next, addr != end); 290c1cc1552SCatalin Marinas } 291c1cc1552SCatalin Marinas 292c1cc1552SCatalin Marinas /* 293c1cc1552SCatalin Marinas * Create the page directory entries and any necessary page tables for the 294c1cc1552SCatalin Marinas * mapping specified by 'md'. 295c1cc1552SCatalin Marinas */ 296da141706SLaura Abbott static void __create_mapping(struct mm_struct *mm, pgd_t *pgd, 297e1e1fddaSArd Biesheuvel phys_addr_t phys, unsigned long virt, 298da141706SLaura Abbott phys_addr_t size, pgprot_t prot, 299da141706SLaura Abbott void *(*alloc)(unsigned long size)) 300c1cc1552SCatalin Marinas { 301c1cc1552SCatalin Marinas unsigned long addr, length, end, next; 302c1cc1552SCatalin Marinas 303c1cc1552SCatalin Marinas addr = virt & PAGE_MASK; 304c1cc1552SCatalin Marinas length = PAGE_ALIGN(size + (virt & ~PAGE_MASK)); 305c1cc1552SCatalin Marinas 306c1cc1552SCatalin Marinas end = addr + length; 307c1cc1552SCatalin Marinas do { 308c1cc1552SCatalin Marinas next = pgd_addr_end(addr, end); 309da141706SLaura Abbott alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc); 310c1cc1552SCatalin Marinas phys += next - addr; 311c1cc1552SCatalin Marinas } while (pgd++, addr = next, addr != end); 312c1cc1552SCatalin Marinas } 313c1cc1552SCatalin Marinas 314da141706SLaura Abbott static void *late_alloc(unsigned long size) 315da141706SLaura Abbott { 316da141706SLaura Abbott void *ptr; 317da141706SLaura Abbott 318da141706SLaura Abbott BUG_ON(size > PAGE_SIZE); 319da141706SLaura Abbott ptr = (void *)__get_free_page(PGALLOC_GFP); 320da141706SLaura Abbott BUG_ON(!ptr); 321da141706SLaura Abbott return ptr; 322da141706SLaura Abbott } 323da141706SLaura Abbott 324c53e0baaSMark Rutland static void __init create_mapping(phys_addr_t phys, unsigned long virt, 325da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 326d7ecbddfSMark Salter { 327d7ecbddfSMark Salter if (virt < VMALLOC_START) { 328d7ecbddfSMark Salter pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 329d7ecbddfSMark Salter &phys, virt); 330d7ecbddfSMark Salter return; 331d7ecbddfSMark Salter } 332e1e1fddaSArd Biesheuvel __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt, 333da141706SLaura Abbott size, prot, early_alloc); 334d7ecbddfSMark Salter } 335d7ecbddfSMark Salter 3368ce837ceSArd Biesheuvel void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys, 3378ce837ceSArd Biesheuvel unsigned long virt, phys_addr_t size, 3388ce837ceSArd Biesheuvel pgprot_t prot) 3398ce837ceSArd Biesheuvel { 340da141706SLaura Abbott __create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot, 34160305db9SArd Biesheuvel late_alloc); 342d7ecbddfSMark Salter } 343d7ecbddfSMark Salter 344da141706SLaura Abbott static void create_mapping_late(phys_addr_t phys, unsigned long virt, 345da141706SLaura Abbott phys_addr_t size, pgprot_t prot) 346da141706SLaura Abbott { 347da141706SLaura Abbott if (virt < VMALLOC_START) { 348da141706SLaura Abbott pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n", 349da141706SLaura Abbott &phys, virt); 350da141706SLaura Abbott return; 351da141706SLaura Abbott } 352da141706SLaura Abbott 353da141706SLaura Abbott return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), 354da141706SLaura Abbott phys, virt, size, prot, late_alloc); 355da141706SLaura Abbott } 356da141706SLaura Abbott 357da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 358da141706SLaura Abbott static void __init __map_memblock(phys_addr_t start, phys_addr_t end) 359da141706SLaura Abbott { 360da141706SLaura Abbott /* 361da141706SLaura Abbott * Set up the executable regions using the existing section mappings 362da141706SLaura Abbott * for now. This will get more fine grained later once all memory 363da141706SLaura Abbott * is mapped 364da141706SLaura Abbott */ 365da141706SLaura Abbott unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE); 366da141706SLaura Abbott unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE); 367da141706SLaura Abbott 368da141706SLaura Abbott if (end < kernel_x_start) { 369da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 370da141706SLaura Abbott end - start, PAGE_KERNEL); 371da141706SLaura Abbott } else if (start >= kernel_x_end) { 372da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 373da141706SLaura Abbott end - start, PAGE_KERNEL); 374da141706SLaura Abbott } else { 375da141706SLaura Abbott if (start < kernel_x_start) 376da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), 377da141706SLaura Abbott kernel_x_start - start, 378da141706SLaura Abbott PAGE_KERNEL); 379da141706SLaura Abbott create_mapping(kernel_x_start, 380da141706SLaura Abbott __phys_to_virt(kernel_x_start), 381da141706SLaura Abbott kernel_x_end - kernel_x_start, 382da141706SLaura Abbott PAGE_KERNEL_EXEC); 383da141706SLaura Abbott if (kernel_x_end < end) 384da141706SLaura Abbott create_mapping(kernel_x_end, 385da141706SLaura Abbott __phys_to_virt(kernel_x_end), 386da141706SLaura Abbott end - kernel_x_end, 387da141706SLaura Abbott PAGE_KERNEL); 388da141706SLaura Abbott } 389da141706SLaura Abbott 390da141706SLaura Abbott } 391da141706SLaura Abbott #else 392da141706SLaura Abbott static void __init __map_memblock(phys_addr_t start, phys_addr_t end) 393da141706SLaura Abbott { 394da141706SLaura Abbott create_mapping(start, __phys_to_virt(start), end - start, 395da141706SLaura Abbott PAGE_KERNEL_EXEC); 396da141706SLaura Abbott } 397da141706SLaura Abbott #endif 398da141706SLaura Abbott 399c1cc1552SCatalin Marinas static void __init map_mem(void) 400c1cc1552SCatalin Marinas { 401c1cc1552SCatalin Marinas struct memblock_region *reg; 402e25208f7SCatalin Marinas phys_addr_t limit; 403c1cc1552SCatalin Marinas 404f6bc87c3SSteve Capper /* 405f6bc87c3SSteve Capper * Temporarily limit the memblock range. We need to do this as 406f6bc87c3SSteve Capper * create_mapping requires puds, pmds and ptes to be allocated from 407f6bc87c3SSteve Capper * memory addressable from the initial direct kernel mapping. 408f6bc87c3SSteve Capper * 4093dec0fe4SCatalin Marinas * The initial direct kernel mapping, located at swapper_pg_dir, gives 410b433dce0SSuzuki K. Poulose * us PUD_SIZE (with SECTION maps) or PMD_SIZE (without SECTION maps, 411b433dce0SSuzuki K. Poulose * memory starting from PHYS_OFFSET (which must be aligned to 2MB as 412b433dce0SSuzuki K. Poulose * per Documentation/arm64/booting.txt). 413f6bc87c3SSteve Capper */ 414b433dce0SSuzuki K. Poulose limit = PHYS_OFFSET + SWAPPER_INIT_MAP_SIZE; 415e25208f7SCatalin Marinas memblock_set_current_limit(limit); 416f6bc87c3SSteve Capper 417c1cc1552SCatalin Marinas /* map all the memory banks */ 418c1cc1552SCatalin Marinas for_each_memblock(memory, reg) { 419c1cc1552SCatalin Marinas phys_addr_t start = reg->base; 420c1cc1552SCatalin Marinas phys_addr_t end = start + reg->size; 421c1cc1552SCatalin Marinas 422c1cc1552SCatalin Marinas if (start >= end) 423c1cc1552SCatalin Marinas break; 424c1cc1552SCatalin Marinas 425b433dce0SSuzuki K. Poulose if (ARM64_SWAPPER_USES_SECTION_MAPS) { 426e25208f7SCatalin Marinas /* 427e25208f7SCatalin Marinas * For the first memory bank align the start address and 428e25208f7SCatalin Marinas * current memblock limit to prevent create_mapping() from 429b433dce0SSuzuki K. Poulose * allocating pte page tables from unmapped memory. With 430b433dce0SSuzuki K. Poulose * the section maps, if the first block doesn't end on section 431b433dce0SSuzuki K. Poulose * size boundary, create_mapping() will try to allocate a pte 432b433dce0SSuzuki K. Poulose * page, which may be returned from an unmapped area. 433b433dce0SSuzuki K. Poulose * When section maps are not used, the pte page table for the 434b433dce0SSuzuki K. Poulose * current limit is already present in swapper_pg_dir. 435e25208f7SCatalin Marinas */ 436e25208f7SCatalin Marinas if (start < limit) 437b433dce0SSuzuki K. Poulose start = ALIGN(start, SECTION_SIZE); 438e25208f7SCatalin Marinas if (end < limit) { 439b433dce0SSuzuki K. Poulose limit = end & SECTION_MASK; 440e25208f7SCatalin Marinas memblock_set_current_limit(limit); 441e25208f7SCatalin Marinas } 442b433dce0SSuzuki K. Poulose } 443da141706SLaura Abbott __map_memblock(start, end); 444c1cc1552SCatalin Marinas } 445f6bc87c3SSteve Capper 446f6bc87c3SSteve Capper /* Limit no longer required. */ 447f6bc87c3SSteve Capper memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE); 448c1cc1552SCatalin Marinas } 449c1cc1552SCatalin Marinas 450da141706SLaura Abbott void __init fixup_executable(void) 451da141706SLaura Abbott { 452da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 453da141706SLaura Abbott /* now that we are actually fully mapped, make the start/end more fine grained */ 454da141706SLaura Abbott if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) { 455da141706SLaura Abbott unsigned long aligned_start = round_down(__pa(_stext), 456da141706SLaura Abbott SECTION_SIZE); 457da141706SLaura Abbott 458da141706SLaura Abbott create_mapping(aligned_start, __phys_to_virt(aligned_start), 459da141706SLaura Abbott __pa(_stext) - aligned_start, 460da141706SLaura Abbott PAGE_KERNEL); 461da141706SLaura Abbott } 462da141706SLaura Abbott 463da141706SLaura Abbott if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) { 464da141706SLaura Abbott unsigned long aligned_end = round_up(__pa(__init_end), 465da141706SLaura Abbott SECTION_SIZE); 466da141706SLaura Abbott create_mapping(__pa(__init_end), (unsigned long)__init_end, 467da141706SLaura Abbott aligned_end - __pa(__init_end), 468da141706SLaura Abbott PAGE_KERNEL); 469da141706SLaura Abbott } 470da141706SLaura Abbott #endif 471da141706SLaura Abbott } 472da141706SLaura Abbott 473da141706SLaura Abbott #ifdef CONFIG_DEBUG_RODATA 474da141706SLaura Abbott void mark_rodata_ro(void) 475da141706SLaura Abbott { 476da141706SLaura Abbott create_mapping_late(__pa(_stext), (unsigned long)_stext, 477da141706SLaura Abbott (unsigned long)_etext - (unsigned long)_stext, 478da141706SLaura Abbott PAGE_KERNEL_EXEC | PTE_RDONLY); 479da141706SLaura Abbott 480da141706SLaura Abbott } 481da141706SLaura Abbott #endif 482da141706SLaura Abbott 483da141706SLaura Abbott void fixup_init(void) 484da141706SLaura Abbott { 485da141706SLaura Abbott create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin, 486da141706SLaura Abbott (unsigned long)__init_end - (unsigned long)__init_begin, 487da141706SLaura Abbott PAGE_KERNEL); 488da141706SLaura Abbott } 489da141706SLaura Abbott 490c1cc1552SCatalin Marinas /* 491c1cc1552SCatalin Marinas * paging_init() sets up the page tables, initialises the zone memory 492c1cc1552SCatalin Marinas * maps and sets up the zero page. 493c1cc1552SCatalin Marinas */ 494c1cc1552SCatalin Marinas void __init paging_init(void) 495c1cc1552SCatalin Marinas { 496c1cc1552SCatalin Marinas void *zero_page; 497c1cc1552SCatalin Marinas 498c1cc1552SCatalin Marinas map_mem(); 499da141706SLaura Abbott fixup_executable(); 500c1cc1552SCatalin Marinas 501c1cc1552SCatalin Marinas /* allocate the zero page. */ 502c1cc1552SCatalin Marinas zero_page = early_alloc(PAGE_SIZE); 503c1cc1552SCatalin Marinas 504c1cc1552SCatalin Marinas bootmem_init(); 505c1cc1552SCatalin Marinas 506c1cc1552SCatalin Marinas empty_zero_page = virt_to_page(zero_page); 507c1cc1552SCatalin Marinas 508c1cc1552SCatalin Marinas /* 509c1cc1552SCatalin Marinas * TTBR0 is only used for the identity mapping at this stage. Make it 510c1cc1552SCatalin Marinas * point to zero page to avoid speculatively fetching new entries. 511c1cc1552SCatalin Marinas */ 512c1cc1552SCatalin Marinas cpu_set_reserved_ttbr0(); 5138e63d388SWill Deacon local_flush_tlb_all(); 514dd006da2SArd Biesheuvel cpu_set_default_tcr_t0sz(); 515c1cc1552SCatalin Marinas } 516c1cc1552SCatalin Marinas 517c1cc1552SCatalin Marinas /* 518c1cc1552SCatalin Marinas * Check whether a kernel address is valid (derived from arch/x86/). 519c1cc1552SCatalin Marinas */ 520c1cc1552SCatalin Marinas int kern_addr_valid(unsigned long addr) 521c1cc1552SCatalin Marinas { 522c1cc1552SCatalin Marinas pgd_t *pgd; 523c1cc1552SCatalin Marinas pud_t *pud; 524c1cc1552SCatalin Marinas pmd_t *pmd; 525c1cc1552SCatalin Marinas pte_t *pte; 526c1cc1552SCatalin Marinas 527c1cc1552SCatalin Marinas if ((((long)addr) >> VA_BITS) != -1UL) 528c1cc1552SCatalin Marinas return 0; 529c1cc1552SCatalin Marinas 530c1cc1552SCatalin Marinas pgd = pgd_offset_k(addr); 531c1cc1552SCatalin Marinas if (pgd_none(*pgd)) 532c1cc1552SCatalin Marinas return 0; 533c1cc1552SCatalin Marinas 534c1cc1552SCatalin Marinas pud = pud_offset(pgd, addr); 535c1cc1552SCatalin Marinas if (pud_none(*pud)) 536c1cc1552SCatalin Marinas return 0; 537c1cc1552SCatalin Marinas 538206a2a73SSteve Capper if (pud_sect(*pud)) 539206a2a73SSteve Capper return pfn_valid(pud_pfn(*pud)); 540206a2a73SSteve Capper 541c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 542c1cc1552SCatalin Marinas if (pmd_none(*pmd)) 543c1cc1552SCatalin Marinas return 0; 544c1cc1552SCatalin Marinas 545da6e4cb6SDave Anderson if (pmd_sect(*pmd)) 546da6e4cb6SDave Anderson return pfn_valid(pmd_pfn(*pmd)); 547da6e4cb6SDave Anderson 548c1cc1552SCatalin Marinas pte = pte_offset_kernel(pmd, addr); 549c1cc1552SCatalin Marinas if (pte_none(*pte)) 550c1cc1552SCatalin Marinas return 0; 551c1cc1552SCatalin Marinas 552c1cc1552SCatalin Marinas return pfn_valid(pte_pfn(*pte)); 553c1cc1552SCatalin Marinas } 554c1cc1552SCatalin Marinas #ifdef CONFIG_SPARSEMEM_VMEMMAP 555b433dce0SSuzuki K. Poulose #if !ARM64_SWAPPER_USES_SECTION_MAPS 5560aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 557c1cc1552SCatalin Marinas { 5580aad818bSJohannes Weiner return vmemmap_populate_basepages(start, end, node); 559c1cc1552SCatalin Marinas } 560b433dce0SSuzuki K. Poulose #else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ 5610aad818bSJohannes Weiner int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 562c1cc1552SCatalin Marinas { 5630aad818bSJohannes Weiner unsigned long addr = start; 564c1cc1552SCatalin Marinas unsigned long next; 565c1cc1552SCatalin Marinas pgd_t *pgd; 566c1cc1552SCatalin Marinas pud_t *pud; 567c1cc1552SCatalin Marinas pmd_t *pmd; 568c1cc1552SCatalin Marinas 569c1cc1552SCatalin Marinas do { 570c1cc1552SCatalin Marinas next = pmd_addr_end(addr, end); 571c1cc1552SCatalin Marinas 572c1cc1552SCatalin Marinas pgd = vmemmap_pgd_populate(addr, node); 573c1cc1552SCatalin Marinas if (!pgd) 574c1cc1552SCatalin Marinas return -ENOMEM; 575c1cc1552SCatalin Marinas 576c1cc1552SCatalin Marinas pud = vmemmap_pud_populate(pgd, addr, node); 577c1cc1552SCatalin Marinas if (!pud) 578c1cc1552SCatalin Marinas return -ENOMEM; 579c1cc1552SCatalin Marinas 580c1cc1552SCatalin Marinas pmd = pmd_offset(pud, addr); 581c1cc1552SCatalin Marinas if (pmd_none(*pmd)) { 582c1cc1552SCatalin Marinas void *p = NULL; 583c1cc1552SCatalin Marinas 584c1cc1552SCatalin Marinas p = vmemmap_alloc_block_buf(PMD_SIZE, node); 585c1cc1552SCatalin Marinas if (!p) 586c1cc1552SCatalin Marinas return -ENOMEM; 587c1cc1552SCatalin Marinas 588a501e324SCatalin Marinas set_pmd(pmd, __pmd(__pa(p) | PROT_SECT_NORMAL)); 589c1cc1552SCatalin Marinas } else 590c1cc1552SCatalin Marinas vmemmap_verify((pte_t *)pmd, node, addr, next); 591c1cc1552SCatalin Marinas } while (addr = next, addr != end); 592c1cc1552SCatalin Marinas 593c1cc1552SCatalin Marinas return 0; 594c1cc1552SCatalin Marinas } 595c1cc1552SCatalin Marinas #endif /* CONFIG_ARM64_64K_PAGES */ 5960aad818bSJohannes Weiner void vmemmap_free(unsigned long start, unsigned long end) 5970197518cSTang Chen { 5980197518cSTang Chen } 599c1cc1552SCatalin Marinas #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 600af86e597SLaura Abbott 601af86e597SLaura Abbott static pte_t bm_pte[PTRS_PER_PTE] __page_aligned_bss; 6029f25e6adSKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS > 2 603af86e597SLaura Abbott static pmd_t bm_pmd[PTRS_PER_PMD] __page_aligned_bss; 604af86e597SLaura Abbott #endif 6059f25e6adSKirill A. Shutemov #if CONFIG_PGTABLE_LEVELS > 3 606af86e597SLaura Abbott static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss; 607af86e597SLaura Abbott #endif 608af86e597SLaura Abbott 609af86e597SLaura Abbott static inline pud_t * fixmap_pud(unsigned long addr) 610af86e597SLaura Abbott { 611af86e597SLaura Abbott pgd_t *pgd = pgd_offset_k(addr); 612af86e597SLaura Abbott 613af86e597SLaura Abbott BUG_ON(pgd_none(*pgd) || pgd_bad(*pgd)); 614af86e597SLaura Abbott 615af86e597SLaura Abbott return pud_offset(pgd, addr); 616af86e597SLaura Abbott } 617af86e597SLaura Abbott 618af86e597SLaura Abbott static inline pmd_t * fixmap_pmd(unsigned long addr) 619af86e597SLaura Abbott { 620af86e597SLaura Abbott pud_t *pud = fixmap_pud(addr); 621af86e597SLaura Abbott 622af86e597SLaura Abbott BUG_ON(pud_none(*pud) || pud_bad(*pud)); 623af86e597SLaura Abbott 624af86e597SLaura Abbott return pmd_offset(pud, addr); 625af86e597SLaura Abbott } 626af86e597SLaura Abbott 627af86e597SLaura Abbott static inline pte_t * fixmap_pte(unsigned long addr) 628af86e597SLaura Abbott { 629af86e597SLaura Abbott pmd_t *pmd = fixmap_pmd(addr); 630af86e597SLaura Abbott 631af86e597SLaura Abbott BUG_ON(pmd_none(*pmd) || pmd_bad(*pmd)); 632af86e597SLaura Abbott 633af86e597SLaura Abbott return pte_offset_kernel(pmd, addr); 634af86e597SLaura Abbott } 635af86e597SLaura Abbott 636af86e597SLaura Abbott void __init early_fixmap_init(void) 637af86e597SLaura Abbott { 638af86e597SLaura Abbott pgd_t *pgd; 639af86e597SLaura Abbott pud_t *pud; 640af86e597SLaura Abbott pmd_t *pmd; 641af86e597SLaura Abbott unsigned long addr = FIXADDR_START; 642af86e597SLaura Abbott 643af86e597SLaura Abbott pgd = pgd_offset_k(addr); 644af86e597SLaura Abbott pgd_populate(&init_mm, pgd, bm_pud); 645af86e597SLaura Abbott pud = pud_offset(pgd, addr); 646af86e597SLaura Abbott pud_populate(&init_mm, pud, bm_pmd); 647af86e597SLaura Abbott pmd = pmd_offset(pud, addr); 648af86e597SLaura Abbott pmd_populate_kernel(&init_mm, pmd, bm_pte); 649af86e597SLaura Abbott 650af86e597SLaura Abbott /* 651af86e597SLaura Abbott * The boot-ioremap range spans multiple pmds, for which 652af86e597SLaura Abbott * we are not preparted: 653af86e597SLaura Abbott */ 654af86e597SLaura Abbott BUILD_BUG_ON((__fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT) 655af86e597SLaura Abbott != (__fix_to_virt(FIX_BTMAP_END) >> PMD_SHIFT)); 656af86e597SLaura Abbott 657af86e597SLaura Abbott if ((pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN))) 658af86e597SLaura Abbott || pmd != fixmap_pmd(fix_to_virt(FIX_BTMAP_END))) { 659af86e597SLaura Abbott WARN_ON(1); 660af86e597SLaura Abbott pr_warn("pmd %p != %p, %p\n", 661af86e597SLaura Abbott pmd, fixmap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)), 662af86e597SLaura Abbott fixmap_pmd(fix_to_virt(FIX_BTMAP_END))); 663af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 664af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_BEGIN)); 665af86e597SLaura Abbott pr_warn("fix_to_virt(FIX_BTMAP_END): %08lx\n", 666af86e597SLaura Abbott fix_to_virt(FIX_BTMAP_END)); 667af86e597SLaura Abbott 668af86e597SLaura Abbott pr_warn("FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 669af86e597SLaura Abbott pr_warn("FIX_BTMAP_BEGIN: %d\n", FIX_BTMAP_BEGIN); 670af86e597SLaura Abbott } 671af86e597SLaura Abbott } 672af86e597SLaura Abbott 673af86e597SLaura Abbott void __set_fixmap(enum fixed_addresses idx, 674af86e597SLaura Abbott phys_addr_t phys, pgprot_t flags) 675af86e597SLaura Abbott { 676af86e597SLaura Abbott unsigned long addr = __fix_to_virt(idx); 677af86e597SLaura Abbott pte_t *pte; 678af86e597SLaura Abbott 679b63dbef9SMark Rutland BUG_ON(idx <= FIX_HOLE || idx >= __end_of_fixed_addresses); 680af86e597SLaura Abbott 681af86e597SLaura Abbott pte = fixmap_pte(addr); 682af86e597SLaura Abbott 683af86e597SLaura Abbott if (pgprot_val(flags)) { 684af86e597SLaura Abbott set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 685af86e597SLaura Abbott } else { 686af86e597SLaura Abbott pte_clear(&init_mm, addr, pte); 687af86e597SLaura Abbott flush_tlb_kernel_range(addr, addr+PAGE_SIZE); 688af86e597SLaura Abbott } 689af86e597SLaura Abbott } 69061bd93ceSArd Biesheuvel 69161bd93ceSArd Biesheuvel void *__init fixmap_remap_fdt(phys_addr_t dt_phys) 69261bd93ceSArd Biesheuvel { 69361bd93ceSArd Biesheuvel const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 69461bd93ceSArd Biesheuvel pgprot_t prot = PAGE_KERNEL | PTE_RDONLY; 695b433dce0SSuzuki K. Poulose int size, offset; 69661bd93ceSArd Biesheuvel void *dt_virt; 69761bd93ceSArd Biesheuvel 69861bd93ceSArd Biesheuvel /* 69961bd93ceSArd Biesheuvel * Check whether the physical FDT address is set and meets the minimum 70061bd93ceSArd Biesheuvel * alignment requirement. Since we are relying on MIN_FDT_ALIGN to be 70161bd93ceSArd Biesheuvel * at least 8 bytes so that we can always access the size field of the 70261bd93ceSArd Biesheuvel * FDT header after mapping the first chunk, double check here if that 70361bd93ceSArd Biesheuvel * is indeed the case. 70461bd93ceSArd Biesheuvel */ 70561bd93ceSArd Biesheuvel BUILD_BUG_ON(MIN_FDT_ALIGN < 8); 70661bd93ceSArd Biesheuvel if (!dt_phys || dt_phys % MIN_FDT_ALIGN) 70761bd93ceSArd Biesheuvel return NULL; 70861bd93ceSArd Biesheuvel 70961bd93ceSArd Biesheuvel /* 71061bd93ceSArd Biesheuvel * Make sure that the FDT region can be mapped without the need to 71161bd93ceSArd Biesheuvel * allocate additional translation table pages, so that it is safe 71261bd93ceSArd Biesheuvel * to call create_mapping() this early. 71361bd93ceSArd Biesheuvel * 71461bd93ceSArd Biesheuvel * On 64k pages, the FDT will be mapped using PTEs, so we need to 71561bd93ceSArd Biesheuvel * be in the same PMD as the rest of the fixmap. 71661bd93ceSArd Biesheuvel * On 4k pages, we'll use section mappings for the FDT so we only 71761bd93ceSArd Biesheuvel * have to be in the same PUD. 71861bd93ceSArd Biesheuvel */ 71961bd93ceSArd Biesheuvel BUILD_BUG_ON(dt_virt_base % SZ_2M); 72061bd93ceSArd Biesheuvel 721b433dce0SSuzuki K. Poulose BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT != 722b433dce0SSuzuki K. Poulose __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT); 72361bd93ceSArd Biesheuvel 724b433dce0SSuzuki K. Poulose offset = dt_phys % SWAPPER_BLOCK_SIZE; 72561bd93ceSArd Biesheuvel dt_virt = (void *)dt_virt_base + offset; 72661bd93ceSArd Biesheuvel 72761bd93ceSArd Biesheuvel /* map the first chunk so we can read the size from the header */ 728b433dce0SSuzuki K. Poulose create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 729b433dce0SSuzuki K. Poulose SWAPPER_BLOCK_SIZE, prot); 73061bd93ceSArd Biesheuvel 73161bd93ceSArd Biesheuvel if (fdt_check_header(dt_virt) != 0) 73261bd93ceSArd Biesheuvel return NULL; 73361bd93ceSArd Biesheuvel 73461bd93ceSArd Biesheuvel size = fdt_totalsize(dt_virt); 73561bd93ceSArd Biesheuvel if (size > MAX_FDT_SIZE) 73661bd93ceSArd Biesheuvel return NULL; 73761bd93ceSArd Biesheuvel 738b433dce0SSuzuki K. Poulose if (offset + size > SWAPPER_BLOCK_SIZE) 739b433dce0SSuzuki K. Poulose create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base, 740b433dce0SSuzuki K. Poulose round_up(offset + size, SWAPPER_BLOCK_SIZE), prot); 74161bd93ceSArd Biesheuvel 74261bd93ceSArd Biesheuvel memblock_reserve(dt_phys, size); 74361bd93ceSArd Biesheuvel 74461bd93ceSArd Biesheuvel return dt_virt; 74561bd93ceSArd Biesheuvel } 746