1e64c8aa0SThomas Gleixner /* 2e64c8aa0SThomas Gleixner * Re-map IO memory to kernel address space so that we can access it. 3e64c8aa0SThomas Gleixner * This is needed for high PCI addresses that aren't mapped in the 4e64c8aa0SThomas Gleixner * 640k-1MB IO memory area on PC's 5e64c8aa0SThomas Gleixner * 6e64c8aa0SThomas Gleixner * (C) Copyright 1995 1996 Linus Torvalds 7e64c8aa0SThomas Gleixner */ 8e64c8aa0SThomas Gleixner 9e64c8aa0SThomas Gleixner #include <linux/bootmem.h> 10e64c8aa0SThomas Gleixner #include <linux/init.h> 11e64c8aa0SThomas Gleixner #include <linux/io.h> 12e64c8aa0SThomas Gleixner #include <linux/module.h> 13e64c8aa0SThomas Gleixner #include <linux/slab.h> 14e64c8aa0SThomas Gleixner #include <linux/vmalloc.h> 15e64c8aa0SThomas Gleixner 16e64c8aa0SThomas Gleixner #include <asm/cacheflush.h> 17e64c8aa0SThomas Gleixner #include <asm/e820.h> 18e64c8aa0SThomas Gleixner #include <asm/fixmap.h> 19e64c8aa0SThomas Gleixner #include <asm/pgtable.h> 20e64c8aa0SThomas Gleixner #include <asm/tlbflush.h> 21f6df72e7SJeremy Fitzhardinge #include <asm/pgalloc.h> 22d7677d40Svenkatesh.pallipadi@intel.com #include <asm/pat.h> 23e64c8aa0SThomas Gleixner 24e64c8aa0SThomas Gleixner #ifdef CONFIG_X86_64 25e64c8aa0SThomas Gleixner 26e64c8aa0SThomas Gleixner unsigned long __phys_addr(unsigned long x) 27e64c8aa0SThomas Gleixner { 28e64c8aa0SThomas Gleixner if (x >= __START_KERNEL_map) 29e64c8aa0SThomas Gleixner return x - __START_KERNEL_map + phys_base; 30e64c8aa0SThomas Gleixner return x - PAGE_OFFSET; 31e64c8aa0SThomas Gleixner } 32e64c8aa0SThomas Gleixner EXPORT_SYMBOL(__phys_addr); 33e64c8aa0SThomas Gleixner 34e3100c82SThomas Gleixner static inline int phys_addr_valid(unsigned long addr) 35e3100c82SThomas Gleixner { 36e3100c82SThomas Gleixner return addr < (1UL << boot_cpu_data.x86_phys_bits); 37e3100c82SThomas Gleixner } 38e3100c82SThomas Gleixner 39e3100c82SThomas Gleixner #else 40e3100c82SThomas Gleixner 41e3100c82SThomas Gleixner static inline int phys_addr_valid(unsigned long addr) 42e3100c82SThomas Gleixner { 43e3100c82SThomas Gleixner return 1; 44e3100c82SThomas Gleixner } 45e3100c82SThomas Gleixner 46e64c8aa0SThomas Gleixner #endif 47e64c8aa0SThomas Gleixner 485f5192b9SThomas Gleixner int page_is_ram(unsigned long pagenr) 495f5192b9SThomas Gleixner { 505f5192b9SThomas Gleixner unsigned long addr, end; 515f5192b9SThomas Gleixner int i; 525f5192b9SThomas Gleixner 53d8a9e6a5SArjan van de Ven /* 54d8a9e6a5SArjan van de Ven * A special case is the first 4Kb of memory; 55d8a9e6a5SArjan van de Ven * This is a BIOS owned area, not kernel ram, but generally 56d8a9e6a5SArjan van de Ven * not listed as such in the E820 table. 57d8a9e6a5SArjan van de Ven */ 58d8a9e6a5SArjan van de Ven if (pagenr == 0) 59d8a9e6a5SArjan van de Ven return 0; 60d8a9e6a5SArjan van de Ven 61156fbc3fSArjan van de Ven /* 62156fbc3fSArjan van de Ven * Second special case: Some BIOSen report the PC BIOS 63156fbc3fSArjan van de Ven * area (640->1Mb) as ram even though it is not. 64156fbc3fSArjan van de Ven */ 65156fbc3fSArjan van de Ven if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) && 66156fbc3fSArjan van de Ven pagenr < (BIOS_END >> PAGE_SHIFT)) 67156fbc3fSArjan van de Ven return 0; 68d8a9e6a5SArjan van de Ven 695f5192b9SThomas Gleixner for (i = 0; i < e820.nr_map; i++) { 705f5192b9SThomas Gleixner /* 715f5192b9SThomas Gleixner * Not usable memory: 725f5192b9SThomas Gleixner */ 735f5192b9SThomas Gleixner if (e820.map[i].type != E820_RAM) 745f5192b9SThomas Gleixner continue; 755f5192b9SThomas Gleixner addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT; 765f5192b9SThomas Gleixner end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT; 77950f9d95SThomas Gleixner 78950f9d95SThomas Gleixner 795f5192b9SThomas Gleixner if ((pagenr >= addr) && (pagenr < end)) 805f5192b9SThomas Gleixner return 1; 815f5192b9SThomas Gleixner } 825f5192b9SThomas Gleixner return 0; 835f5192b9SThomas Gleixner } 845f5192b9SThomas Gleixner 85e64c8aa0SThomas Gleixner /* 86e64c8aa0SThomas Gleixner * Fix up the linear direct mapping of the kernel to avoid cache attribute 87e64c8aa0SThomas Gleixner * conflicts. 88e64c8aa0SThomas Gleixner */ 893a96ce8cSvenkatesh.pallipadi@intel.com int ioremap_change_attr(unsigned long vaddr, unsigned long size, 903a96ce8cSvenkatesh.pallipadi@intel.com unsigned long prot_val) 91e64c8aa0SThomas Gleixner { 92d806e5eeSThomas Gleixner unsigned long nrpages = size >> PAGE_SHIFT; 9393809be8SHarvey Harrison int err; 94e64c8aa0SThomas Gleixner 953a96ce8cSvenkatesh.pallipadi@intel.com switch (prot_val) { 963a96ce8cSvenkatesh.pallipadi@intel.com case _PAGE_CACHE_UC: 97d806e5eeSThomas Gleixner default: 981219333dSvenkatesh.pallipadi@intel.com err = _set_memory_uc(vaddr, nrpages); 99d806e5eeSThomas Gleixner break; 100b310f381Svenkatesh.pallipadi@intel.com case _PAGE_CACHE_WC: 101b310f381Svenkatesh.pallipadi@intel.com err = _set_memory_wc(vaddr, nrpages); 102b310f381Svenkatesh.pallipadi@intel.com break; 1033a96ce8cSvenkatesh.pallipadi@intel.com case _PAGE_CACHE_WB: 1041219333dSvenkatesh.pallipadi@intel.com err = _set_memory_wb(vaddr, nrpages); 105d806e5eeSThomas Gleixner break; 106d806e5eeSThomas Gleixner } 107e64c8aa0SThomas Gleixner 108e64c8aa0SThomas Gleixner return err; 109e64c8aa0SThomas Gleixner } 110e64c8aa0SThomas Gleixner 111e64c8aa0SThomas Gleixner /* 112e64c8aa0SThomas Gleixner * Remap an arbitrary physical address space into the kernel virtual 113e64c8aa0SThomas Gleixner * address space. Needed when the kernel wants to access high addresses 114e64c8aa0SThomas Gleixner * directly. 115e64c8aa0SThomas Gleixner * 116e64c8aa0SThomas Gleixner * NOTE! We need to allow non-page-aligned mappings too: we will obviously 117e64c8aa0SThomas Gleixner * have to convert them into an offset in a page-aligned mapping, but the 118e64c8aa0SThomas Gleixner * caller shouldn't need to know that small detail. 119e64c8aa0SThomas Gleixner */ 120b9e76a00SLinus Torvalds static void __iomem *__ioremap(resource_size_t phys_addr, unsigned long size, 1213a96ce8cSvenkatesh.pallipadi@intel.com unsigned long prot_val) 122e64c8aa0SThomas Gleixner { 123e66aadbeSThomas Gleixner unsigned long pfn, offset, last_addr, vaddr; 124e64c8aa0SThomas Gleixner struct vm_struct *area; 125d7677d40Svenkatesh.pallipadi@intel.com unsigned long new_prot_val; 126d806e5eeSThomas Gleixner pgprot_t prot; 127dee7cbb2SVenki Pallipadi int retval; 128e64c8aa0SThomas Gleixner 129e64c8aa0SThomas Gleixner /* Don't allow wraparound or zero size */ 130e64c8aa0SThomas Gleixner last_addr = phys_addr + size - 1; 131e64c8aa0SThomas Gleixner if (!size || last_addr < phys_addr) 132e64c8aa0SThomas Gleixner return NULL; 133e64c8aa0SThomas Gleixner 134e3100c82SThomas Gleixner if (!phys_addr_valid(phys_addr)) { 1356997ab49Svenkatesh.pallipadi@intel.com printk(KERN_WARNING "ioremap: invalid physical address %llx\n", 136e3100c82SThomas Gleixner phys_addr); 137e3100c82SThomas Gleixner WARN_ON_ONCE(1); 138e3100c82SThomas Gleixner return NULL; 139e3100c82SThomas Gleixner } 140e3100c82SThomas Gleixner 141e64c8aa0SThomas Gleixner /* 142e64c8aa0SThomas Gleixner * Don't remap the low PCI/ISA area, it's always mapped.. 143e64c8aa0SThomas Gleixner */ 144e64c8aa0SThomas Gleixner if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS) 145e64c8aa0SThomas Gleixner return (__force void __iomem *)phys_to_virt(phys_addr); 146e64c8aa0SThomas Gleixner 147e64c8aa0SThomas Gleixner /* 148e64c8aa0SThomas Gleixner * Don't allow anybody to remap normal RAM that we're using.. 149e64c8aa0SThomas Gleixner */ 150bdd3cee2SIngo Molnar for (pfn = phys_addr >> PAGE_SHIFT; 15138cb47baSIngo Molnar (pfn << PAGE_SHIFT) < last_addr; pfn++) { 152bdd3cee2SIngo Molnar 153ba748d22SIngo Molnar int is_ram = page_is_ram(pfn); 154ba748d22SIngo Molnar 155ba748d22SIngo Molnar if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn))) 156e64c8aa0SThomas Gleixner return NULL; 157ba748d22SIngo Molnar WARN_ON_ONCE(is_ram); 158e64c8aa0SThomas Gleixner } 159e64c8aa0SThomas Gleixner 160d7677d40Svenkatesh.pallipadi@intel.com /* 161d7677d40Svenkatesh.pallipadi@intel.com * Mappings have to be page-aligned 162d7677d40Svenkatesh.pallipadi@intel.com */ 163d7677d40Svenkatesh.pallipadi@intel.com offset = phys_addr & ~PAGE_MASK; 164d7677d40Svenkatesh.pallipadi@intel.com phys_addr &= PAGE_MASK; 165d7677d40Svenkatesh.pallipadi@intel.com size = PAGE_ALIGN(last_addr+1) - phys_addr; 166d7677d40Svenkatesh.pallipadi@intel.com 167dee7cbb2SVenki Pallipadi retval = reserve_memtype(phys_addr, phys_addr + size, 168dee7cbb2SVenki Pallipadi prot_val, &new_prot_val); 169dee7cbb2SVenki Pallipadi if (retval) { 170dee7cbb2SVenki Pallipadi printk("reserve_memtype returned %d\n", retval); 171dee7cbb2SVenki Pallipadi return NULL; 172dee7cbb2SVenki Pallipadi } 173dee7cbb2SVenki Pallipadi 174dee7cbb2SVenki Pallipadi if (prot_val != new_prot_val) { 175d7677d40Svenkatesh.pallipadi@intel.com /* 176d7677d40Svenkatesh.pallipadi@intel.com * Do not fallback to certain memory types with certain 177d7677d40Svenkatesh.pallipadi@intel.com * requested type: 178d7677d40Svenkatesh.pallipadi@intel.com * - request is uncached, return cannot be write-back 179b310f381Svenkatesh.pallipadi@intel.com * - request is uncached, return cannot be write-combine 180b310f381Svenkatesh.pallipadi@intel.com * - request is write-combine, return cannot be write-back 181d7677d40Svenkatesh.pallipadi@intel.com */ 182d7677d40Svenkatesh.pallipadi@intel.com if ((prot_val == _PAGE_CACHE_UC && 183b310f381Svenkatesh.pallipadi@intel.com (new_prot_val == _PAGE_CACHE_WB || 184b310f381Svenkatesh.pallipadi@intel.com new_prot_val == _PAGE_CACHE_WC)) || 185b310f381Svenkatesh.pallipadi@intel.com (prot_val == _PAGE_CACHE_WC && 186d7677d40Svenkatesh.pallipadi@intel.com new_prot_val == _PAGE_CACHE_WB)) { 1876997ab49Svenkatesh.pallipadi@intel.com printk( 1886997ab49Svenkatesh.pallipadi@intel.com "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n", 1896997ab49Svenkatesh.pallipadi@intel.com phys_addr, phys_addr + size, 1906997ab49Svenkatesh.pallipadi@intel.com prot_val, new_prot_val); 191d7677d40Svenkatesh.pallipadi@intel.com free_memtype(phys_addr, phys_addr + size); 192d7677d40Svenkatesh.pallipadi@intel.com return NULL; 193d7677d40Svenkatesh.pallipadi@intel.com } 194d7677d40Svenkatesh.pallipadi@intel.com prot_val = new_prot_val; 195d7677d40Svenkatesh.pallipadi@intel.com } 196d7677d40Svenkatesh.pallipadi@intel.com 1973a96ce8cSvenkatesh.pallipadi@intel.com switch (prot_val) { 1983a96ce8cSvenkatesh.pallipadi@intel.com case _PAGE_CACHE_UC: 199d806e5eeSThomas Gleixner default: 20055c62682SIngo Molnar prot = PAGE_KERNEL_NOCACHE; 201d806e5eeSThomas Gleixner break; 202b310f381Svenkatesh.pallipadi@intel.com case _PAGE_CACHE_WC: 203b310f381Svenkatesh.pallipadi@intel.com prot = PAGE_KERNEL_WC; 204b310f381Svenkatesh.pallipadi@intel.com break; 2053a96ce8cSvenkatesh.pallipadi@intel.com case _PAGE_CACHE_WB: 206d806e5eeSThomas Gleixner prot = PAGE_KERNEL; 207d806e5eeSThomas Gleixner break; 208d806e5eeSThomas Gleixner } 209e64c8aa0SThomas Gleixner 210e64c8aa0SThomas Gleixner /* 211e64c8aa0SThomas Gleixner * Ok, go for it.. 212e64c8aa0SThomas Gleixner */ 213e64c8aa0SThomas Gleixner area = get_vm_area(size, VM_IOREMAP); 214e64c8aa0SThomas Gleixner if (!area) 215e64c8aa0SThomas Gleixner return NULL; 216e64c8aa0SThomas Gleixner area->phys_addr = phys_addr; 217e66aadbeSThomas Gleixner vaddr = (unsigned long) area->addr; 218e66aadbeSThomas Gleixner if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) { 219d7677d40Svenkatesh.pallipadi@intel.com free_memtype(phys_addr, phys_addr + size); 220b16bf712SIngo Molnar free_vm_area(area); 221e64c8aa0SThomas Gleixner return NULL; 222e64c8aa0SThomas Gleixner } 223e64c8aa0SThomas Gleixner 2243a96ce8cSvenkatesh.pallipadi@intel.com if (ioremap_change_attr(vaddr, size, prot_val) < 0) { 225d7677d40Svenkatesh.pallipadi@intel.com free_memtype(phys_addr, phys_addr + size); 226e66aadbeSThomas Gleixner vunmap(area->addr); 227e64c8aa0SThomas Gleixner return NULL; 228e64c8aa0SThomas Gleixner } 229e64c8aa0SThomas Gleixner 230e66aadbeSThomas Gleixner return (void __iomem *) (vaddr + offset); 231e64c8aa0SThomas Gleixner } 232e64c8aa0SThomas Gleixner 233e64c8aa0SThomas Gleixner /** 234e64c8aa0SThomas Gleixner * ioremap_nocache - map bus memory into CPU space 235e64c8aa0SThomas Gleixner * @offset: bus address of the memory 236e64c8aa0SThomas Gleixner * @size: size of the resource to map 237e64c8aa0SThomas Gleixner * 238e64c8aa0SThomas Gleixner * ioremap_nocache performs a platform specific sequence of operations to 239e64c8aa0SThomas Gleixner * make bus memory CPU accessible via the readb/readw/readl/writeb/ 240e64c8aa0SThomas Gleixner * writew/writel functions and the other mmio helpers. The returned 241e64c8aa0SThomas Gleixner * address is not guaranteed to be usable directly as a virtual 242e64c8aa0SThomas Gleixner * address. 243e64c8aa0SThomas Gleixner * 244e64c8aa0SThomas Gleixner * This version of ioremap ensures that the memory is marked uncachable 245e64c8aa0SThomas Gleixner * on the CPU as well as honouring existing caching rules from things like 246e64c8aa0SThomas Gleixner * the PCI bus. Note that there are other caches and buffers on many 247e64c8aa0SThomas Gleixner * busses. In particular driver authors should read up on PCI writes 248e64c8aa0SThomas Gleixner * 249e64c8aa0SThomas Gleixner * It's useful if some control registers are in such an area and 250e64c8aa0SThomas Gleixner * write combining or read caching is not desirable: 251e64c8aa0SThomas Gleixner * 252e64c8aa0SThomas Gleixner * Must be freed with iounmap. 253e64c8aa0SThomas Gleixner */ 254b9e76a00SLinus Torvalds void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size) 255e64c8aa0SThomas Gleixner { 2563a96ce8cSvenkatesh.pallipadi@intel.com return __ioremap(phys_addr, size, _PAGE_CACHE_UC); 257e64c8aa0SThomas Gleixner } 258e64c8aa0SThomas Gleixner EXPORT_SYMBOL(ioremap_nocache); 259e64c8aa0SThomas Gleixner 260b310f381Svenkatesh.pallipadi@intel.com /** 261b310f381Svenkatesh.pallipadi@intel.com * ioremap_wc - map memory into CPU space write combined 262b310f381Svenkatesh.pallipadi@intel.com * @offset: bus address of the memory 263b310f381Svenkatesh.pallipadi@intel.com * @size: size of the resource to map 264b310f381Svenkatesh.pallipadi@intel.com * 265b310f381Svenkatesh.pallipadi@intel.com * This version of ioremap ensures that the memory is marked write combining. 266b310f381Svenkatesh.pallipadi@intel.com * Write combining allows faster writes to some hardware devices. 267b310f381Svenkatesh.pallipadi@intel.com * 268b310f381Svenkatesh.pallipadi@intel.com * Must be freed with iounmap. 269b310f381Svenkatesh.pallipadi@intel.com */ 270b310f381Svenkatesh.pallipadi@intel.com void __iomem *ioremap_wc(unsigned long phys_addr, unsigned long size) 271b310f381Svenkatesh.pallipadi@intel.com { 272b310f381Svenkatesh.pallipadi@intel.com if (pat_wc_enabled) 273b310f381Svenkatesh.pallipadi@intel.com return __ioremap(phys_addr, size, _PAGE_CACHE_WC); 274b310f381Svenkatesh.pallipadi@intel.com else 275b310f381Svenkatesh.pallipadi@intel.com return ioremap_nocache(phys_addr, size); 276b310f381Svenkatesh.pallipadi@intel.com } 277b310f381Svenkatesh.pallipadi@intel.com EXPORT_SYMBOL(ioremap_wc); 278b310f381Svenkatesh.pallipadi@intel.com 279b9e76a00SLinus Torvalds void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size) 2805f868152SThomas Gleixner { 2813a96ce8cSvenkatesh.pallipadi@intel.com return __ioremap(phys_addr, size, _PAGE_CACHE_WB); 2825f868152SThomas Gleixner } 2835f868152SThomas Gleixner EXPORT_SYMBOL(ioremap_cache); 2845f868152SThomas Gleixner 285e64c8aa0SThomas Gleixner /** 286e64c8aa0SThomas Gleixner * iounmap - Free a IO remapping 287e64c8aa0SThomas Gleixner * @addr: virtual address from ioremap_* 288e64c8aa0SThomas Gleixner * 289e64c8aa0SThomas Gleixner * Caller must ensure there is only one unmapping for the same pointer. 290e64c8aa0SThomas Gleixner */ 291e64c8aa0SThomas Gleixner void iounmap(volatile void __iomem *addr) 292e64c8aa0SThomas Gleixner { 293e64c8aa0SThomas Gleixner struct vm_struct *p, *o; 294e64c8aa0SThomas Gleixner 295e64c8aa0SThomas Gleixner if ((void __force *)addr <= high_memory) 296e64c8aa0SThomas Gleixner return; 297e64c8aa0SThomas Gleixner 298e64c8aa0SThomas Gleixner /* 299e64c8aa0SThomas Gleixner * __ioremap special-cases the PCI/ISA range by not instantiating a 300e64c8aa0SThomas Gleixner * vm_area and by simply returning an address into the kernel mapping 301e64c8aa0SThomas Gleixner * of ISA space. So handle that here. 302e64c8aa0SThomas Gleixner */ 303e64c8aa0SThomas Gleixner if (addr >= phys_to_virt(ISA_START_ADDRESS) && 304e64c8aa0SThomas Gleixner addr < phys_to_virt(ISA_END_ADDRESS)) 305e64c8aa0SThomas Gleixner return; 306e64c8aa0SThomas Gleixner 307e64c8aa0SThomas Gleixner addr = (volatile void __iomem *) 308e64c8aa0SThomas Gleixner (PAGE_MASK & (unsigned long __force)addr); 309e64c8aa0SThomas Gleixner 310e64c8aa0SThomas Gleixner /* Use the vm area unlocked, assuming the caller 311e64c8aa0SThomas Gleixner ensures there isn't another iounmap for the same address 312e64c8aa0SThomas Gleixner in parallel. Reuse of the virtual address is prevented by 313e64c8aa0SThomas Gleixner leaving it in the global lists until we're done with it. 314e64c8aa0SThomas Gleixner cpa takes care of the direct mappings. */ 315e64c8aa0SThomas Gleixner read_lock(&vmlist_lock); 316e64c8aa0SThomas Gleixner for (p = vmlist; p; p = p->next) { 317e64c8aa0SThomas Gleixner if (p->addr == addr) 318e64c8aa0SThomas Gleixner break; 319e64c8aa0SThomas Gleixner } 320e64c8aa0SThomas Gleixner read_unlock(&vmlist_lock); 321e64c8aa0SThomas Gleixner 322e64c8aa0SThomas Gleixner if (!p) { 323e64c8aa0SThomas Gleixner printk(KERN_ERR "iounmap: bad address %p\n", addr); 324e64c8aa0SThomas Gleixner dump_stack(); 325e64c8aa0SThomas Gleixner return; 326e64c8aa0SThomas Gleixner } 327e64c8aa0SThomas Gleixner 328d7677d40Svenkatesh.pallipadi@intel.com free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p)); 329d7677d40Svenkatesh.pallipadi@intel.com 330e64c8aa0SThomas Gleixner /* Finally remove it */ 331e64c8aa0SThomas Gleixner o = remove_vm_area((void *)addr); 332e64c8aa0SThomas Gleixner BUG_ON(p != o || o == NULL); 333e64c8aa0SThomas Gleixner kfree(p); 334e64c8aa0SThomas Gleixner } 335e64c8aa0SThomas Gleixner EXPORT_SYMBOL(iounmap); 336e64c8aa0SThomas Gleixner 337e64c8aa0SThomas Gleixner #ifdef CONFIG_X86_32 338e64c8aa0SThomas Gleixner 339e64c8aa0SThomas Gleixner int __initdata early_ioremap_debug; 340e64c8aa0SThomas Gleixner 341e64c8aa0SThomas Gleixner static int __init early_ioremap_debug_setup(char *str) 342e64c8aa0SThomas Gleixner { 343e64c8aa0SThomas Gleixner early_ioremap_debug = 1; 344e64c8aa0SThomas Gleixner 345e64c8aa0SThomas Gleixner return 0; 346e64c8aa0SThomas Gleixner } 347e64c8aa0SThomas Gleixner early_param("early_ioremap_debug", early_ioremap_debug_setup); 348e64c8aa0SThomas Gleixner 349e64c8aa0SThomas Gleixner static __initdata int after_paging_init; 350c92a7a54SIan Campbell static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] 351c92a7a54SIan Campbell __section(.bss.page_aligned); 352e64c8aa0SThomas Gleixner 353551889a6SIan Campbell static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) 354e64c8aa0SThomas Gleixner { 35537cc8d7fSJeremy Fitzhardinge /* Don't assume we're using swapper_pg_dir at this point */ 35637cc8d7fSJeremy Fitzhardinge pgd_t *base = __va(read_cr3()); 35737cc8d7fSJeremy Fitzhardinge pgd_t *pgd = &base[pgd_index(addr)]; 358551889a6SIan Campbell pud_t *pud = pud_offset(pgd, addr); 359551889a6SIan Campbell pmd_t *pmd = pmd_offset(pud, addr); 360551889a6SIan Campbell 361551889a6SIan Campbell return pmd; 362e64c8aa0SThomas Gleixner } 363e64c8aa0SThomas Gleixner 364551889a6SIan Campbell static inline pte_t * __init early_ioremap_pte(unsigned long addr) 365e64c8aa0SThomas Gleixner { 366551889a6SIan Campbell return &bm_pte[pte_index(addr)]; 367e64c8aa0SThomas Gleixner } 368e64c8aa0SThomas Gleixner 369e64c8aa0SThomas Gleixner void __init early_ioremap_init(void) 370e64c8aa0SThomas Gleixner { 371551889a6SIan Campbell pmd_t *pmd; 372e64c8aa0SThomas Gleixner 373e64c8aa0SThomas Gleixner if (early_ioremap_debug) 374adafdf6aSIngo Molnar printk(KERN_INFO "early_ioremap_init()\n"); 375e64c8aa0SThomas Gleixner 376551889a6SIan Campbell pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); 377e64c8aa0SThomas Gleixner memset(bm_pte, 0, sizeof(bm_pte)); 378b6fbb669SIan Campbell pmd_populate_kernel(&init_mm, pmd, bm_pte); 379551889a6SIan Campbell 380e64c8aa0SThomas Gleixner /* 381551889a6SIan Campbell * The boot-ioremap range spans multiple pmds, for which 382e64c8aa0SThomas Gleixner * we are not prepared: 383e64c8aa0SThomas Gleixner */ 384551889a6SIan Campbell if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) { 385e64c8aa0SThomas Gleixner WARN_ON(1); 386551889a6SIan Campbell printk(KERN_WARNING "pmd %p != %p\n", 387551889a6SIan Campbell pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))); 388e64c8aa0SThomas Gleixner printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 389e64c8aa0SThomas Gleixner fix_to_virt(FIX_BTMAP_BEGIN)); 390e64c8aa0SThomas Gleixner printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n", 391e64c8aa0SThomas Gleixner fix_to_virt(FIX_BTMAP_END)); 392e64c8aa0SThomas Gleixner 393e64c8aa0SThomas Gleixner printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 394e64c8aa0SThomas Gleixner printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n", 395e64c8aa0SThomas Gleixner FIX_BTMAP_BEGIN); 396e64c8aa0SThomas Gleixner } 397e64c8aa0SThomas Gleixner } 398e64c8aa0SThomas Gleixner 399e64c8aa0SThomas Gleixner void __init early_ioremap_clear(void) 400e64c8aa0SThomas Gleixner { 401551889a6SIan Campbell pmd_t *pmd; 402e64c8aa0SThomas Gleixner 403e64c8aa0SThomas Gleixner if (early_ioremap_debug) 404adafdf6aSIngo Molnar printk(KERN_INFO "early_ioremap_clear()\n"); 405e64c8aa0SThomas Gleixner 406551889a6SIan Campbell pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); 407551889a6SIan Campbell pmd_clear(pmd); 408b6fbb669SIan Campbell paravirt_release_pt(__pa(bm_pte) >> PAGE_SHIFT); 409e64c8aa0SThomas Gleixner __flush_tlb_all(); 410e64c8aa0SThomas Gleixner } 411e64c8aa0SThomas Gleixner 412e64c8aa0SThomas Gleixner void __init early_ioremap_reset(void) 413e64c8aa0SThomas Gleixner { 414e64c8aa0SThomas Gleixner enum fixed_addresses idx; 415551889a6SIan Campbell unsigned long addr, phys; 416551889a6SIan Campbell pte_t *pte; 417e64c8aa0SThomas Gleixner 418e64c8aa0SThomas Gleixner after_paging_init = 1; 419e64c8aa0SThomas Gleixner for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) { 420e64c8aa0SThomas Gleixner addr = fix_to_virt(idx); 421e64c8aa0SThomas Gleixner pte = early_ioremap_pte(addr); 422551889a6SIan Campbell if (pte_present(*pte)) { 423551889a6SIan Campbell phys = pte_val(*pte) & PAGE_MASK; 424e64c8aa0SThomas Gleixner set_fixmap(idx, phys); 425e64c8aa0SThomas Gleixner } 426e64c8aa0SThomas Gleixner } 427e64c8aa0SThomas Gleixner } 428e64c8aa0SThomas Gleixner 429e64c8aa0SThomas Gleixner static void __init __early_set_fixmap(enum fixed_addresses idx, 430e64c8aa0SThomas Gleixner unsigned long phys, pgprot_t flags) 431e64c8aa0SThomas Gleixner { 432551889a6SIan Campbell unsigned long addr = __fix_to_virt(idx); 433551889a6SIan Campbell pte_t *pte; 434e64c8aa0SThomas Gleixner 435e64c8aa0SThomas Gleixner if (idx >= __end_of_fixed_addresses) { 436e64c8aa0SThomas Gleixner BUG(); 437e64c8aa0SThomas Gleixner return; 438e64c8aa0SThomas Gleixner } 439e64c8aa0SThomas Gleixner pte = early_ioremap_pte(addr); 440e64c8aa0SThomas Gleixner if (pgprot_val(flags)) 441551889a6SIan Campbell set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 442e64c8aa0SThomas Gleixner else 443551889a6SIan Campbell pte_clear(NULL, addr, pte); 444e64c8aa0SThomas Gleixner __flush_tlb_one(addr); 445e64c8aa0SThomas Gleixner } 446e64c8aa0SThomas Gleixner 447e64c8aa0SThomas Gleixner static inline void __init early_set_fixmap(enum fixed_addresses idx, 448e64c8aa0SThomas Gleixner unsigned long phys) 449e64c8aa0SThomas Gleixner { 450e64c8aa0SThomas Gleixner if (after_paging_init) 451e64c8aa0SThomas Gleixner set_fixmap(idx, phys); 452e64c8aa0SThomas Gleixner else 453e64c8aa0SThomas Gleixner __early_set_fixmap(idx, phys, PAGE_KERNEL); 454e64c8aa0SThomas Gleixner } 455e64c8aa0SThomas Gleixner 456e64c8aa0SThomas Gleixner static inline void __init early_clear_fixmap(enum fixed_addresses idx) 457e64c8aa0SThomas Gleixner { 458e64c8aa0SThomas Gleixner if (after_paging_init) 459e64c8aa0SThomas Gleixner clear_fixmap(idx); 460e64c8aa0SThomas Gleixner else 461e64c8aa0SThomas Gleixner __early_set_fixmap(idx, 0, __pgprot(0)); 462e64c8aa0SThomas Gleixner } 463e64c8aa0SThomas Gleixner 464e64c8aa0SThomas Gleixner 465e64c8aa0SThomas Gleixner int __initdata early_ioremap_nested; 466e64c8aa0SThomas Gleixner 467e64c8aa0SThomas Gleixner static int __init check_early_ioremap_leak(void) 468e64c8aa0SThomas Gleixner { 469e64c8aa0SThomas Gleixner if (!early_ioremap_nested) 470e64c8aa0SThomas Gleixner return 0; 471e64c8aa0SThomas Gleixner 472e64c8aa0SThomas Gleixner printk(KERN_WARNING 473e64c8aa0SThomas Gleixner "Debug warning: early ioremap leak of %d areas detected.\n", 474e64c8aa0SThomas Gleixner early_ioremap_nested); 475e64c8aa0SThomas Gleixner printk(KERN_WARNING 476e64c8aa0SThomas Gleixner "please boot with early_ioremap_debug and report the dmesg.\n"); 477e64c8aa0SThomas Gleixner WARN_ON(1); 478e64c8aa0SThomas Gleixner 479e64c8aa0SThomas Gleixner return 1; 480e64c8aa0SThomas Gleixner } 481e64c8aa0SThomas Gleixner late_initcall(check_early_ioremap_leak); 482e64c8aa0SThomas Gleixner 483e64c8aa0SThomas Gleixner void __init *early_ioremap(unsigned long phys_addr, unsigned long size) 484e64c8aa0SThomas Gleixner { 485e64c8aa0SThomas Gleixner unsigned long offset, last_addr; 486e64c8aa0SThomas Gleixner unsigned int nrpages, nesting; 487e64c8aa0SThomas Gleixner enum fixed_addresses idx0, idx; 488e64c8aa0SThomas Gleixner 489e64c8aa0SThomas Gleixner WARN_ON(system_state != SYSTEM_BOOTING); 490e64c8aa0SThomas Gleixner 491e64c8aa0SThomas Gleixner nesting = early_ioremap_nested; 492e64c8aa0SThomas Gleixner if (early_ioremap_debug) { 493adafdf6aSIngo Molnar printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ", 494e64c8aa0SThomas Gleixner phys_addr, size, nesting); 495e64c8aa0SThomas Gleixner dump_stack(); 496e64c8aa0SThomas Gleixner } 497e64c8aa0SThomas Gleixner 498e64c8aa0SThomas Gleixner /* Don't allow wraparound or zero size */ 499e64c8aa0SThomas Gleixner last_addr = phys_addr + size - 1; 500e64c8aa0SThomas Gleixner if (!size || last_addr < phys_addr) { 501e64c8aa0SThomas Gleixner WARN_ON(1); 502e64c8aa0SThomas Gleixner return NULL; 503e64c8aa0SThomas Gleixner } 504e64c8aa0SThomas Gleixner 505e64c8aa0SThomas Gleixner if (nesting >= FIX_BTMAPS_NESTING) { 506e64c8aa0SThomas Gleixner WARN_ON(1); 507e64c8aa0SThomas Gleixner return NULL; 508e64c8aa0SThomas Gleixner } 509e64c8aa0SThomas Gleixner early_ioremap_nested++; 510e64c8aa0SThomas Gleixner /* 511e64c8aa0SThomas Gleixner * Mappings have to be page-aligned 512e64c8aa0SThomas Gleixner */ 513e64c8aa0SThomas Gleixner offset = phys_addr & ~PAGE_MASK; 514e64c8aa0SThomas Gleixner phys_addr &= PAGE_MASK; 515e64c8aa0SThomas Gleixner size = PAGE_ALIGN(last_addr) - phys_addr; 516e64c8aa0SThomas Gleixner 517e64c8aa0SThomas Gleixner /* 518e64c8aa0SThomas Gleixner * Mappings have to fit in the FIX_BTMAP area. 519e64c8aa0SThomas Gleixner */ 520e64c8aa0SThomas Gleixner nrpages = size >> PAGE_SHIFT; 521e64c8aa0SThomas Gleixner if (nrpages > NR_FIX_BTMAPS) { 522e64c8aa0SThomas Gleixner WARN_ON(1); 523e64c8aa0SThomas Gleixner return NULL; 524e64c8aa0SThomas Gleixner } 525e64c8aa0SThomas Gleixner 526e64c8aa0SThomas Gleixner /* 527e64c8aa0SThomas Gleixner * Ok, go for it.. 528e64c8aa0SThomas Gleixner */ 529e64c8aa0SThomas Gleixner idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting; 530e64c8aa0SThomas Gleixner idx = idx0; 531e64c8aa0SThomas Gleixner while (nrpages > 0) { 532e64c8aa0SThomas Gleixner early_set_fixmap(idx, phys_addr); 533e64c8aa0SThomas Gleixner phys_addr += PAGE_SIZE; 534e64c8aa0SThomas Gleixner --idx; 535e64c8aa0SThomas Gleixner --nrpages; 536e64c8aa0SThomas Gleixner } 537e64c8aa0SThomas Gleixner if (early_ioremap_debug) 538e64c8aa0SThomas Gleixner printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0)); 539e64c8aa0SThomas Gleixner 540e64c8aa0SThomas Gleixner return (void *) (offset + fix_to_virt(idx0)); 541e64c8aa0SThomas Gleixner } 542e64c8aa0SThomas Gleixner 543e64c8aa0SThomas Gleixner void __init early_iounmap(void *addr, unsigned long size) 544e64c8aa0SThomas Gleixner { 545e64c8aa0SThomas Gleixner unsigned long virt_addr; 546e64c8aa0SThomas Gleixner unsigned long offset; 547e64c8aa0SThomas Gleixner unsigned int nrpages; 548e64c8aa0SThomas Gleixner enum fixed_addresses idx; 549e64c8aa0SThomas Gleixner unsigned int nesting; 550e64c8aa0SThomas Gleixner 551e64c8aa0SThomas Gleixner nesting = --early_ioremap_nested; 552e64c8aa0SThomas Gleixner WARN_ON(nesting < 0); 553e64c8aa0SThomas Gleixner 554e64c8aa0SThomas Gleixner if (early_ioremap_debug) { 555adafdf6aSIngo Molnar printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr, 556e64c8aa0SThomas Gleixner size, nesting); 557e64c8aa0SThomas Gleixner dump_stack(); 558e64c8aa0SThomas Gleixner } 559e64c8aa0SThomas Gleixner 560e64c8aa0SThomas Gleixner virt_addr = (unsigned long)addr; 561e64c8aa0SThomas Gleixner if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) { 562e64c8aa0SThomas Gleixner WARN_ON(1); 563e64c8aa0SThomas Gleixner return; 564e64c8aa0SThomas Gleixner } 565e64c8aa0SThomas Gleixner offset = virt_addr & ~PAGE_MASK; 566e64c8aa0SThomas Gleixner nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT; 567e64c8aa0SThomas Gleixner 568e64c8aa0SThomas Gleixner idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting; 569e64c8aa0SThomas Gleixner while (nrpages > 0) { 570e64c8aa0SThomas Gleixner early_clear_fixmap(idx); 571e64c8aa0SThomas Gleixner --idx; 572e64c8aa0SThomas Gleixner --nrpages; 573e64c8aa0SThomas Gleixner } 574e64c8aa0SThomas Gleixner } 575e64c8aa0SThomas Gleixner 576e64c8aa0SThomas Gleixner void __this_fixmap_does_not_exist(void) 577e64c8aa0SThomas Gleixner { 578e64c8aa0SThomas Gleixner WARN_ON(1); 579e64c8aa0SThomas Gleixner } 580e64c8aa0SThomas Gleixner 581e64c8aa0SThomas Gleixner #endif /* CONFIG_X86_32 */ 582