1 /* 2 * Re-map IO memory to kernel address space so that we can access it. 3 * This is needed for high PCI addresses that aren't mapped in the 4 * 640k-1MB IO memory area on PC's 5 * 6 * (C) Copyright 1995 1996 Linus Torvalds 7 */ 8 9 #include <linux/bootmem.h> 10 #include <linux/init.h> 11 #include <linux/io.h> 12 #include <linux/module.h> 13 #include <linux/slab.h> 14 #include <linux/vmalloc.h> 15 #include <linux/mmiotrace.h> 16 17 #include <asm/cacheflush.h> 18 #include <asm/e820.h> 19 #include <asm/fixmap.h> 20 #include <asm/pgtable.h> 21 #include <asm/tlbflush.h> 22 #include <asm/pgalloc.h> 23 #include <asm/pat.h> 24 25 #include "physaddr.h" 26 27 int page_is_ram(unsigned long pagenr) 28 { 29 resource_size_t addr, end; 30 int i; 31 32 /* 33 * A special case is the first 4Kb of memory; 34 * This is a BIOS owned area, not kernel ram, but generally 35 * not listed as such in the E820 table. 36 */ 37 if (pagenr == 0) 38 return 0; 39 40 /* 41 * Second special case: Some BIOSen report the PC BIOS 42 * area (640->1Mb) as ram even though it is not. 43 */ 44 if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) && 45 pagenr < (BIOS_END >> PAGE_SHIFT)) 46 return 0; 47 48 for (i = 0; i < e820.nr_map; i++) { 49 /* 50 * Not usable memory: 51 */ 52 if (e820.map[i].type != E820_RAM) 53 continue; 54 addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT; 55 end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT; 56 57 58 if ((pagenr >= addr) && (pagenr < end)) 59 return 1; 60 } 61 return 0; 62 } 63 64 /* 65 * Fix up the linear direct mapping of the kernel to avoid cache attribute 66 * conflicts. 67 */ 68 int ioremap_change_attr(unsigned long vaddr, unsigned long size, 69 unsigned long prot_val) 70 { 71 unsigned long nrpages = size >> PAGE_SHIFT; 72 int err; 73 74 switch (prot_val) { 75 case _PAGE_CACHE_UC: 76 default: 77 err = _set_memory_uc(vaddr, nrpages); 78 break; 79 case _PAGE_CACHE_WC: 80 err = _set_memory_wc(vaddr, nrpages); 81 break; 82 case _PAGE_CACHE_WB: 83 err = _set_memory_wb(vaddr, nrpages); 84 break; 85 } 86 87 return err; 88 } 89 90 /* 91 * Remap an arbitrary physical address space into the kernel virtual 92 * address space. Needed when the kernel wants to access high addresses 93 * directly. 94 * 95 * NOTE! We need to allow non-page-aligned mappings too: we will obviously 96 * have to convert them into an offset in a page-aligned mapping, but the 97 * caller shouldn't need to know that small detail. 98 */ 99 static void __iomem *__ioremap_caller(resource_size_t phys_addr, 100 unsigned long size, unsigned long prot_val, void *caller) 101 { 102 unsigned long pfn, offset, vaddr; 103 resource_size_t last_addr; 104 const resource_size_t unaligned_phys_addr = phys_addr; 105 const unsigned long unaligned_size = size; 106 struct vm_struct *area; 107 unsigned long new_prot_val; 108 pgprot_t prot; 109 int retval; 110 void __iomem *ret_addr; 111 112 /* Don't allow wraparound or zero size */ 113 last_addr = phys_addr + size - 1; 114 if (!size || last_addr < phys_addr) 115 return NULL; 116 117 if (!phys_addr_valid(phys_addr)) { 118 printk(KERN_WARNING "ioremap: invalid physical address %llx\n", 119 (unsigned long long)phys_addr); 120 WARN_ON_ONCE(1); 121 return NULL; 122 } 123 124 /* 125 * Don't remap the low PCI/ISA area, it's always mapped.. 126 */ 127 if (is_ISA_range(phys_addr, last_addr)) 128 return (__force void __iomem *)phys_to_virt(phys_addr); 129 130 /* 131 * Check if the request spans more than any BAR in the iomem resource 132 * tree. 133 */ 134 WARN_ONCE(iomem_map_sanity_check(phys_addr, size), 135 KERN_INFO "Info: mapping multiple BARs. Your kernel is fine."); 136 137 /* 138 * Don't allow anybody to remap normal RAM that we're using.. 139 */ 140 for (pfn = phys_addr >> PAGE_SHIFT; 141 (pfn << PAGE_SHIFT) < (last_addr & PAGE_MASK); 142 pfn++) { 143 144 int is_ram = page_is_ram(pfn); 145 146 if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn))) 147 return NULL; 148 WARN_ON_ONCE(is_ram); 149 } 150 151 /* 152 * Mappings have to be page-aligned 153 */ 154 offset = phys_addr & ~PAGE_MASK; 155 phys_addr &= PAGE_MASK; 156 size = PAGE_ALIGN(last_addr+1) - phys_addr; 157 158 retval = reserve_memtype(phys_addr, (u64)phys_addr + size, 159 prot_val, &new_prot_val); 160 if (retval) { 161 printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval); 162 return NULL; 163 } 164 165 if (prot_val != new_prot_val) { 166 if (!is_new_memtype_allowed(phys_addr, size, 167 prot_val, new_prot_val)) { 168 printk(KERN_ERR 169 "ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n", 170 (unsigned long long)phys_addr, 171 (unsigned long long)(phys_addr + size), 172 prot_val, new_prot_val); 173 goto err_free_memtype; 174 } 175 prot_val = new_prot_val; 176 } 177 178 switch (prot_val) { 179 case _PAGE_CACHE_UC: 180 default: 181 prot = PAGE_KERNEL_IO_NOCACHE; 182 break; 183 case _PAGE_CACHE_UC_MINUS: 184 prot = PAGE_KERNEL_IO_UC_MINUS; 185 break; 186 case _PAGE_CACHE_WC: 187 prot = PAGE_KERNEL_IO_WC; 188 break; 189 case _PAGE_CACHE_WB: 190 prot = PAGE_KERNEL_IO; 191 break; 192 } 193 194 /* 195 * Ok, go for it.. 196 */ 197 area = get_vm_area_caller(size, VM_IOREMAP, caller); 198 if (!area) 199 goto err_free_memtype; 200 area->phys_addr = phys_addr; 201 vaddr = (unsigned long) area->addr; 202 203 if (kernel_map_sync_memtype(phys_addr, size, prot_val)) 204 goto err_free_area; 205 206 if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot)) 207 goto err_free_area; 208 209 ret_addr = (void __iomem *) (vaddr + offset); 210 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr); 211 212 return ret_addr; 213 err_free_area: 214 free_vm_area(area); 215 err_free_memtype: 216 free_memtype(phys_addr, phys_addr + size); 217 return NULL; 218 } 219 220 /** 221 * ioremap_nocache - map bus memory into CPU space 222 * @offset: bus address of the memory 223 * @size: size of the resource to map 224 * 225 * ioremap_nocache performs a platform specific sequence of operations to 226 * make bus memory CPU accessible via the readb/readw/readl/writeb/ 227 * writew/writel functions and the other mmio helpers. The returned 228 * address is not guaranteed to be usable directly as a virtual 229 * address. 230 * 231 * This version of ioremap ensures that the memory is marked uncachable 232 * on the CPU as well as honouring existing caching rules from things like 233 * the PCI bus. Note that there are other caches and buffers on many 234 * busses. In particular driver authors should read up on PCI writes 235 * 236 * It's useful if some control registers are in such an area and 237 * write combining or read caching is not desirable: 238 * 239 * Must be freed with iounmap. 240 */ 241 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size) 242 { 243 /* 244 * Ideally, this should be: 245 * pat_enabled ? _PAGE_CACHE_UC : _PAGE_CACHE_UC_MINUS; 246 * 247 * Till we fix all X drivers to use ioremap_wc(), we will use 248 * UC MINUS. 249 */ 250 unsigned long val = _PAGE_CACHE_UC_MINUS; 251 252 return __ioremap_caller(phys_addr, size, val, 253 __builtin_return_address(0)); 254 } 255 EXPORT_SYMBOL(ioremap_nocache); 256 257 /** 258 * ioremap_wc - map memory into CPU space write combined 259 * @offset: bus address of the memory 260 * @size: size of the resource to map 261 * 262 * This version of ioremap ensures that the memory is marked write combining. 263 * Write combining allows faster writes to some hardware devices. 264 * 265 * Must be freed with iounmap. 266 */ 267 void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size) 268 { 269 if (pat_enabled) 270 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WC, 271 __builtin_return_address(0)); 272 else 273 return ioremap_nocache(phys_addr, size); 274 } 275 EXPORT_SYMBOL(ioremap_wc); 276 277 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size) 278 { 279 return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WB, 280 __builtin_return_address(0)); 281 } 282 EXPORT_SYMBOL(ioremap_cache); 283 284 void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size, 285 unsigned long prot_val) 286 { 287 return __ioremap_caller(phys_addr, size, (prot_val & _PAGE_CACHE_MASK), 288 __builtin_return_address(0)); 289 } 290 EXPORT_SYMBOL(ioremap_prot); 291 292 /** 293 * iounmap - Free a IO remapping 294 * @addr: virtual address from ioremap_* 295 * 296 * Caller must ensure there is only one unmapping for the same pointer. 297 */ 298 void iounmap(volatile void __iomem *addr) 299 { 300 struct vm_struct *p, *o; 301 302 if ((void __force *)addr <= high_memory) 303 return; 304 305 /* 306 * __ioremap special-cases the PCI/ISA range by not instantiating a 307 * vm_area and by simply returning an address into the kernel mapping 308 * of ISA space. So handle that here. 309 */ 310 if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) && 311 (void __force *)addr < phys_to_virt(ISA_END_ADDRESS)) 312 return; 313 314 addr = (volatile void __iomem *) 315 (PAGE_MASK & (unsigned long __force)addr); 316 317 mmiotrace_iounmap(addr); 318 319 /* Use the vm area unlocked, assuming the caller 320 ensures there isn't another iounmap for the same address 321 in parallel. Reuse of the virtual address is prevented by 322 leaving it in the global lists until we're done with it. 323 cpa takes care of the direct mappings. */ 324 read_lock(&vmlist_lock); 325 for (p = vmlist; p; p = p->next) { 326 if (p->addr == (void __force *)addr) 327 break; 328 } 329 read_unlock(&vmlist_lock); 330 331 if (!p) { 332 printk(KERN_ERR "iounmap: bad address %p\n", addr); 333 dump_stack(); 334 return; 335 } 336 337 free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p)); 338 339 /* Finally remove it */ 340 o = remove_vm_area((void __force *)addr); 341 BUG_ON(p != o || o == NULL); 342 kfree(p); 343 } 344 EXPORT_SYMBOL(iounmap); 345 346 /* 347 * Convert a physical pointer to a virtual kernel pointer for /dev/mem 348 * access 349 */ 350 void *xlate_dev_mem_ptr(unsigned long phys) 351 { 352 void *addr; 353 unsigned long start = phys & PAGE_MASK; 354 355 /* If page is RAM, we can use __va. Otherwise ioremap and unmap. */ 356 if (page_is_ram(start >> PAGE_SHIFT)) 357 return __va(phys); 358 359 addr = (void __force *)ioremap_cache(start, PAGE_SIZE); 360 if (addr) 361 addr = (void *)((unsigned long)addr | (phys & ~PAGE_MASK)); 362 363 return addr; 364 } 365 366 void unxlate_dev_mem_ptr(unsigned long phys, void *addr) 367 { 368 if (page_is_ram(phys >> PAGE_SHIFT)) 369 return; 370 371 iounmap((void __iomem *)((unsigned long)addr & PAGE_MASK)); 372 return; 373 } 374 375 static int __initdata early_ioremap_debug; 376 377 static int __init early_ioremap_debug_setup(char *str) 378 { 379 early_ioremap_debug = 1; 380 381 return 0; 382 } 383 early_param("early_ioremap_debug", early_ioremap_debug_setup); 384 385 static __initdata int after_paging_init; 386 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss; 387 388 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) 389 { 390 /* Don't assume we're using swapper_pg_dir at this point */ 391 pgd_t *base = __va(read_cr3()); 392 pgd_t *pgd = &base[pgd_index(addr)]; 393 pud_t *pud = pud_offset(pgd, addr); 394 pmd_t *pmd = pmd_offset(pud, addr); 395 396 return pmd; 397 } 398 399 static inline pte_t * __init early_ioremap_pte(unsigned long addr) 400 { 401 return &bm_pte[pte_index(addr)]; 402 } 403 404 static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata; 405 406 void __init early_ioremap_init(void) 407 { 408 pmd_t *pmd; 409 int i; 410 411 if (early_ioremap_debug) 412 printk(KERN_INFO "early_ioremap_init()\n"); 413 414 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) 415 slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i); 416 417 pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); 418 memset(bm_pte, 0, sizeof(bm_pte)); 419 pmd_populate_kernel(&init_mm, pmd, bm_pte); 420 421 /* 422 * The boot-ioremap range spans multiple pmds, for which 423 * we are not prepared: 424 */ 425 if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) { 426 WARN_ON(1); 427 printk(KERN_WARNING "pmd %p != %p\n", 428 pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))); 429 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n", 430 fix_to_virt(FIX_BTMAP_BEGIN)); 431 printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END): %08lx\n", 432 fix_to_virt(FIX_BTMAP_END)); 433 434 printk(KERN_WARNING "FIX_BTMAP_END: %d\n", FIX_BTMAP_END); 435 printk(KERN_WARNING "FIX_BTMAP_BEGIN: %d\n", 436 FIX_BTMAP_BEGIN); 437 } 438 } 439 440 void __init early_ioremap_reset(void) 441 { 442 after_paging_init = 1; 443 } 444 445 static void __init __early_set_fixmap(enum fixed_addresses idx, 446 phys_addr_t phys, pgprot_t flags) 447 { 448 unsigned long addr = __fix_to_virt(idx); 449 pte_t *pte; 450 451 if (idx >= __end_of_fixed_addresses) { 452 BUG(); 453 return; 454 } 455 pte = early_ioremap_pte(addr); 456 457 if (pgprot_val(flags)) 458 set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags)); 459 else 460 pte_clear(&init_mm, addr, pte); 461 __flush_tlb_one(addr); 462 } 463 464 static inline void __init early_set_fixmap(enum fixed_addresses idx, 465 phys_addr_t phys, pgprot_t prot) 466 { 467 if (after_paging_init) 468 __set_fixmap(idx, phys, prot); 469 else 470 __early_set_fixmap(idx, phys, prot); 471 } 472 473 static inline void __init early_clear_fixmap(enum fixed_addresses idx) 474 { 475 if (after_paging_init) 476 clear_fixmap(idx); 477 else 478 __early_set_fixmap(idx, 0, __pgprot(0)); 479 } 480 481 static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata; 482 static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata; 483 484 static int __init check_early_ioremap_leak(void) 485 { 486 int count = 0; 487 int i; 488 489 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) 490 if (prev_map[i]) 491 count++; 492 493 if (!count) 494 return 0; 495 WARN(1, KERN_WARNING 496 "Debug warning: early ioremap leak of %d areas detected.\n", 497 count); 498 printk(KERN_WARNING 499 "please boot with early_ioremap_debug and report the dmesg.\n"); 500 501 return 1; 502 } 503 late_initcall(check_early_ioremap_leak); 504 505 static void __init __iomem * 506 __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot) 507 { 508 unsigned long offset; 509 resource_size_t last_addr; 510 unsigned int nrpages; 511 enum fixed_addresses idx0, idx; 512 int i, slot; 513 514 WARN_ON(system_state != SYSTEM_BOOTING); 515 516 slot = -1; 517 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) { 518 if (!prev_map[i]) { 519 slot = i; 520 break; 521 } 522 } 523 524 if (slot < 0) { 525 printk(KERN_INFO "early_iomap(%08llx, %08lx) not found slot\n", 526 (u64)phys_addr, size); 527 WARN_ON(1); 528 return NULL; 529 } 530 531 if (early_ioremap_debug) { 532 printk(KERN_INFO "early_ioremap(%08llx, %08lx) [%d] => ", 533 (u64)phys_addr, size, slot); 534 dump_stack(); 535 } 536 537 /* Don't allow wraparound or zero size */ 538 last_addr = phys_addr + size - 1; 539 if (!size || last_addr < phys_addr) { 540 WARN_ON(1); 541 return NULL; 542 } 543 544 prev_size[slot] = size; 545 /* 546 * Mappings have to be page-aligned 547 */ 548 offset = phys_addr & ~PAGE_MASK; 549 phys_addr &= PAGE_MASK; 550 size = PAGE_ALIGN(last_addr + 1) - phys_addr; 551 552 /* 553 * Mappings have to fit in the FIX_BTMAP area. 554 */ 555 nrpages = size >> PAGE_SHIFT; 556 if (nrpages > NR_FIX_BTMAPS) { 557 WARN_ON(1); 558 return NULL; 559 } 560 561 /* 562 * Ok, go for it.. 563 */ 564 idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot; 565 idx = idx0; 566 while (nrpages > 0) { 567 early_set_fixmap(idx, phys_addr, prot); 568 phys_addr += PAGE_SIZE; 569 --idx; 570 --nrpages; 571 } 572 if (early_ioremap_debug) 573 printk(KERN_CONT "%08lx + %08lx\n", offset, slot_virt[slot]); 574 575 prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]); 576 return prev_map[slot]; 577 } 578 579 /* Remap an IO device */ 580 void __init __iomem * 581 early_ioremap(resource_size_t phys_addr, unsigned long size) 582 { 583 return __early_ioremap(phys_addr, size, PAGE_KERNEL_IO); 584 } 585 586 /* Remap memory */ 587 void __init __iomem * 588 early_memremap(resource_size_t phys_addr, unsigned long size) 589 { 590 return __early_ioremap(phys_addr, size, PAGE_KERNEL); 591 } 592 593 void __init early_iounmap(void __iomem *addr, unsigned long size) 594 { 595 unsigned long virt_addr; 596 unsigned long offset; 597 unsigned int nrpages; 598 enum fixed_addresses idx; 599 int i, slot; 600 601 slot = -1; 602 for (i = 0; i < FIX_BTMAPS_SLOTS; i++) { 603 if (prev_map[i] == addr) { 604 slot = i; 605 break; 606 } 607 } 608 609 if (slot < 0) { 610 printk(KERN_INFO "early_iounmap(%p, %08lx) not found slot\n", 611 addr, size); 612 WARN_ON(1); 613 return; 614 } 615 616 if (prev_size[slot] != size) { 617 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n", 618 addr, size, slot, prev_size[slot]); 619 WARN_ON(1); 620 return; 621 } 622 623 if (early_ioremap_debug) { 624 printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr, 625 size, slot); 626 dump_stack(); 627 } 628 629 virt_addr = (unsigned long)addr; 630 if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) { 631 WARN_ON(1); 632 return; 633 } 634 offset = virt_addr & ~PAGE_MASK; 635 nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT; 636 637 idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot; 638 while (nrpages > 0) { 639 early_clear_fixmap(idx); 640 --idx; 641 --nrpages; 642 } 643 prev_map[slot] = NULL; 644 } 645