xref: /openbmc/linux/arch/x86/mm/ioremap.c (revision 9d56dd3b083a3bec56e9da35ce07baca81030b03)
1 /*
2  * Re-map IO memory to kernel address space so that we can access it.
3  * This is needed for high PCI addresses that aren't mapped in the
4  * 640k-1MB IO memory area on PC's
5  *
6  * (C) Copyright 1995 1996 Linus Torvalds
7  */
8 
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
11 #include <linux/io.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
15 #include <linux/mmiotrace.h>
16 
17 #include <asm/cacheflush.h>
18 #include <asm/e820.h>
19 #include <asm/fixmap.h>
20 #include <asm/pgtable.h>
21 #include <asm/tlbflush.h>
22 #include <asm/pgalloc.h>
23 #include <asm/pat.h>
24 
25 #include "physaddr.h"
26 
27 int page_is_ram(unsigned long pagenr)
28 {
29 	resource_size_t addr, end;
30 	int i;
31 
32 	/*
33 	 * A special case is the first 4Kb of memory;
34 	 * This is a BIOS owned area, not kernel ram, but generally
35 	 * not listed as such in the E820 table.
36 	 */
37 	if (pagenr == 0)
38 		return 0;
39 
40 	/*
41 	 * Second special case: Some BIOSen report the PC BIOS
42 	 * area (640->1Mb) as ram even though it is not.
43 	 */
44 	if (pagenr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
45 		    pagenr < (BIOS_END >> PAGE_SHIFT))
46 		return 0;
47 
48 	for (i = 0; i < e820.nr_map; i++) {
49 		/*
50 		 * Not usable memory:
51 		 */
52 		if (e820.map[i].type != E820_RAM)
53 			continue;
54 		addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
55 		end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
56 
57 
58 		if ((pagenr >= addr) && (pagenr < end))
59 			return 1;
60 	}
61 	return 0;
62 }
63 
64 /*
65  * Fix up the linear direct mapping of the kernel to avoid cache attribute
66  * conflicts.
67  */
68 int ioremap_change_attr(unsigned long vaddr, unsigned long size,
69 			       unsigned long prot_val)
70 {
71 	unsigned long nrpages = size >> PAGE_SHIFT;
72 	int err;
73 
74 	switch (prot_val) {
75 	case _PAGE_CACHE_UC:
76 	default:
77 		err = _set_memory_uc(vaddr, nrpages);
78 		break;
79 	case _PAGE_CACHE_WC:
80 		err = _set_memory_wc(vaddr, nrpages);
81 		break;
82 	case _PAGE_CACHE_WB:
83 		err = _set_memory_wb(vaddr, nrpages);
84 		break;
85 	}
86 
87 	return err;
88 }
89 
90 /*
91  * Remap an arbitrary physical address space into the kernel virtual
92  * address space. Needed when the kernel wants to access high addresses
93  * directly.
94  *
95  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
96  * have to convert them into an offset in a page-aligned mapping, but the
97  * caller shouldn't need to know that small detail.
98  */
99 static void __iomem *__ioremap_caller(resource_size_t phys_addr,
100 		unsigned long size, unsigned long prot_val, void *caller)
101 {
102 	unsigned long pfn, offset, vaddr;
103 	resource_size_t last_addr;
104 	const resource_size_t unaligned_phys_addr = phys_addr;
105 	const unsigned long unaligned_size = size;
106 	struct vm_struct *area;
107 	unsigned long new_prot_val;
108 	pgprot_t prot;
109 	int retval;
110 	void __iomem *ret_addr;
111 
112 	/* Don't allow wraparound or zero size */
113 	last_addr = phys_addr + size - 1;
114 	if (!size || last_addr < phys_addr)
115 		return NULL;
116 
117 	if (!phys_addr_valid(phys_addr)) {
118 		printk(KERN_WARNING "ioremap: invalid physical address %llx\n",
119 		       (unsigned long long)phys_addr);
120 		WARN_ON_ONCE(1);
121 		return NULL;
122 	}
123 
124 	/*
125 	 * Don't remap the low PCI/ISA area, it's always mapped..
126 	 */
127 	if (is_ISA_range(phys_addr, last_addr))
128 		return (__force void __iomem *)phys_to_virt(phys_addr);
129 
130 	/*
131 	 * Check if the request spans more than any BAR in the iomem resource
132 	 * tree.
133 	 */
134 	WARN_ONCE(iomem_map_sanity_check(phys_addr, size),
135 		  KERN_INFO "Info: mapping multiple BARs. Your kernel is fine.");
136 
137 	/*
138 	 * Don't allow anybody to remap normal RAM that we're using..
139 	 */
140 	for (pfn = phys_addr >> PAGE_SHIFT;
141 				(pfn << PAGE_SHIFT) < (last_addr & PAGE_MASK);
142 				pfn++) {
143 
144 		int is_ram = page_is_ram(pfn);
145 
146 		if (is_ram && pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
147 			return NULL;
148 		WARN_ON_ONCE(is_ram);
149 	}
150 
151 	/*
152 	 * Mappings have to be page-aligned
153 	 */
154 	offset = phys_addr & ~PAGE_MASK;
155 	phys_addr &= PAGE_MASK;
156 	size = PAGE_ALIGN(last_addr+1) - phys_addr;
157 
158 	retval = reserve_memtype(phys_addr, (u64)phys_addr + size,
159 						prot_val, &new_prot_val);
160 	if (retval) {
161 		printk(KERN_ERR "ioremap reserve_memtype failed %d\n", retval);
162 		return NULL;
163 	}
164 
165 	if (prot_val != new_prot_val) {
166 		if (!is_new_memtype_allowed(phys_addr, size,
167 					    prot_val, new_prot_val)) {
168 			printk(KERN_ERR
169 		"ioremap error for 0x%llx-0x%llx, requested 0x%lx, got 0x%lx\n",
170 				(unsigned long long)phys_addr,
171 				(unsigned long long)(phys_addr + size),
172 				prot_val, new_prot_val);
173 			goto err_free_memtype;
174 		}
175 		prot_val = new_prot_val;
176 	}
177 
178 	switch (prot_val) {
179 	case _PAGE_CACHE_UC:
180 	default:
181 		prot = PAGE_KERNEL_IO_NOCACHE;
182 		break;
183 	case _PAGE_CACHE_UC_MINUS:
184 		prot = PAGE_KERNEL_IO_UC_MINUS;
185 		break;
186 	case _PAGE_CACHE_WC:
187 		prot = PAGE_KERNEL_IO_WC;
188 		break;
189 	case _PAGE_CACHE_WB:
190 		prot = PAGE_KERNEL_IO;
191 		break;
192 	}
193 
194 	/*
195 	 * Ok, go for it..
196 	 */
197 	area = get_vm_area_caller(size, VM_IOREMAP, caller);
198 	if (!area)
199 		goto err_free_memtype;
200 	area->phys_addr = phys_addr;
201 	vaddr = (unsigned long) area->addr;
202 
203 	if (kernel_map_sync_memtype(phys_addr, size, prot_val))
204 		goto err_free_area;
205 
206 	if (ioremap_page_range(vaddr, vaddr + size, phys_addr, prot))
207 		goto err_free_area;
208 
209 	ret_addr = (void __iomem *) (vaddr + offset);
210 	mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
211 
212 	return ret_addr;
213 err_free_area:
214 	free_vm_area(area);
215 err_free_memtype:
216 	free_memtype(phys_addr, phys_addr + size);
217 	return NULL;
218 }
219 
220 /**
221  * ioremap_nocache     -   map bus memory into CPU space
222  * @offset:    bus address of the memory
223  * @size:      size of the resource to map
224  *
225  * ioremap_nocache performs a platform specific sequence of operations to
226  * make bus memory CPU accessible via the readb/readw/readl/writeb/
227  * writew/writel functions and the other mmio helpers. The returned
228  * address is not guaranteed to be usable directly as a virtual
229  * address.
230  *
231  * This version of ioremap ensures that the memory is marked uncachable
232  * on the CPU as well as honouring existing caching rules from things like
233  * the PCI bus. Note that there are other caches and buffers on many
234  * busses. In particular driver authors should read up on PCI writes
235  *
236  * It's useful if some control registers are in such an area and
237  * write combining or read caching is not desirable:
238  *
239  * Must be freed with iounmap.
240  */
241 void __iomem *ioremap_nocache(resource_size_t phys_addr, unsigned long size)
242 {
243 	/*
244 	 * Ideally, this should be:
245 	 *	pat_enabled ? _PAGE_CACHE_UC : _PAGE_CACHE_UC_MINUS;
246 	 *
247 	 * Till we fix all X drivers to use ioremap_wc(), we will use
248 	 * UC MINUS.
249 	 */
250 	unsigned long val = _PAGE_CACHE_UC_MINUS;
251 
252 	return __ioremap_caller(phys_addr, size, val,
253 				__builtin_return_address(0));
254 }
255 EXPORT_SYMBOL(ioremap_nocache);
256 
257 /**
258  * ioremap_wc	-	map memory into CPU space write combined
259  * @offset:	bus address of the memory
260  * @size:	size of the resource to map
261  *
262  * This version of ioremap ensures that the memory is marked write combining.
263  * Write combining allows faster writes to some hardware devices.
264  *
265  * Must be freed with iounmap.
266  */
267 void __iomem *ioremap_wc(resource_size_t phys_addr, unsigned long size)
268 {
269 	if (pat_enabled)
270 		return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WC,
271 					__builtin_return_address(0));
272 	else
273 		return ioremap_nocache(phys_addr, size);
274 }
275 EXPORT_SYMBOL(ioremap_wc);
276 
277 void __iomem *ioremap_cache(resource_size_t phys_addr, unsigned long size)
278 {
279 	return __ioremap_caller(phys_addr, size, _PAGE_CACHE_WB,
280 				__builtin_return_address(0));
281 }
282 EXPORT_SYMBOL(ioremap_cache);
283 
284 void __iomem *ioremap_prot(resource_size_t phys_addr, unsigned long size,
285 				unsigned long prot_val)
286 {
287 	return __ioremap_caller(phys_addr, size, (prot_val & _PAGE_CACHE_MASK),
288 				__builtin_return_address(0));
289 }
290 EXPORT_SYMBOL(ioremap_prot);
291 
292 /**
293  * iounmap - Free a IO remapping
294  * @addr: virtual address from ioremap_*
295  *
296  * Caller must ensure there is only one unmapping for the same pointer.
297  */
298 void iounmap(volatile void __iomem *addr)
299 {
300 	struct vm_struct *p, *o;
301 
302 	if ((void __force *)addr <= high_memory)
303 		return;
304 
305 	/*
306 	 * __ioremap special-cases the PCI/ISA range by not instantiating a
307 	 * vm_area and by simply returning an address into the kernel mapping
308 	 * of ISA space.   So handle that here.
309 	 */
310 	if ((void __force *)addr >= phys_to_virt(ISA_START_ADDRESS) &&
311 	    (void __force *)addr < phys_to_virt(ISA_END_ADDRESS))
312 		return;
313 
314 	addr = (volatile void __iomem *)
315 		(PAGE_MASK & (unsigned long __force)addr);
316 
317 	mmiotrace_iounmap(addr);
318 
319 	/* Use the vm area unlocked, assuming the caller
320 	   ensures there isn't another iounmap for the same address
321 	   in parallel. Reuse of the virtual address is prevented by
322 	   leaving it in the global lists until we're done with it.
323 	   cpa takes care of the direct mappings. */
324 	read_lock(&vmlist_lock);
325 	for (p = vmlist; p; p = p->next) {
326 		if (p->addr == (void __force *)addr)
327 			break;
328 	}
329 	read_unlock(&vmlist_lock);
330 
331 	if (!p) {
332 		printk(KERN_ERR "iounmap: bad address %p\n", addr);
333 		dump_stack();
334 		return;
335 	}
336 
337 	free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
338 
339 	/* Finally remove it */
340 	o = remove_vm_area((void __force *)addr);
341 	BUG_ON(p != o || o == NULL);
342 	kfree(p);
343 }
344 EXPORT_SYMBOL(iounmap);
345 
346 /*
347  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
348  * access
349  */
350 void *xlate_dev_mem_ptr(unsigned long phys)
351 {
352 	void *addr;
353 	unsigned long start = phys & PAGE_MASK;
354 
355 	/* If page is RAM, we can use __va. Otherwise ioremap and unmap. */
356 	if (page_is_ram(start >> PAGE_SHIFT))
357 		return __va(phys);
358 
359 	addr = (void __force *)ioremap_cache(start, PAGE_SIZE);
360 	if (addr)
361 		addr = (void *)((unsigned long)addr | (phys & ~PAGE_MASK));
362 
363 	return addr;
364 }
365 
366 void unxlate_dev_mem_ptr(unsigned long phys, void *addr)
367 {
368 	if (page_is_ram(phys >> PAGE_SHIFT))
369 		return;
370 
371 	iounmap((void __iomem *)((unsigned long)addr & PAGE_MASK));
372 	return;
373 }
374 
375 static int __initdata early_ioremap_debug;
376 
377 static int __init early_ioremap_debug_setup(char *str)
378 {
379 	early_ioremap_debug = 1;
380 
381 	return 0;
382 }
383 early_param("early_ioremap_debug", early_ioremap_debug_setup);
384 
385 static __initdata int after_paging_init;
386 static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss;
387 
388 static inline pmd_t * __init early_ioremap_pmd(unsigned long addr)
389 {
390 	/* Don't assume we're using swapper_pg_dir at this point */
391 	pgd_t *base = __va(read_cr3());
392 	pgd_t *pgd = &base[pgd_index(addr)];
393 	pud_t *pud = pud_offset(pgd, addr);
394 	pmd_t *pmd = pmd_offset(pud, addr);
395 
396 	return pmd;
397 }
398 
399 static inline pte_t * __init early_ioremap_pte(unsigned long addr)
400 {
401 	return &bm_pte[pte_index(addr)];
402 }
403 
404 static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;
405 
406 void __init early_ioremap_init(void)
407 {
408 	pmd_t *pmd;
409 	int i;
410 
411 	if (early_ioremap_debug)
412 		printk(KERN_INFO "early_ioremap_init()\n");
413 
414 	for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
415 		slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i);
416 
417 	pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN));
418 	memset(bm_pte, 0, sizeof(bm_pte));
419 	pmd_populate_kernel(&init_mm, pmd, bm_pte);
420 
421 	/*
422 	 * The boot-ioremap range spans multiple pmds, for which
423 	 * we are not prepared:
424 	 */
425 	if (pmd != early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END))) {
426 		WARN_ON(1);
427 		printk(KERN_WARNING "pmd %p != %p\n",
428 		       pmd, early_ioremap_pmd(fix_to_virt(FIX_BTMAP_END)));
429 		printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
430 			fix_to_virt(FIX_BTMAP_BEGIN));
431 		printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
432 			fix_to_virt(FIX_BTMAP_END));
433 
434 		printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
435 		printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
436 		       FIX_BTMAP_BEGIN);
437 	}
438 }
439 
440 void __init early_ioremap_reset(void)
441 {
442 	after_paging_init = 1;
443 }
444 
445 static void __init __early_set_fixmap(enum fixed_addresses idx,
446 				      phys_addr_t phys, pgprot_t flags)
447 {
448 	unsigned long addr = __fix_to_virt(idx);
449 	pte_t *pte;
450 
451 	if (idx >= __end_of_fixed_addresses) {
452 		BUG();
453 		return;
454 	}
455 	pte = early_ioremap_pte(addr);
456 
457 	if (pgprot_val(flags))
458 		set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, flags));
459 	else
460 		pte_clear(&init_mm, addr, pte);
461 	__flush_tlb_one(addr);
462 }
463 
464 static inline void __init early_set_fixmap(enum fixed_addresses idx,
465 					   phys_addr_t phys, pgprot_t prot)
466 {
467 	if (after_paging_init)
468 		__set_fixmap(idx, phys, prot);
469 	else
470 		__early_set_fixmap(idx, phys, prot);
471 }
472 
473 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
474 {
475 	if (after_paging_init)
476 		clear_fixmap(idx);
477 	else
478 		__early_set_fixmap(idx, 0, __pgprot(0));
479 }
480 
481 static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
482 static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
483 
484 static int __init check_early_ioremap_leak(void)
485 {
486 	int count = 0;
487 	int i;
488 
489 	for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
490 		if (prev_map[i])
491 			count++;
492 
493 	if (!count)
494 		return 0;
495 	WARN(1, KERN_WARNING
496 	       "Debug warning: early ioremap leak of %d areas detected.\n",
497 		count);
498 	printk(KERN_WARNING
499 		"please boot with early_ioremap_debug and report the dmesg.\n");
500 
501 	return 1;
502 }
503 late_initcall(check_early_ioremap_leak);
504 
505 static void __init __iomem *
506 __early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot)
507 {
508 	unsigned long offset;
509 	resource_size_t last_addr;
510 	unsigned int nrpages;
511 	enum fixed_addresses idx0, idx;
512 	int i, slot;
513 
514 	WARN_ON(system_state != SYSTEM_BOOTING);
515 
516 	slot = -1;
517 	for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
518 		if (!prev_map[i]) {
519 			slot = i;
520 			break;
521 		}
522 	}
523 
524 	if (slot < 0) {
525 		printk(KERN_INFO "early_iomap(%08llx, %08lx) not found slot\n",
526 			 (u64)phys_addr, size);
527 		WARN_ON(1);
528 		return NULL;
529 	}
530 
531 	if (early_ioremap_debug) {
532 		printk(KERN_INFO "early_ioremap(%08llx, %08lx) [%d] => ",
533 		       (u64)phys_addr, size, slot);
534 		dump_stack();
535 	}
536 
537 	/* Don't allow wraparound or zero size */
538 	last_addr = phys_addr + size - 1;
539 	if (!size || last_addr < phys_addr) {
540 		WARN_ON(1);
541 		return NULL;
542 	}
543 
544 	prev_size[slot] = size;
545 	/*
546 	 * Mappings have to be page-aligned
547 	 */
548 	offset = phys_addr & ~PAGE_MASK;
549 	phys_addr &= PAGE_MASK;
550 	size = PAGE_ALIGN(last_addr + 1) - phys_addr;
551 
552 	/*
553 	 * Mappings have to fit in the FIX_BTMAP area.
554 	 */
555 	nrpages = size >> PAGE_SHIFT;
556 	if (nrpages > NR_FIX_BTMAPS) {
557 		WARN_ON(1);
558 		return NULL;
559 	}
560 
561 	/*
562 	 * Ok, go for it..
563 	 */
564 	idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
565 	idx = idx0;
566 	while (nrpages > 0) {
567 		early_set_fixmap(idx, phys_addr, prot);
568 		phys_addr += PAGE_SIZE;
569 		--idx;
570 		--nrpages;
571 	}
572 	if (early_ioremap_debug)
573 		printk(KERN_CONT "%08lx + %08lx\n", offset, slot_virt[slot]);
574 
575 	prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]);
576 	return prev_map[slot];
577 }
578 
579 /* Remap an IO device */
580 void __init __iomem *
581 early_ioremap(resource_size_t phys_addr, unsigned long size)
582 {
583 	return __early_ioremap(phys_addr, size, PAGE_KERNEL_IO);
584 }
585 
586 /* Remap memory */
587 void __init __iomem *
588 early_memremap(resource_size_t phys_addr, unsigned long size)
589 {
590 	return __early_ioremap(phys_addr, size, PAGE_KERNEL);
591 }
592 
593 void __init early_iounmap(void __iomem *addr, unsigned long size)
594 {
595 	unsigned long virt_addr;
596 	unsigned long offset;
597 	unsigned int nrpages;
598 	enum fixed_addresses idx;
599 	int i, slot;
600 
601 	slot = -1;
602 	for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
603 		if (prev_map[i] == addr) {
604 			slot = i;
605 			break;
606 		}
607 	}
608 
609 	if (slot < 0) {
610 		printk(KERN_INFO "early_iounmap(%p, %08lx) not found slot\n",
611 			 addr, size);
612 		WARN_ON(1);
613 		return;
614 	}
615 
616 	if (prev_size[slot] != size) {
617 		printk(KERN_INFO "early_iounmap(%p, %08lx) [%d] size not consistent %08lx\n",
618 			 addr, size, slot, prev_size[slot]);
619 		WARN_ON(1);
620 		return;
621 	}
622 
623 	if (early_ioremap_debug) {
624 		printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
625 		       size, slot);
626 		dump_stack();
627 	}
628 
629 	virt_addr = (unsigned long)addr;
630 	if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
631 		WARN_ON(1);
632 		return;
633 	}
634 	offset = virt_addr & ~PAGE_MASK;
635 	nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
636 
637 	idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
638 	while (nrpages > 0) {
639 		early_clear_fixmap(idx);
640 		--idx;
641 		--nrpages;
642 	}
643 	prev_map[slot] = NULL;
644 }
645