xref: /openbmc/linux/arch/x86/mm/ioremap.c (revision 643d1f7f)
1 /*
2  * Re-map IO memory to kernel address space so that we can access it.
3  * This is needed for high PCI addresses that aren't mapped in the
4  * 640k-1MB IO memory area on PC's
5  *
6  * (C) Copyright 1995 1996 Linus Torvalds
7  */
8 
9 #include <linux/bootmem.h>
10 #include <linux/init.h>
11 #include <linux/io.h>
12 #include <linux/module.h>
13 #include <linux/slab.h>
14 #include <linux/vmalloc.h>
15 
16 #include <asm/cacheflush.h>
17 #include <asm/e820.h>
18 #include <asm/fixmap.h>
19 #include <asm/pgtable.h>
20 #include <asm/tlbflush.h>
21 #include <asm/pgalloc.h>
22 
23 enum ioremap_mode {
24 	IOR_MODE_UNCACHED,
25 	IOR_MODE_CACHED,
26 };
27 
28 #ifdef CONFIG_X86_64
29 
30 unsigned long __phys_addr(unsigned long x)
31 {
32 	if (x >= __START_KERNEL_map)
33 		return x - __START_KERNEL_map + phys_base;
34 	return x - PAGE_OFFSET;
35 }
36 EXPORT_SYMBOL(__phys_addr);
37 
38 #endif
39 
40 int page_is_ram(unsigned long pagenr)
41 {
42 	unsigned long addr, end;
43 	int i;
44 
45 	for (i = 0; i < e820.nr_map; i++) {
46 		/*
47 		 * Not usable memory:
48 		 */
49 		if (e820.map[i].type != E820_RAM)
50 			continue;
51 		addr = (e820.map[i].addr + PAGE_SIZE-1) >> PAGE_SHIFT;
52 		end = (e820.map[i].addr + e820.map[i].size) >> PAGE_SHIFT;
53 
54 		/*
55 		 * Sanity check: Some BIOSen report areas as RAM that
56 		 * are not. Notably the 640->1Mb area, which is the
57 		 * PCI BIOS area.
58 		 */
59 		if (addr >= (BIOS_BEGIN >> PAGE_SHIFT) &&
60 		    end < (BIOS_END >> PAGE_SHIFT))
61 			continue;
62 
63 		if ((pagenr >= addr) && (pagenr < end))
64 			return 1;
65 	}
66 	return 0;
67 }
68 
69 /*
70  * Fix up the linear direct mapping of the kernel to avoid cache attribute
71  * conflicts.
72  */
73 static int ioremap_change_attr(unsigned long paddr, unsigned long size,
74 			       enum ioremap_mode mode)
75 {
76 	unsigned long vaddr = (unsigned long)__va(paddr);
77 	unsigned long nrpages = size >> PAGE_SHIFT;
78 	unsigned int level;
79 	int err;
80 
81 	/* No change for pages after the last mapping */
82 	if ((paddr + size - 1) >= (max_pfn_mapped << PAGE_SHIFT))
83 		return 0;
84 
85 	/*
86 	 * If there is no identity map for this address,
87 	 * change_page_attr_addr is unnecessary
88 	 */
89 	if (!lookup_address(vaddr, &level))
90 		return 0;
91 
92 	switch (mode) {
93 	case IOR_MODE_UNCACHED:
94 	default:
95 		err = set_memory_uc(vaddr, nrpages);
96 		break;
97 	case IOR_MODE_CACHED:
98 		err = set_memory_wb(vaddr, nrpages);
99 		break;
100 	}
101 
102 	return err;
103 }
104 
105 /*
106  * Remap an arbitrary physical address space into the kernel virtual
107  * address space. Needed when the kernel wants to access high addresses
108  * directly.
109  *
110  * NOTE! We need to allow non-page-aligned mappings too: we will obviously
111  * have to convert them into an offset in a page-aligned mapping, but the
112  * caller shouldn't need to know that small detail.
113  */
114 static void __iomem *__ioremap(unsigned long phys_addr, unsigned long size,
115 			       enum ioremap_mode mode)
116 {
117 	void __iomem *addr;
118 	struct vm_struct *area;
119 	unsigned long offset, last_addr;
120 	pgprot_t prot;
121 
122 	/* Don't allow wraparound or zero size */
123 	last_addr = phys_addr + size - 1;
124 	if (!size || last_addr < phys_addr)
125 		return NULL;
126 
127 	/*
128 	 * Don't remap the low PCI/ISA area, it's always mapped..
129 	 */
130 	if (phys_addr >= ISA_START_ADDRESS && last_addr < ISA_END_ADDRESS)
131 		return (__force void __iomem *)phys_to_virt(phys_addr);
132 
133 	/*
134 	 * Don't allow anybody to remap normal RAM that we're using..
135 	 */
136 	for (offset = phys_addr >> PAGE_SHIFT; offset < max_pfn_mapped &&
137 	     (offset << PAGE_SHIFT) < last_addr; offset++) {
138 		if (page_is_ram(offset))
139 			return NULL;
140 	}
141 
142 	switch (mode) {
143 	case IOR_MODE_UNCACHED:
144 	default:
145 		prot = PAGE_KERNEL_NOCACHE;
146 		break;
147 	case IOR_MODE_CACHED:
148 		prot = PAGE_KERNEL;
149 		break;
150 	}
151 
152 	/*
153 	 * Mappings have to be page-aligned
154 	 */
155 	offset = phys_addr & ~PAGE_MASK;
156 	phys_addr &= PAGE_MASK;
157 	size = PAGE_ALIGN(last_addr+1) - phys_addr;
158 
159 	/*
160 	 * Ok, go for it..
161 	 */
162 	area = get_vm_area(size, VM_IOREMAP);
163 	if (!area)
164 		return NULL;
165 	area->phys_addr = phys_addr;
166 	addr = (void __iomem *) area->addr;
167 	if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
168 			       phys_addr, prot)) {
169 		remove_vm_area((void *)(PAGE_MASK & (unsigned long) addr));
170 		return NULL;
171 	}
172 
173 	if (ioremap_change_attr(phys_addr, size, mode) < 0) {
174 		vunmap(addr);
175 		return NULL;
176 	}
177 
178 	return (void __iomem *) (offset + (char __iomem *)addr);
179 }
180 
181 /**
182  * ioremap_nocache     -   map bus memory into CPU space
183  * @offset:    bus address of the memory
184  * @size:      size of the resource to map
185  *
186  * ioremap_nocache performs a platform specific sequence of operations to
187  * make bus memory CPU accessible via the readb/readw/readl/writeb/
188  * writew/writel functions and the other mmio helpers. The returned
189  * address is not guaranteed to be usable directly as a virtual
190  * address.
191  *
192  * This version of ioremap ensures that the memory is marked uncachable
193  * on the CPU as well as honouring existing caching rules from things like
194  * the PCI bus. Note that there are other caches and buffers on many
195  * busses. In particular driver authors should read up on PCI writes
196  *
197  * It's useful if some control registers are in such an area and
198  * write combining or read caching is not desirable:
199  *
200  * Must be freed with iounmap.
201  */
202 void __iomem *ioremap_nocache(unsigned long phys_addr, unsigned long size)
203 {
204 	return __ioremap(phys_addr, size, IOR_MODE_UNCACHED);
205 }
206 EXPORT_SYMBOL(ioremap_nocache);
207 
208 void __iomem *ioremap_cache(unsigned long phys_addr, unsigned long size)
209 {
210 	return __ioremap(phys_addr, size, IOR_MODE_CACHED);
211 }
212 EXPORT_SYMBOL(ioremap_cache);
213 
214 /**
215  * iounmap - Free a IO remapping
216  * @addr: virtual address from ioremap_*
217  *
218  * Caller must ensure there is only one unmapping for the same pointer.
219  */
220 void iounmap(volatile void __iomem *addr)
221 {
222 	struct vm_struct *p, *o;
223 
224 	if ((void __force *)addr <= high_memory)
225 		return;
226 
227 	/*
228 	 * __ioremap special-cases the PCI/ISA range by not instantiating a
229 	 * vm_area and by simply returning an address into the kernel mapping
230 	 * of ISA space.   So handle that here.
231 	 */
232 	if (addr >= phys_to_virt(ISA_START_ADDRESS) &&
233 	    addr < phys_to_virt(ISA_END_ADDRESS))
234 		return;
235 
236 	addr = (volatile void __iomem *)
237 		(PAGE_MASK & (unsigned long __force)addr);
238 
239 	/* Use the vm area unlocked, assuming the caller
240 	   ensures there isn't another iounmap for the same address
241 	   in parallel. Reuse of the virtual address is prevented by
242 	   leaving it in the global lists until we're done with it.
243 	   cpa takes care of the direct mappings. */
244 	read_lock(&vmlist_lock);
245 	for (p = vmlist; p; p = p->next) {
246 		if (p->addr == addr)
247 			break;
248 	}
249 	read_unlock(&vmlist_lock);
250 
251 	if (!p) {
252 		printk(KERN_ERR "iounmap: bad address %p\n", addr);
253 		dump_stack();
254 		return;
255 	}
256 
257 	/* Reset the direct mapping. Can block */
258 	ioremap_change_attr(p->phys_addr, p->size, IOR_MODE_CACHED);
259 
260 	/* Finally remove it */
261 	o = remove_vm_area((void *)addr);
262 	BUG_ON(p != o || o == NULL);
263 	kfree(p);
264 }
265 EXPORT_SYMBOL(iounmap);
266 
267 #ifdef CONFIG_X86_32
268 
269 int __initdata early_ioremap_debug;
270 
271 static int __init early_ioremap_debug_setup(char *str)
272 {
273 	early_ioremap_debug = 1;
274 
275 	return 0;
276 }
277 early_param("early_ioremap_debug", early_ioremap_debug_setup);
278 
279 static __initdata int after_paging_init;
280 static __initdata unsigned long bm_pte[1024]
281 				__attribute__((aligned(PAGE_SIZE)));
282 
283 static inline unsigned long * __init early_ioremap_pgd(unsigned long addr)
284 {
285 	return (unsigned long *)swapper_pg_dir + ((addr >> 22) & 1023);
286 }
287 
288 static inline unsigned long * __init early_ioremap_pte(unsigned long addr)
289 {
290 	return bm_pte + ((addr >> PAGE_SHIFT) & 1023);
291 }
292 
293 void __init early_ioremap_init(void)
294 {
295 	unsigned long *pgd;
296 
297 	if (early_ioremap_debug)
298 		printk(KERN_INFO "early_ioremap_init()\n");
299 
300 	pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
301 	*pgd = __pa(bm_pte) | _PAGE_TABLE;
302 	memset(bm_pte, 0, sizeof(bm_pte));
303 	/*
304 	 * The boot-ioremap range spans multiple pgds, for which
305 	 * we are not prepared:
306 	 */
307 	if (pgd != early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END))) {
308 		WARN_ON(1);
309 		printk(KERN_WARNING "pgd %p != %p\n",
310 		       pgd, early_ioremap_pgd(fix_to_virt(FIX_BTMAP_END)));
311 		printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_BEGIN): %08lx\n",
312 		       fix_to_virt(FIX_BTMAP_BEGIN));
313 		printk(KERN_WARNING "fix_to_virt(FIX_BTMAP_END):   %08lx\n",
314 		       fix_to_virt(FIX_BTMAP_END));
315 
316 		printk(KERN_WARNING "FIX_BTMAP_END:       %d\n", FIX_BTMAP_END);
317 		printk(KERN_WARNING "FIX_BTMAP_BEGIN:     %d\n",
318 		       FIX_BTMAP_BEGIN);
319 	}
320 }
321 
322 void __init early_ioremap_clear(void)
323 {
324 	unsigned long *pgd;
325 
326 	if (early_ioremap_debug)
327 		printk(KERN_INFO "early_ioremap_clear()\n");
328 
329 	pgd = early_ioremap_pgd(fix_to_virt(FIX_BTMAP_BEGIN));
330 	*pgd = 0;
331 	paravirt_release_pt(__pa(pgd) >> PAGE_SHIFT);
332 	__flush_tlb_all();
333 }
334 
335 void __init early_ioremap_reset(void)
336 {
337 	enum fixed_addresses idx;
338 	unsigned long *pte, phys, addr;
339 
340 	after_paging_init = 1;
341 	for (idx = FIX_BTMAP_BEGIN; idx >= FIX_BTMAP_END; idx--) {
342 		addr = fix_to_virt(idx);
343 		pte = early_ioremap_pte(addr);
344 		if (*pte & _PAGE_PRESENT) {
345 			phys = *pte & PAGE_MASK;
346 			set_fixmap(idx, phys);
347 		}
348 	}
349 }
350 
351 static void __init __early_set_fixmap(enum fixed_addresses idx,
352 				   unsigned long phys, pgprot_t flags)
353 {
354 	unsigned long *pte, addr = __fix_to_virt(idx);
355 
356 	if (idx >= __end_of_fixed_addresses) {
357 		BUG();
358 		return;
359 	}
360 	pte = early_ioremap_pte(addr);
361 	if (pgprot_val(flags))
362 		*pte = (phys & PAGE_MASK) | pgprot_val(flags);
363 	else
364 		*pte = 0;
365 	__flush_tlb_one(addr);
366 }
367 
368 static inline void __init early_set_fixmap(enum fixed_addresses idx,
369 					unsigned long phys)
370 {
371 	if (after_paging_init)
372 		set_fixmap(idx, phys);
373 	else
374 		__early_set_fixmap(idx, phys, PAGE_KERNEL);
375 }
376 
377 static inline void __init early_clear_fixmap(enum fixed_addresses idx)
378 {
379 	if (after_paging_init)
380 		clear_fixmap(idx);
381 	else
382 		__early_set_fixmap(idx, 0, __pgprot(0));
383 }
384 
385 
386 int __initdata early_ioremap_nested;
387 
388 static int __init check_early_ioremap_leak(void)
389 {
390 	if (!early_ioremap_nested)
391 		return 0;
392 
393 	printk(KERN_WARNING
394 	       "Debug warning: early ioremap leak of %d areas detected.\n",
395 	       early_ioremap_nested);
396 	printk(KERN_WARNING
397 	       "please boot with early_ioremap_debug and report the dmesg.\n");
398 	WARN_ON(1);
399 
400 	return 1;
401 }
402 late_initcall(check_early_ioremap_leak);
403 
404 void __init *early_ioremap(unsigned long phys_addr, unsigned long size)
405 {
406 	unsigned long offset, last_addr;
407 	unsigned int nrpages, nesting;
408 	enum fixed_addresses idx0, idx;
409 
410 	WARN_ON(system_state != SYSTEM_BOOTING);
411 
412 	nesting = early_ioremap_nested;
413 	if (early_ioremap_debug) {
414 		printk(KERN_INFO "early_ioremap(%08lx, %08lx) [%d] => ",
415 		       phys_addr, size, nesting);
416 		dump_stack();
417 	}
418 
419 	/* Don't allow wraparound or zero size */
420 	last_addr = phys_addr + size - 1;
421 	if (!size || last_addr < phys_addr) {
422 		WARN_ON(1);
423 		return NULL;
424 	}
425 
426 	if (nesting >= FIX_BTMAPS_NESTING) {
427 		WARN_ON(1);
428 		return NULL;
429 	}
430 	early_ioremap_nested++;
431 	/*
432 	 * Mappings have to be page-aligned
433 	 */
434 	offset = phys_addr & ~PAGE_MASK;
435 	phys_addr &= PAGE_MASK;
436 	size = PAGE_ALIGN(last_addr) - phys_addr;
437 
438 	/*
439 	 * Mappings have to fit in the FIX_BTMAP area.
440 	 */
441 	nrpages = size >> PAGE_SHIFT;
442 	if (nrpages > NR_FIX_BTMAPS) {
443 		WARN_ON(1);
444 		return NULL;
445 	}
446 
447 	/*
448 	 * Ok, go for it..
449 	 */
450 	idx0 = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
451 	idx = idx0;
452 	while (nrpages > 0) {
453 		early_set_fixmap(idx, phys_addr);
454 		phys_addr += PAGE_SIZE;
455 		--idx;
456 		--nrpages;
457 	}
458 	if (early_ioremap_debug)
459 		printk(KERN_CONT "%08lx + %08lx\n", offset, fix_to_virt(idx0));
460 
461 	return (void *) (offset + fix_to_virt(idx0));
462 }
463 
464 void __init early_iounmap(void *addr, unsigned long size)
465 {
466 	unsigned long virt_addr;
467 	unsigned long offset;
468 	unsigned int nrpages;
469 	enum fixed_addresses idx;
470 	unsigned int nesting;
471 
472 	nesting = --early_ioremap_nested;
473 	WARN_ON(nesting < 0);
474 
475 	if (early_ioremap_debug) {
476 		printk(KERN_INFO "early_iounmap(%p, %08lx) [%d]\n", addr,
477 		       size, nesting);
478 		dump_stack();
479 	}
480 
481 	virt_addr = (unsigned long)addr;
482 	if (virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)) {
483 		WARN_ON(1);
484 		return;
485 	}
486 	offset = virt_addr & ~PAGE_MASK;
487 	nrpages = PAGE_ALIGN(offset + size - 1) >> PAGE_SHIFT;
488 
489 	idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*nesting;
490 	while (nrpages > 0) {
491 		early_clear_fixmap(idx);
492 		--idx;
493 		--nrpages;
494 	}
495 }
496 
497 void __this_fixmap_does_not_exist(void)
498 {
499 	WARN_ON(1);
500 }
501 
502 #endif /* CONFIG_X86_32 */
503