1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Re-map IO memory to kernel address space so that we can access it. 4 * This is needed for high PCI addresses that aren't mapped in the 5 * 640k-1MB IO memory area on PC's 6 * 7 * (C) Copyright 1995 1996 Linus Torvalds 8 */ 9 #include <linux/vmalloc.h> 10 #include <linux/mm.h> 11 #include <linux/sched.h> 12 #include <linux/io.h> 13 #include <linux/export.h> 14 #include <asm/cacheflush.h> 15 16 #include "pgalloc-track.h" 17 18 #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 19 static int __read_mostly ioremap_p4d_capable; 20 static int __read_mostly ioremap_pud_capable; 21 static int __read_mostly ioremap_pmd_capable; 22 static int __read_mostly ioremap_huge_disabled; 23 24 static int __init set_nohugeiomap(char *str) 25 { 26 ioremap_huge_disabled = 1; 27 return 0; 28 } 29 early_param("nohugeiomap", set_nohugeiomap); 30 31 void __init ioremap_huge_init(void) 32 { 33 if (!ioremap_huge_disabled) { 34 if (arch_ioremap_p4d_supported()) 35 ioremap_p4d_capable = 1; 36 if (arch_ioremap_pud_supported()) 37 ioremap_pud_capable = 1; 38 if (arch_ioremap_pmd_supported()) 39 ioremap_pmd_capable = 1; 40 } 41 } 42 43 static inline int ioremap_p4d_enabled(void) 44 { 45 return ioremap_p4d_capable; 46 } 47 48 static inline int ioremap_pud_enabled(void) 49 { 50 return ioremap_pud_capable; 51 } 52 53 static inline int ioremap_pmd_enabled(void) 54 { 55 return ioremap_pmd_capable; 56 } 57 58 #else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */ 59 static inline int ioremap_p4d_enabled(void) { return 0; } 60 static inline int ioremap_pud_enabled(void) { return 0; } 61 static inline int ioremap_pmd_enabled(void) { return 0; } 62 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 63 64 static int ioremap_pte_range(pmd_t *pmd, unsigned long addr, 65 unsigned long end, phys_addr_t phys_addr, pgprot_t prot, 66 pgtbl_mod_mask *mask) 67 { 68 pte_t *pte; 69 u64 pfn; 70 71 pfn = phys_addr >> PAGE_SHIFT; 72 pte = pte_alloc_kernel_track(pmd, addr, mask); 73 if (!pte) 74 return -ENOMEM; 75 do { 76 BUG_ON(!pte_none(*pte)); 77 set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot)); 78 pfn++; 79 } while (pte++, addr += PAGE_SIZE, addr != end); 80 *mask |= PGTBL_PTE_MODIFIED; 81 return 0; 82 } 83 84 static int ioremap_try_huge_pmd(pmd_t *pmd, unsigned long addr, 85 unsigned long end, phys_addr_t phys_addr, 86 pgprot_t prot) 87 { 88 if (!ioremap_pmd_enabled()) 89 return 0; 90 91 if ((end - addr) != PMD_SIZE) 92 return 0; 93 94 if (!IS_ALIGNED(addr, PMD_SIZE)) 95 return 0; 96 97 if (!IS_ALIGNED(phys_addr, PMD_SIZE)) 98 return 0; 99 100 if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) 101 return 0; 102 103 return pmd_set_huge(pmd, phys_addr, prot); 104 } 105 106 static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr, 107 unsigned long end, phys_addr_t phys_addr, pgprot_t prot, 108 pgtbl_mod_mask *mask) 109 { 110 pmd_t *pmd; 111 unsigned long next; 112 113 pmd = pmd_alloc_track(&init_mm, pud, addr, mask); 114 if (!pmd) 115 return -ENOMEM; 116 do { 117 next = pmd_addr_end(addr, end); 118 119 if (ioremap_try_huge_pmd(pmd, addr, next, phys_addr, prot)) { 120 *mask |= PGTBL_PMD_MODIFIED; 121 continue; 122 } 123 124 if (ioremap_pte_range(pmd, addr, next, phys_addr, prot, mask)) 125 return -ENOMEM; 126 } while (pmd++, phys_addr += (next - addr), addr = next, addr != end); 127 return 0; 128 } 129 130 static int ioremap_try_huge_pud(pud_t *pud, unsigned long addr, 131 unsigned long end, phys_addr_t phys_addr, 132 pgprot_t prot) 133 { 134 if (!ioremap_pud_enabled()) 135 return 0; 136 137 if ((end - addr) != PUD_SIZE) 138 return 0; 139 140 if (!IS_ALIGNED(addr, PUD_SIZE)) 141 return 0; 142 143 if (!IS_ALIGNED(phys_addr, PUD_SIZE)) 144 return 0; 145 146 if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) 147 return 0; 148 149 return pud_set_huge(pud, phys_addr, prot); 150 } 151 152 static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr, 153 unsigned long end, phys_addr_t phys_addr, pgprot_t prot, 154 pgtbl_mod_mask *mask) 155 { 156 pud_t *pud; 157 unsigned long next; 158 159 pud = pud_alloc_track(&init_mm, p4d, addr, mask); 160 if (!pud) 161 return -ENOMEM; 162 do { 163 next = pud_addr_end(addr, end); 164 165 if (ioremap_try_huge_pud(pud, addr, next, phys_addr, prot)) { 166 *mask |= PGTBL_PUD_MODIFIED; 167 continue; 168 } 169 170 if (ioremap_pmd_range(pud, addr, next, phys_addr, prot, mask)) 171 return -ENOMEM; 172 } while (pud++, phys_addr += (next - addr), addr = next, addr != end); 173 return 0; 174 } 175 176 static int ioremap_try_huge_p4d(p4d_t *p4d, unsigned long addr, 177 unsigned long end, phys_addr_t phys_addr, 178 pgprot_t prot) 179 { 180 if (!ioremap_p4d_enabled()) 181 return 0; 182 183 if ((end - addr) != P4D_SIZE) 184 return 0; 185 186 if (!IS_ALIGNED(addr, P4D_SIZE)) 187 return 0; 188 189 if (!IS_ALIGNED(phys_addr, P4D_SIZE)) 190 return 0; 191 192 if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) 193 return 0; 194 195 return p4d_set_huge(p4d, phys_addr, prot); 196 } 197 198 static inline int ioremap_p4d_range(pgd_t *pgd, unsigned long addr, 199 unsigned long end, phys_addr_t phys_addr, pgprot_t prot, 200 pgtbl_mod_mask *mask) 201 { 202 p4d_t *p4d; 203 unsigned long next; 204 205 p4d = p4d_alloc_track(&init_mm, pgd, addr, mask); 206 if (!p4d) 207 return -ENOMEM; 208 do { 209 next = p4d_addr_end(addr, end); 210 211 if (ioremap_try_huge_p4d(p4d, addr, next, phys_addr, prot)) { 212 *mask |= PGTBL_P4D_MODIFIED; 213 continue; 214 } 215 216 if (ioremap_pud_range(p4d, addr, next, phys_addr, prot, mask)) 217 return -ENOMEM; 218 } while (p4d++, phys_addr += (next - addr), addr = next, addr != end); 219 return 0; 220 } 221 222 int ioremap_page_range(unsigned long addr, 223 unsigned long end, phys_addr_t phys_addr, pgprot_t prot) 224 { 225 pgd_t *pgd; 226 unsigned long start; 227 unsigned long next; 228 int err; 229 pgtbl_mod_mask mask = 0; 230 231 might_sleep(); 232 BUG_ON(addr >= end); 233 234 start = addr; 235 pgd = pgd_offset_k(addr); 236 do { 237 next = pgd_addr_end(addr, end); 238 err = ioremap_p4d_range(pgd, addr, next, phys_addr, prot, 239 &mask); 240 if (err) 241 break; 242 } while (pgd++, phys_addr += (next - addr), addr = next, addr != end); 243 244 flush_cache_vmap(start, end); 245 246 if (mask & ARCH_PAGE_TABLE_SYNC_MASK) 247 arch_sync_kernel_mappings(start, end); 248 249 return err; 250 } 251 252 #ifdef CONFIG_GENERIC_IOREMAP 253 void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) 254 { 255 unsigned long offset, vaddr; 256 phys_addr_t last_addr; 257 struct vm_struct *area; 258 259 /* Disallow wrap-around or zero size */ 260 last_addr = addr + size - 1; 261 if (!size || last_addr < addr) 262 return NULL; 263 264 /* Page-align mappings */ 265 offset = addr & (~PAGE_MASK); 266 addr -= offset; 267 size = PAGE_ALIGN(size + offset); 268 269 area = get_vm_area_caller(size, VM_IOREMAP, 270 __builtin_return_address(0)); 271 if (!area) 272 return NULL; 273 vaddr = (unsigned long)area->addr; 274 275 if (ioremap_page_range(vaddr, vaddr + size, addr, __pgprot(prot))) { 276 free_vm_area(area); 277 return NULL; 278 } 279 280 return (void __iomem *)(vaddr + offset); 281 } 282 EXPORT_SYMBOL(ioremap_prot); 283 284 void iounmap(volatile void __iomem *addr) 285 { 286 vunmap((void *)((unsigned long)addr & PAGE_MASK)); 287 } 288 EXPORT_SYMBOL(iounmap); 289 #endif /* CONFIG_GENERIC_IOREMAP */ 290