1 /* 2 * Copyright IBM Corp. 2006 3 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> 4 */ 5 6 #include <linux/bootmem.h> 7 #include <linux/pfn.h> 8 #include <linux/mm.h> 9 #include <linux/module.h> 10 #include <linux/list.h> 11 #include <linux/hugetlb.h> 12 #include <linux/slab.h> 13 #include <linux/memblock.h> 14 #include <asm/pgalloc.h> 15 #include <asm/pgtable.h> 16 #include <asm/setup.h> 17 #include <asm/tlbflush.h> 18 #include <asm/sections.h> 19 20 static DEFINE_MUTEX(vmem_mutex); 21 22 struct memory_segment { 23 struct list_head list; 24 unsigned long start; 25 unsigned long size; 26 }; 27 28 static LIST_HEAD(mem_segs); 29 30 static void __ref *vmem_alloc_pages(unsigned int order) 31 { 32 if (slab_is_available()) 33 return (void *)__get_free_pages(GFP_KERNEL, order); 34 return alloc_bootmem_pages((1 << order) * PAGE_SIZE); 35 } 36 37 static inline pud_t *vmem_pud_alloc(void) 38 { 39 pud_t *pud = NULL; 40 41 pud = vmem_alloc_pages(2); 42 if (!pud) 43 return NULL; 44 clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); 45 return pud; 46 } 47 48 static inline pmd_t *vmem_pmd_alloc(void) 49 { 50 pmd_t *pmd = NULL; 51 52 pmd = vmem_alloc_pages(2); 53 if (!pmd) 54 return NULL; 55 clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); 56 return pmd; 57 } 58 59 static pte_t __ref *vmem_pte_alloc(void) 60 { 61 pte_t *pte; 62 63 if (slab_is_available()) 64 pte = (pte_t *) page_table_alloc(&init_mm); 65 else 66 pte = alloc_bootmem_align(PTRS_PER_PTE * sizeof(pte_t), 67 PTRS_PER_PTE * sizeof(pte_t)); 68 if (!pte) 69 return NULL; 70 clear_table((unsigned long *) pte, _PAGE_INVALID, 71 PTRS_PER_PTE * sizeof(pte_t)); 72 return pte; 73 } 74 75 /* 76 * Add a physical memory range to the 1:1 mapping. 77 */ 78 static int vmem_add_mem(unsigned long start, unsigned long size, int ro) 79 { 80 unsigned long end = start + size; 81 unsigned long address = start; 82 pgd_t *pg_dir; 83 pud_t *pu_dir; 84 pmd_t *pm_dir; 85 pte_t *pt_dir; 86 int ret = -ENOMEM; 87 88 while (address < end) { 89 pg_dir = pgd_offset_k(address); 90 if (pgd_none(*pg_dir)) { 91 pu_dir = vmem_pud_alloc(); 92 if (!pu_dir) 93 goto out; 94 pgd_populate(&init_mm, pg_dir, pu_dir); 95 } 96 pu_dir = pud_offset(pg_dir, address); 97 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address && 98 !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) && 99 !debug_pagealloc_enabled()) { 100 pud_val(*pu_dir) = __pa(address) | 101 _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE | 102 (ro ? _REGION_ENTRY_PROTECT : 0); 103 address += PUD_SIZE; 104 continue; 105 } 106 if (pud_none(*pu_dir)) { 107 pm_dir = vmem_pmd_alloc(); 108 if (!pm_dir) 109 goto out; 110 pud_populate(&init_mm, pu_dir, pm_dir); 111 } 112 pm_dir = pmd_offset(pu_dir, address); 113 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address && 114 !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) && 115 !debug_pagealloc_enabled()) { 116 pmd_val(*pm_dir) = __pa(address) | 117 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE | 118 _SEGMENT_ENTRY_YOUNG | 119 (ro ? _SEGMENT_ENTRY_PROTECT : 0); 120 address += PMD_SIZE; 121 continue; 122 } 123 if (pmd_none(*pm_dir)) { 124 pt_dir = vmem_pte_alloc(); 125 if (!pt_dir) 126 goto out; 127 pmd_populate(&init_mm, pm_dir, pt_dir); 128 } 129 130 pt_dir = pte_offset_kernel(pm_dir, address); 131 pte_val(*pt_dir) = __pa(address) | 132 pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); 133 address += PAGE_SIZE; 134 } 135 ret = 0; 136 out: 137 return ret; 138 } 139 140 /* 141 * Remove a physical memory range from the 1:1 mapping. 142 * Currently only invalidates page table entries. 143 */ 144 static void vmem_remove_range(unsigned long start, unsigned long size) 145 { 146 unsigned long end = start + size; 147 unsigned long address = start; 148 pgd_t *pg_dir; 149 pud_t *pu_dir; 150 pmd_t *pm_dir; 151 pte_t *pt_dir; 152 pte_t pte; 153 154 pte_val(pte) = _PAGE_INVALID; 155 while (address < end) { 156 pg_dir = pgd_offset_k(address); 157 if (pgd_none(*pg_dir)) { 158 address += PGDIR_SIZE; 159 continue; 160 } 161 pu_dir = pud_offset(pg_dir, address); 162 if (pud_none(*pu_dir)) { 163 address += PUD_SIZE; 164 continue; 165 } 166 if (pud_large(*pu_dir)) { 167 pud_clear(pu_dir); 168 address += PUD_SIZE; 169 continue; 170 } 171 pm_dir = pmd_offset(pu_dir, address); 172 if (pmd_none(*pm_dir)) { 173 address += PMD_SIZE; 174 continue; 175 } 176 if (pmd_large(*pm_dir)) { 177 pmd_clear(pm_dir); 178 address += PMD_SIZE; 179 continue; 180 } 181 pt_dir = pte_offset_kernel(pm_dir, address); 182 *pt_dir = pte; 183 address += PAGE_SIZE; 184 } 185 flush_tlb_kernel_range(start, end); 186 } 187 188 /* 189 * Add a backed mem_map array to the virtual mem_map array. 190 */ 191 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 192 { 193 unsigned long address = start; 194 pgd_t *pg_dir; 195 pud_t *pu_dir; 196 pmd_t *pm_dir; 197 pte_t *pt_dir; 198 int ret = -ENOMEM; 199 200 for (address = start; address < end;) { 201 pg_dir = pgd_offset_k(address); 202 if (pgd_none(*pg_dir)) { 203 pu_dir = vmem_pud_alloc(); 204 if (!pu_dir) 205 goto out; 206 pgd_populate(&init_mm, pg_dir, pu_dir); 207 } 208 209 pu_dir = pud_offset(pg_dir, address); 210 if (pud_none(*pu_dir)) { 211 pm_dir = vmem_pmd_alloc(); 212 if (!pm_dir) 213 goto out; 214 pud_populate(&init_mm, pu_dir, pm_dir); 215 } 216 217 pm_dir = pmd_offset(pu_dir, address); 218 if (pmd_none(*pm_dir)) { 219 /* Use 1MB frames for vmemmap if available. We always 220 * use large frames even if they are only partially 221 * used. 222 * Otherwise we would have also page tables since 223 * vmemmap_populate gets called for each section 224 * separately. */ 225 if (MACHINE_HAS_EDAT1) { 226 void *new_page; 227 228 new_page = vmemmap_alloc_block(PMD_SIZE, node); 229 if (!new_page) 230 goto out; 231 pmd_val(*pm_dir) = __pa(new_page) | 232 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE; 233 address = (address + PMD_SIZE) & PMD_MASK; 234 continue; 235 } 236 pt_dir = vmem_pte_alloc(); 237 if (!pt_dir) 238 goto out; 239 pmd_populate(&init_mm, pm_dir, pt_dir); 240 } else if (pmd_large(*pm_dir)) { 241 address = (address + PMD_SIZE) & PMD_MASK; 242 continue; 243 } 244 245 pt_dir = pte_offset_kernel(pm_dir, address); 246 if (pte_none(*pt_dir)) { 247 void *new_page; 248 249 new_page = vmemmap_alloc_block(PAGE_SIZE, node); 250 if (!new_page) 251 goto out; 252 pte_val(*pt_dir) = 253 __pa(new_page) | pgprot_val(PAGE_KERNEL); 254 } 255 address += PAGE_SIZE; 256 } 257 ret = 0; 258 out: 259 return ret; 260 } 261 262 void vmemmap_free(unsigned long start, unsigned long end) 263 { 264 } 265 266 /* 267 * Add memory segment to the segment list if it doesn't overlap with 268 * an already present segment. 269 */ 270 static int insert_memory_segment(struct memory_segment *seg) 271 { 272 struct memory_segment *tmp; 273 274 if (seg->start + seg->size > VMEM_MAX_PHYS || 275 seg->start + seg->size < seg->start) 276 return -ERANGE; 277 278 list_for_each_entry(tmp, &mem_segs, list) { 279 if (seg->start >= tmp->start + tmp->size) 280 continue; 281 if (seg->start + seg->size <= tmp->start) 282 continue; 283 return -ENOSPC; 284 } 285 list_add(&seg->list, &mem_segs); 286 return 0; 287 } 288 289 /* 290 * Remove memory segment from the segment list. 291 */ 292 static void remove_memory_segment(struct memory_segment *seg) 293 { 294 list_del(&seg->list); 295 } 296 297 static void __remove_shared_memory(struct memory_segment *seg) 298 { 299 remove_memory_segment(seg); 300 vmem_remove_range(seg->start, seg->size); 301 } 302 303 int vmem_remove_mapping(unsigned long start, unsigned long size) 304 { 305 struct memory_segment *seg; 306 int ret; 307 308 mutex_lock(&vmem_mutex); 309 310 ret = -ENOENT; 311 list_for_each_entry(seg, &mem_segs, list) { 312 if (seg->start == start && seg->size == size) 313 break; 314 } 315 316 if (seg->start != start || seg->size != size) 317 goto out; 318 319 ret = 0; 320 __remove_shared_memory(seg); 321 kfree(seg); 322 out: 323 mutex_unlock(&vmem_mutex); 324 return ret; 325 } 326 327 int vmem_add_mapping(unsigned long start, unsigned long size) 328 { 329 struct memory_segment *seg; 330 int ret; 331 332 mutex_lock(&vmem_mutex); 333 ret = -ENOMEM; 334 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 335 if (!seg) 336 goto out; 337 seg->start = start; 338 seg->size = size; 339 340 ret = insert_memory_segment(seg); 341 if (ret) 342 goto out_free; 343 344 ret = vmem_add_mem(start, size, 0); 345 if (ret) 346 goto out_remove; 347 goto out; 348 349 out_remove: 350 __remove_shared_memory(seg); 351 out_free: 352 kfree(seg); 353 out: 354 mutex_unlock(&vmem_mutex); 355 return ret; 356 } 357 358 /* 359 * map whole physical memory to virtual memory (identity mapping) 360 * we reserve enough space in the vmalloc area for vmemmap to hotplug 361 * additional memory segments. 362 */ 363 void __init vmem_map_init(void) 364 { 365 unsigned long ro_start, ro_end; 366 struct memblock_region *reg; 367 phys_addr_t start, end; 368 369 ro_start = PFN_ALIGN((unsigned long)&_stext); 370 ro_end = (unsigned long)&_eshared & PAGE_MASK; 371 for_each_memblock(memory, reg) { 372 start = reg->base; 373 end = reg->base + reg->size; 374 if (start >= ro_end || end <= ro_start) 375 vmem_add_mem(start, end - start, 0); 376 else if (start >= ro_start && end <= ro_end) 377 vmem_add_mem(start, end - start, 1); 378 else if (start >= ro_start) { 379 vmem_add_mem(start, ro_end - start, 1); 380 vmem_add_mem(ro_end, end - ro_end, 0); 381 } else if (end < ro_end) { 382 vmem_add_mem(start, ro_start - start, 0); 383 vmem_add_mem(ro_start, end - ro_start, 1); 384 } else { 385 vmem_add_mem(start, ro_start - start, 0); 386 vmem_add_mem(ro_start, ro_end - ro_start, 1); 387 vmem_add_mem(ro_end, end - ro_end, 0); 388 } 389 } 390 } 391 392 /* 393 * Convert memblock.memory to a memory segment list so there is a single 394 * list that contains all memory segments. 395 */ 396 static int __init vmem_convert_memory_chunk(void) 397 { 398 struct memblock_region *reg; 399 struct memory_segment *seg; 400 401 mutex_lock(&vmem_mutex); 402 for_each_memblock(memory, reg) { 403 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 404 if (!seg) 405 panic("Out of memory...\n"); 406 seg->start = reg->base; 407 seg->size = reg->size; 408 insert_memory_segment(seg); 409 } 410 mutex_unlock(&vmem_mutex); 411 return 0; 412 } 413 414 core_initcall(vmem_convert_memory_chunk); 415