1 /* 2 * Copyright IBM Corp. 2006 3 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> 4 */ 5 6 #include <linux/bootmem.h> 7 #include <linux/pfn.h> 8 #include <linux/mm.h> 9 #include <linux/module.h> 10 #include <linux/list.h> 11 #include <linux/hugetlb.h> 12 #include <linux/slab.h> 13 #include <asm/pgalloc.h> 14 #include <asm/pgtable.h> 15 #include <asm/setup.h> 16 #include <asm/tlbflush.h> 17 #include <asm/sections.h> 18 19 static DEFINE_MUTEX(vmem_mutex); 20 21 struct memory_segment { 22 struct list_head list; 23 unsigned long start; 24 unsigned long size; 25 }; 26 27 static LIST_HEAD(mem_segs); 28 29 static void __ref *vmem_alloc_pages(unsigned int order) 30 { 31 if (slab_is_available()) 32 return (void *)__get_free_pages(GFP_KERNEL, order); 33 return alloc_bootmem_pages((1 << order) * PAGE_SIZE); 34 } 35 36 static inline pud_t *vmem_pud_alloc(void) 37 { 38 pud_t *pud = NULL; 39 40 #ifdef CONFIG_64BIT 41 pud = vmem_alloc_pages(2); 42 if (!pud) 43 return NULL; 44 clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); 45 #endif 46 return pud; 47 } 48 49 static inline pmd_t *vmem_pmd_alloc(void) 50 { 51 pmd_t *pmd = NULL; 52 53 #ifdef CONFIG_64BIT 54 pmd = vmem_alloc_pages(2); 55 if (!pmd) 56 return NULL; 57 clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); 58 #endif 59 return pmd; 60 } 61 62 static pte_t __ref *vmem_pte_alloc(unsigned long address) 63 { 64 pte_t *pte; 65 66 if (slab_is_available()) 67 pte = (pte_t *) page_table_alloc(&init_mm, address); 68 else 69 pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); 70 if (!pte) 71 return NULL; 72 clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY, 73 PTRS_PER_PTE * sizeof(pte_t)); 74 return pte; 75 } 76 77 /* 78 * Add a physical memory range to the 1:1 mapping. 79 */ 80 static int vmem_add_mem(unsigned long start, unsigned long size, int ro) 81 { 82 unsigned long end = start + size; 83 unsigned long address = start; 84 pgd_t *pg_dir; 85 pud_t *pu_dir; 86 pmd_t *pm_dir; 87 pte_t *pt_dir; 88 int ret = -ENOMEM; 89 90 while (address < end) { 91 pg_dir = pgd_offset_k(address); 92 if (pgd_none(*pg_dir)) { 93 pu_dir = vmem_pud_alloc(); 94 if (!pu_dir) 95 goto out; 96 pgd_populate(&init_mm, pg_dir, pu_dir); 97 } 98 pu_dir = pud_offset(pg_dir, address); 99 #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC) 100 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address && 101 !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) { 102 pud_val(*pu_dir) = __pa(address) | 103 _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE | 104 (ro ? _REGION_ENTRY_RO : 0); 105 address += PUD_SIZE; 106 continue; 107 } 108 #endif 109 if (pud_none(*pu_dir)) { 110 pm_dir = vmem_pmd_alloc(); 111 if (!pm_dir) 112 goto out; 113 pud_populate(&init_mm, pu_dir, pm_dir); 114 } 115 pm_dir = pmd_offset(pu_dir, address); 116 #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC) 117 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address && 118 !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) { 119 pmd_val(*pm_dir) = __pa(address) | 120 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE | 121 (ro ? _SEGMENT_ENTRY_RO : 0); 122 address += PMD_SIZE; 123 continue; 124 } 125 #endif 126 if (pmd_none(*pm_dir)) { 127 pt_dir = vmem_pte_alloc(address); 128 if (!pt_dir) 129 goto out; 130 pmd_populate(&init_mm, pm_dir, pt_dir); 131 } 132 133 pt_dir = pte_offset_kernel(pm_dir, address); 134 pte_val(*pt_dir) = __pa(address) | (ro ? _PAGE_RO : 0); 135 address += PAGE_SIZE; 136 } 137 ret = 0; 138 out: 139 flush_tlb_kernel_range(start, end); 140 return ret; 141 } 142 143 /* 144 * Remove a physical memory range from the 1:1 mapping. 145 * Currently only invalidates page table entries. 146 */ 147 static void vmem_remove_range(unsigned long start, unsigned long size) 148 { 149 unsigned long end = start + size; 150 unsigned long address = start; 151 pgd_t *pg_dir; 152 pud_t *pu_dir; 153 pmd_t *pm_dir; 154 pte_t *pt_dir; 155 pte_t pte; 156 157 pte_val(pte) = _PAGE_TYPE_EMPTY; 158 while (address < end) { 159 pg_dir = pgd_offset_k(address); 160 if (pgd_none(*pg_dir)) { 161 address += PGDIR_SIZE; 162 continue; 163 } 164 pu_dir = pud_offset(pg_dir, address); 165 if (pud_none(*pu_dir)) { 166 address += PUD_SIZE; 167 continue; 168 } 169 if (pud_large(*pu_dir)) { 170 pud_clear(pu_dir); 171 address += PUD_SIZE; 172 continue; 173 } 174 pm_dir = pmd_offset(pu_dir, address); 175 if (pmd_none(*pm_dir)) { 176 address += PMD_SIZE; 177 continue; 178 } 179 if (pmd_large(*pm_dir)) { 180 pmd_clear(pm_dir); 181 address += PMD_SIZE; 182 continue; 183 } 184 pt_dir = pte_offset_kernel(pm_dir, address); 185 *pt_dir = pte; 186 address += PAGE_SIZE; 187 } 188 flush_tlb_kernel_range(start, end); 189 } 190 191 /* 192 * Add a backed mem_map array to the virtual mem_map array. 193 */ 194 int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) 195 { 196 unsigned long address, start_addr, end_addr; 197 pgd_t *pg_dir; 198 pud_t *pu_dir; 199 pmd_t *pm_dir; 200 pte_t *pt_dir; 201 int ret = -ENOMEM; 202 203 start_addr = (unsigned long) start; 204 end_addr = (unsigned long) (start + nr); 205 206 for (address = start_addr; address < end_addr;) { 207 pg_dir = pgd_offset_k(address); 208 if (pgd_none(*pg_dir)) { 209 pu_dir = vmem_pud_alloc(); 210 if (!pu_dir) 211 goto out; 212 pgd_populate(&init_mm, pg_dir, pu_dir); 213 } 214 215 pu_dir = pud_offset(pg_dir, address); 216 if (pud_none(*pu_dir)) { 217 pm_dir = vmem_pmd_alloc(); 218 if (!pm_dir) 219 goto out; 220 pud_populate(&init_mm, pu_dir, pm_dir); 221 } 222 223 pm_dir = pmd_offset(pu_dir, address); 224 if (pmd_none(*pm_dir)) { 225 #ifdef CONFIG_64BIT 226 /* Use 1MB frames for vmemmap if available. We always 227 * use large frames even if they are only partially 228 * used. 229 * Otherwise we would have also page tables since 230 * vmemmap_populate gets called for each section 231 * separately. */ 232 if (MACHINE_HAS_EDAT1) { 233 void *new_page; 234 235 new_page = vmemmap_alloc_block(PMD_SIZE, node); 236 if (!new_page) 237 goto out; 238 pmd_val(*pm_dir) = __pa(new_page) | 239 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE | 240 _SEGMENT_ENTRY_CO; 241 address = (address + PMD_SIZE) & PMD_MASK; 242 continue; 243 } 244 #endif 245 pt_dir = vmem_pte_alloc(address); 246 if (!pt_dir) 247 goto out; 248 pmd_populate(&init_mm, pm_dir, pt_dir); 249 } else if (pmd_large(*pm_dir)) { 250 address = (address + PMD_SIZE) & PMD_MASK; 251 continue; 252 } 253 254 pt_dir = pte_offset_kernel(pm_dir, address); 255 if (pte_none(*pt_dir)) { 256 unsigned long new_page; 257 258 new_page =__pa(vmem_alloc_pages(0)); 259 if (!new_page) 260 goto out; 261 pte_val(*pt_dir) = __pa(new_page); 262 } 263 address += PAGE_SIZE; 264 } 265 memset(start, 0, nr * sizeof(struct page)); 266 ret = 0; 267 out: 268 flush_tlb_kernel_range(start_addr, end_addr); 269 return ret; 270 } 271 272 void vmemmap_free(struct page *memmap, unsigned long nr_pages) 273 { 274 } 275 276 /* 277 * Add memory segment to the segment list if it doesn't overlap with 278 * an already present segment. 279 */ 280 static int insert_memory_segment(struct memory_segment *seg) 281 { 282 struct memory_segment *tmp; 283 284 if (seg->start + seg->size > VMEM_MAX_PHYS || 285 seg->start + seg->size < seg->start) 286 return -ERANGE; 287 288 list_for_each_entry(tmp, &mem_segs, list) { 289 if (seg->start >= tmp->start + tmp->size) 290 continue; 291 if (seg->start + seg->size <= tmp->start) 292 continue; 293 return -ENOSPC; 294 } 295 list_add(&seg->list, &mem_segs); 296 return 0; 297 } 298 299 /* 300 * Remove memory segment from the segment list. 301 */ 302 static void remove_memory_segment(struct memory_segment *seg) 303 { 304 list_del(&seg->list); 305 } 306 307 static void __remove_shared_memory(struct memory_segment *seg) 308 { 309 remove_memory_segment(seg); 310 vmem_remove_range(seg->start, seg->size); 311 } 312 313 int vmem_remove_mapping(unsigned long start, unsigned long size) 314 { 315 struct memory_segment *seg; 316 int ret; 317 318 mutex_lock(&vmem_mutex); 319 320 ret = -ENOENT; 321 list_for_each_entry(seg, &mem_segs, list) { 322 if (seg->start == start && seg->size == size) 323 break; 324 } 325 326 if (seg->start != start || seg->size != size) 327 goto out; 328 329 ret = 0; 330 __remove_shared_memory(seg); 331 kfree(seg); 332 out: 333 mutex_unlock(&vmem_mutex); 334 return ret; 335 } 336 337 int vmem_add_mapping(unsigned long start, unsigned long size) 338 { 339 struct memory_segment *seg; 340 int ret; 341 342 mutex_lock(&vmem_mutex); 343 ret = -ENOMEM; 344 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 345 if (!seg) 346 goto out; 347 seg->start = start; 348 seg->size = size; 349 350 ret = insert_memory_segment(seg); 351 if (ret) 352 goto out_free; 353 354 ret = vmem_add_mem(start, size, 0); 355 if (ret) 356 goto out_remove; 357 goto out; 358 359 out_remove: 360 __remove_shared_memory(seg); 361 out_free: 362 kfree(seg); 363 out: 364 mutex_unlock(&vmem_mutex); 365 return ret; 366 } 367 368 /* 369 * map whole physical memory to virtual memory (identity mapping) 370 * we reserve enough space in the vmalloc area for vmemmap to hotplug 371 * additional memory segments. 372 */ 373 void __init vmem_map_init(void) 374 { 375 unsigned long ro_start, ro_end; 376 unsigned long start, end; 377 int i; 378 379 ro_start = PFN_ALIGN((unsigned long)&_stext); 380 ro_end = (unsigned long)&_eshared & PAGE_MASK; 381 for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) { 382 if (memory_chunk[i].type == CHUNK_CRASHK || 383 memory_chunk[i].type == CHUNK_OLDMEM) 384 continue; 385 start = memory_chunk[i].addr; 386 end = memory_chunk[i].addr + memory_chunk[i].size; 387 if (start >= ro_end || end <= ro_start) 388 vmem_add_mem(start, end - start, 0); 389 else if (start >= ro_start && end <= ro_end) 390 vmem_add_mem(start, end - start, 1); 391 else if (start >= ro_start) { 392 vmem_add_mem(start, ro_end - start, 1); 393 vmem_add_mem(ro_end, end - ro_end, 0); 394 } else if (end < ro_end) { 395 vmem_add_mem(start, ro_start - start, 0); 396 vmem_add_mem(ro_start, end - ro_start, 1); 397 } else { 398 vmem_add_mem(start, ro_start - start, 0); 399 vmem_add_mem(ro_start, ro_end - ro_start, 1); 400 vmem_add_mem(ro_end, end - ro_end, 0); 401 } 402 } 403 } 404 405 /* 406 * Convert memory chunk array to a memory segment list so there is a single 407 * list that contains both r/w memory and shared memory segments. 408 */ 409 static int __init vmem_convert_memory_chunk(void) 410 { 411 struct memory_segment *seg; 412 int i; 413 414 mutex_lock(&vmem_mutex); 415 for (i = 0; i < MEMORY_CHUNKS; i++) { 416 if (!memory_chunk[i].size) 417 continue; 418 if (memory_chunk[i].type == CHUNK_CRASHK || 419 memory_chunk[i].type == CHUNK_OLDMEM) 420 continue; 421 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 422 if (!seg) 423 panic("Out of memory...\n"); 424 seg->start = memory_chunk[i].addr; 425 seg->size = memory_chunk[i].size; 426 insert_memory_segment(seg); 427 } 428 mutex_unlock(&vmem_mutex); 429 return 0; 430 } 431 432 core_initcall(vmem_convert_memory_chunk); 433