1 /* 2 * Copyright IBM Corp. 2006 3 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> 4 */ 5 6 #include <linux/bootmem.h> 7 #include <linux/pfn.h> 8 #include <linux/mm.h> 9 #include <linux/module.h> 10 #include <linux/list.h> 11 #include <linux/hugetlb.h> 12 #include <linux/slab.h> 13 #include <asm/pgalloc.h> 14 #include <asm/pgtable.h> 15 #include <asm/setup.h> 16 #include <asm/tlbflush.h> 17 #include <asm/sections.h> 18 19 static DEFINE_MUTEX(vmem_mutex); 20 21 struct memory_segment { 22 struct list_head list; 23 unsigned long start; 24 unsigned long size; 25 }; 26 27 static LIST_HEAD(mem_segs); 28 29 static void __ref *vmem_alloc_pages(unsigned int order) 30 { 31 if (slab_is_available()) 32 return (void *)__get_free_pages(GFP_KERNEL, order); 33 return alloc_bootmem_pages((1 << order) * PAGE_SIZE); 34 } 35 36 static inline pud_t *vmem_pud_alloc(void) 37 { 38 pud_t *pud = NULL; 39 40 #ifdef CONFIG_64BIT 41 pud = vmem_alloc_pages(2); 42 if (!pud) 43 return NULL; 44 clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); 45 #endif 46 return pud; 47 } 48 49 static inline pmd_t *vmem_pmd_alloc(void) 50 { 51 pmd_t *pmd = NULL; 52 53 #ifdef CONFIG_64BIT 54 pmd = vmem_alloc_pages(2); 55 if (!pmd) 56 return NULL; 57 clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); 58 #endif 59 return pmd; 60 } 61 62 static pte_t __ref *vmem_pte_alloc(unsigned long address) 63 { 64 pte_t *pte; 65 66 if (slab_is_available()) 67 pte = (pte_t *) page_table_alloc(&init_mm, address); 68 else 69 pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t)); 70 if (!pte) 71 return NULL; 72 clear_table((unsigned long *) pte, _PAGE_INVALID, 73 PTRS_PER_PTE * sizeof(pte_t)); 74 return pte; 75 } 76 77 /* 78 * Add a physical memory range to the 1:1 mapping. 79 */ 80 static int vmem_add_mem(unsigned long start, unsigned long size, int ro) 81 { 82 unsigned long end = start + size; 83 unsigned long address = start; 84 pgd_t *pg_dir; 85 pud_t *pu_dir; 86 pmd_t *pm_dir; 87 pte_t *pt_dir; 88 int ret = -ENOMEM; 89 90 while (address < end) { 91 pg_dir = pgd_offset_k(address); 92 if (pgd_none(*pg_dir)) { 93 pu_dir = vmem_pud_alloc(); 94 if (!pu_dir) 95 goto out; 96 pgd_populate(&init_mm, pg_dir, pu_dir); 97 } 98 pu_dir = pud_offset(pg_dir, address); 99 #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC) 100 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address && 101 !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) { 102 pud_val(*pu_dir) = __pa(address) | 103 _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE | 104 (ro ? _REGION_ENTRY_PROTECT : 0); 105 address += PUD_SIZE; 106 continue; 107 } 108 #endif 109 if (pud_none(*pu_dir)) { 110 pm_dir = vmem_pmd_alloc(); 111 if (!pm_dir) 112 goto out; 113 pud_populate(&init_mm, pu_dir, pm_dir); 114 } 115 pm_dir = pmd_offset(pu_dir, address); 116 #if defined(CONFIG_64BIT) && !defined(CONFIG_DEBUG_PAGEALLOC) 117 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address && 118 !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) { 119 pmd_val(*pm_dir) = __pa(address) | 120 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE | 121 _SEGMENT_ENTRY_YOUNG | 122 (ro ? _SEGMENT_ENTRY_PROTECT : 0); 123 address += PMD_SIZE; 124 continue; 125 } 126 #endif 127 if (pmd_none(*pm_dir)) { 128 pt_dir = vmem_pte_alloc(address); 129 if (!pt_dir) 130 goto out; 131 pmd_populate(&init_mm, pm_dir, pt_dir); 132 } 133 134 pt_dir = pte_offset_kernel(pm_dir, address); 135 pte_val(*pt_dir) = __pa(address) | 136 pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); 137 address += PAGE_SIZE; 138 } 139 ret = 0; 140 out: 141 flush_tlb_kernel_range(start, end); 142 return ret; 143 } 144 145 /* 146 * Remove a physical memory range from the 1:1 mapping. 147 * Currently only invalidates page table entries. 148 */ 149 static void vmem_remove_range(unsigned long start, unsigned long size) 150 { 151 unsigned long end = start + size; 152 unsigned long address = start; 153 pgd_t *pg_dir; 154 pud_t *pu_dir; 155 pmd_t *pm_dir; 156 pte_t *pt_dir; 157 pte_t pte; 158 159 pte_val(pte) = _PAGE_INVALID; 160 while (address < end) { 161 pg_dir = pgd_offset_k(address); 162 if (pgd_none(*pg_dir)) { 163 address += PGDIR_SIZE; 164 continue; 165 } 166 pu_dir = pud_offset(pg_dir, address); 167 if (pud_none(*pu_dir)) { 168 address += PUD_SIZE; 169 continue; 170 } 171 if (pud_large(*pu_dir)) { 172 pud_clear(pu_dir); 173 address += PUD_SIZE; 174 continue; 175 } 176 pm_dir = pmd_offset(pu_dir, address); 177 if (pmd_none(*pm_dir)) { 178 address += PMD_SIZE; 179 continue; 180 } 181 if (pmd_large(*pm_dir)) { 182 pmd_clear(pm_dir); 183 address += PMD_SIZE; 184 continue; 185 } 186 pt_dir = pte_offset_kernel(pm_dir, address); 187 *pt_dir = pte; 188 address += PAGE_SIZE; 189 } 190 flush_tlb_kernel_range(start, end); 191 } 192 193 /* 194 * Add a backed mem_map array to the virtual mem_map array. 195 */ 196 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 197 { 198 unsigned long address = start; 199 pgd_t *pg_dir; 200 pud_t *pu_dir; 201 pmd_t *pm_dir; 202 pte_t *pt_dir; 203 int ret = -ENOMEM; 204 205 for (address = start; address < end;) { 206 pg_dir = pgd_offset_k(address); 207 if (pgd_none(*pg_dir)) { 208 pu_dir = vmem_pud_alloc(); 209 if (!pu_dir) 210 goto out; 211 pgd_populate(&init_mm, pg_dir, pu_dir); 212 } 213 214 pu_dir = pud_offset(pg_dir, address); 215 if (pud_none(*pu_dir)) { 216 pm_dir = vmem_pmd_alloc(); 217 if (!pm_dir) 218 goto out; 219 pud_populate(&init_mm, pu_dir, pm_dir); 220 } 221 222 pm_dir = pmd_offset(pu_dir, address); 223 if (pmd_none(*pm_dir)) { 224 #ifdef CONFIG_64BIT 225 /* Use 1MB frames for vmemmap if available. We always 226 * use large frames even if they are only partially 227 * used. 228 * Otherwise we would have also page tables since 229 * vmemmap_populate gets called for each section 230 * separately. */ 231 if (MACHINE_HAS_EDAT1) { 232 void *new_page; 233 234 new_page = vmemmap_alloc_block(PMD_SIZE, node); 235 if (!new_page) 236 goto out; 237 pmd_val(*pm_dir) = __pa(new_page) | 238 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE | 239 _SEGMENT_ENTRY_CO; 240 address = (address + PMD_SIZE) & PMD_MASK; 241 continue; 242 } 243 #endif 244 pt_dir = vmem_pte_alloc(address); 245 if (!pt_dir) 246 goto out; 247 pmd_populate(&init_mm, pm_dir, pt_dir); 248 } else if (pmd_large(*pm_dir)) { 249 address = (address + PMD_SIZE) & PMD_MASK; 250 continue; 251 } 252 253 pt_dir = pte_offset_kernel(pm_dir, address); 254 if (pte_none(*pt_dir)) { 255 unsigned long new_page; 256 257 new_page =__pa(vmem_alloc_pages(0)); 258 if (!new_page) 259 goto out; 260 pte_val(*pt_dir) = 261 __pa(new_page) | pgprot_val(PAGE_KERNEL); 262 } 263 address += PAGE_SIZE; 264 } 265 memset((void *)start, 0, end - start); 266 ret = 0; 267 out: 268 flush_tlb_kernel_range(start, end); 269 return ret; 270 } 271 272 void vmemmap_free(unsigned long start, unsigned long end) 273 { 274 } 275 276 /* 277 * Add memory segment to the segment list if it doesn't overlap with 278 * an already present segment. 279 */ 280 static int insert_memory_segment(struct memory_segment *seg) 281 { 282 struct memory_segment *tmp; 283 284 if (seg->start + seg->size > VMEM_MAX_PHYS || 285 seg->start + seg->size < seg->start) 286 return -ERANGE; 287 288 list_for_each_entry(tmp, &mem_segs, list) { 289 if (seg->start >= tmp->start + tmp->size) 290 continue; 291 if (seg->start + seg->size <= tmp->start) 292 continue; 293 return -ENOSPC; 294 } 295 list_add(&seg->list, &mem_segs); 296 return 0; 297 } 298 299 /* 300 * Remove memory segment from the segment list. 301 */ 302 static void remove_memory_segment(struct memory_segment *seg) 303 { 304 list_del(&seg->list); 305 } 306 307 static void __remove_shared_memory(struct memory_segment *seg) 308 { 309 remove_memory_segment(seg); 310 vmem_remove_range(seg->start, seg->size); 311 } 312 313 int vmem_remove_mapping(unsigned long start, unsigned long size) 314 { 315 struct memory_segment *seg; 316 int ret; 317 318 mutex_lock(&vmem_mutex); 319 320 ret = -ENOENT; 321 list_for_each_entry(seg, &mem_segs, list) { 322 if (seg->start == start && seg->size == size) 323 break; 324 } 325 326 if (seg->start != start || seg->size != size) 327 goto out; 328 329 ret = 0; 330 __remove_shared_memory(seg); 331 kfree(seg); 332 out: 333 mutex_unlock(&vmem_mutex); 334 return ret; 335 } 336 337 int vmem_add_mapping(unsigned long start, unsigned long size) 338 { 339 struct memory_segment *seg; 340 int ret; 341 342 mutex_lock(&vmem_mutex); 343 ret = -ENOMEM; 344 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 345 if (!seg) 346 goto out; 347 seg->start = start; 348 seg->size = size; 349 350 ret = insert_memory_segment(seg); 351 if (ret) 352 goto out_free; 353 354 ret = vmem_add_mem(start, size, 0); 355 if (ret) 356 goto out_remove; 357 goto out; 358 359 out_remove: 360 __remove_shared_memory(seg); 361 out_free: 362 kfree(seg); 363 out: 364 mutex_unlock(&vmem_mutex); 365 return ret; 366 } 367 368 /* 369 * map whole physical memory to virtual memory (identity mapping) 370 * we reserve enough space in the vmalloc area for vmemmap to hotplug 371 * additional memory segments. 372 */ 373 void __init vmem_map_init(void) 374 { 375 unsigned long ro_start, ro_end; 376 unsigned long start, end; 377 int i; 378 379 ro_start = PFN_ALIGN((unsigned long)&_stext); 380 ro_end = (unsigned long)&_eshared & PAGE_MASK; 381 for (i = 0; i < MEMORY_CHUNKS; i++) { 382 if (!memory_chunk[i].size) 383 continue; 384 start = memory_chunk[i].addr; 385 end = memory_chunk[i].addr + memory_chunk[i].size; 386 if (start >= ro_end || end <= ro_start) 387 vmem_add_mem(start, end - start, 0); 388 else if (start >= ro_start && end <= ro_end) 389 vmem_add_mem(start, end - start, 1); 390 else if (start >= ro_start) { 391 vmem_add_mem(start, ro_end - start, 1); 392 vmem_add_mem(ro_end, end - ro_end, 0); 393 } else if (end < ro_end) { 394 vmem_add_mem(start, ro_start - start, 0); 395 vmem_add_mem(ro_start, end - ro_start, 1); 396 } else { 397 vmem_add_mem(start, ro_start - start, 0); 398 vmem_add_mem(ro_start, ro_end - ro_start, 1); 399 vmem_add_mem(ro_end, end - ro_end, 0); 400 } 401 } 402 } 403 404 /* 405 * Convert memory chunk array to a memory segment list so there is a single 406 * list that contains both r/w memory and shared memory segments. 407 */ 408 static int __init vmem_convert_memory_chunk(void) 409 { 410 struct memory_segment *seg; 411 int i; 412 413 mutex_lock(&vmem_mutex); 414 for (i = 0; i < MEMORY_CHUNKS; i++) { 415 if (!memory_chunk[i].size) 416 continue; 417 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 418 if (!seg) 419 panic("Out of memory...\n"); 420 seg->start = memory_chunk[i].addr; 421 seg->size = memory_chunk[i].size; 422 insert_memory_segment(seg); 423 } 424 mutex_unlock(&vmem_mutex); 425 return 0; 426 } 427 428 core_initcall(vmem_convert_memory_chunk); 429