1 /* 2 * Copyright IBM Corp. 2006 3 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> 4 */ 5 6 #include <linux/bootmem.h> 7 #include <linux/pfn.h> 8 #include <linux/mm.h> 9 #include <linux/module.h> 10 #include <linux/list.h> 11 #include <linux/hugetlb.h> 12 #include <linux/slab.h> 13 #include <linux/memblock.h> 14 #include <asm/cacheflush.h> 15 #include <asm/pgalloc.h> 16 #include <asm/pgtable.h> 17 #include <asm/setup.h> 18 #include <asm/tlbflush.h> 19 #include <asm/sections.h> 20 21 static DEFINE_MUTEX(vmem_mutex); 22 23 struct memory_segment { 24 struct list_head list; 25 unsigned long start; 26 unsigned long size; 27 }; 28 29 static LIST_HEAD(mem_segs); 30 31 static void __ref *vmem_alloc_pages(unsigned int order) 32 { 33 unsigned long size = PAGE_SIZE << order; 34 35 if (slab_is_available()) 36 return (void *)__get_free_pages(GFP_KERNEL, order); 37 return alloc_bootmem_align(size, size); 38 } 39 40 static inline pud_t *vmem_pud_alloc(void) 41 { 42 pud_t *pud = NULL; 43 44 pud = vmem_alloc_pages(2); 45 if (!pud) 46 return NULL; 47 clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); 48 return pud; 49 } 50 51 pmd_t *vmem_pmd_alloc(void) 52 { 53 pmd_t *pmd = NULL; 54 55 pmd = vmem_alloc_pages(2); 56 if (!pmd) 57 return NULL; 58 clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); 59 return pmd; 60 } 61 62 pte_t __ref *vmem_pte_alloc(void) 63 { 64 pte_t *pte; 65 66 if (slab_is_available()) 67 pte = (pte_t *) page_table_alloc(&init_mm); 68 else 69 pte = alloc_bootmem_align(PTRS_PER_PTE * sizeof(pte_t), 70 PTRS_PER_PTE * sizeof(pte_t)); 71 if (!pte) 72 return NULL; 73 clear_table((unsigned long *) pte, _PAGE_INVALID, 74 PTRS_PER_PTE * sizeof(pte_t)); 75 return pte; 76 } 77 78 /* 79 * Add a physical memory range to the 1:1 mapping. 80 */ 81 static int vmem_add_mem(unsigned long start, unsigned long size) 82 { 83 unsigned long pages4k, pages1m, pages2g; 84 unsigned long end = start + size; 85 unsigned long address = start; 86 pgd_t *pg_dir; 87 pud_t *pu_dir; 88 pmd_t *pm_dir; 89 pte_t *pt_dir; 90 int ret = -ENOMEM; 91 92 pages4k = pages1m = pages2g = 0; 93 while (address < end) { 94 pg_dir = pgd_offset_k(address); 95 if (pgd_none(*pg_dir)) { 96 pu_dir = vmem_pud_alloc(); 97 if (!pu_dir) 98 goto out; 99 pgd_populate(&init_mm, pg_dir, pu_dir); 100 } 101 pu_dir = pud_offset(pg_dir, address); 102 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address && 103 !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) && 104 !debug_pagealloc_enabled()) { 105 pud_val(*pu_dir) = address | pgprot_val(REGION3_KERNEL); 106 address += PUD_SIZE; 107 pages2g++; 108 continue; 109 } 110 if (pud_none(*pu_dir)) { 111 pm_dir = vmem_pmd_alloc(); 112 if (!pm_dir) 113 goto out; 114 pud_populate(&init_mm, pu_dir, pm_dir); 115 } 116 pm_dir = pmd_offset(pu_dir, address); 117 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address && 118 !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) && 119 !debug_pagealloc_enabled()) { 120 pmd_val(*pm_dir) = address | pgprot_val(SEGMENT_KERNEL); 121 address += PMD_SIZE; 122 pages1m++; 123 continue; 124 } 125 if (pmd_none(*pm_dir)) { 126 pt_dir = vmem_pte_alloc(); 127 if (!pt_dir) 128 goto out; 129 pmd_populate(&init_mm, pm_dir, pt_dir); 130 } 131 132 pt_dir = pte_offset_kernel(pm_dir, address); 133 pte_val(*pt_dir) = address | pgprot_val(PAGE_KERNEL); 134 address += PAGE_SIZE; 135 pages4k++; 136 } 137 ret = 0; 138 out: 139 update_page_count(PG_DIRECT_MAP_4K, pages4k); 140 update_page_count(PG_DIRECT_MAP_1M, pages1m); 141 update_page_count(PG_DIRECT_MAP_2G, pages2g); 142 return ret; 143 } 144 145 /* 146 * Remove a physical memory range from the 1:1 mapping. 147 * Currently only invalidates page table entries. 148 */ 149 static void vmem_remove_range(unsigned long start, unsigned long size) 150 { 151 unsigned long pages4k, pages1m, pages2g; 152 unsigned long end = start + size; 153 unsigned long address = start; 154 pgd_t *pg_dir; 155 pud_t *pu_dir; 156 pmd_t *pm_dir; 157 pte_t *pt_dir; 158 159 pages4k = pages1m = pages2g = 0; 160 while (address < end) { 161 pg_dir = pgd_offset_k(address); 162 if (pgd_none(*pg_dir)) { 163 address += PGDIR_SIZE; 164 continue; 165 } 166 pu_dir = pud_offset(pg_dir, address); 167 if (pud_none(*pu_dir)) { 168 address += PUD_SIZE; 169 continue; 170 } 171 if (pud_large(*pu_dir)) { 172 pud_clear(pu_dir); 173 address += PUD_SIZE; 174 pages2g++; 175 continue; 176 } 177 pm_dir = pmd_offset(pu_dir, address); 178 if (pmd_none(*pm_dir)) { 179 address += PMD_SIZE; 180 continue; 181 } 182 if (pmd_large(*pm_dir)) { 183 pmd_clear(pm_dir); 184 address += PMD_SIZE; 185 pages1m++; 186 continue; 187 } 188 pt_dir = pte_offset_kernel(pm_dir, address); 189 pte_clear(&init_mm, address, pt_dir); 190 address += PAGE_SIZE; 191 pages4k++; 192 } 193 flush_tlb_kernel_range(start, end); 194 update_page_count(PG_DIRECT_MAP_4K, -pages4k); 195 update_page_count(PG_DIRECT_MAP_1M, -pages1m); 196 update_page_count(PG_DIRECT_MAP_2G, -pages2g); 197 } 198 199 /* 200 * Add a backed mem_map array to the virtual mem_map array. 201 */ 202 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 203 { 204 unsigned long address = start; 205 pgd_t *pg_dir; 206 pud_t *pu_dir; 207 pmd_t *pm_dir; 208 pte_t *pt_dir; 209 int ret = -ENOMEM; 210 211 for (address = start; address < end;) { 212 pg_dir = pgd_offset_k(address); 213 if (pgd_none(*pg_dir)) { 214 pu_dir = vmem_pud_alloc(); 215 if (!pu_dir) 216 goto out; 217 pgd_populate(&init_mm, pg_dir, pu_dir); 218 } 219 220 pu_dir = pud_offset(pg_dir, address); 221 if (pud_none(*pu_dir)) { 222 pm_dir = vmem_pmd_alloc(); 223 if (!pm_dir) 224 goto out; 225 pud_populate(&init_mm, pu_dir, pm_dir); 226 } 227 228 pm_dir = pmd_offset(pu_dir, address); 229 if (pmd_none(*pm_dir)) { 230 /* Use 1MB frames for vmemmap if available. We always 231 * use large frames even if they are only partially 232 * used. 233 * Otherwise we would have also page tables since 234 * vmemmap_populate gets called for each section 235 * separately. */ 236 if (MACHINE_HAS_EDAT1) { 237 void *new_page; 238 239 new_page = vmemmap_alloc_block(PMD_SIZE, node); 240 if (!new_page) 241 goto out; 242 pmd_val(*pm_dir) = __pa(new_page) | 243 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE; 244 address = (address + PMD_SIZE) & PMD_MASK; 245 continue; 246 } 247 pt_dir = vmem_pte_alloc(); 248 if (!pt_dir) 249 goto out; 250 pmd_populate(&init_mm, pm_dir, pt_dir); 251 } else if (pmd_large(*pm_dir)) { 252 address = (address + PMD_SIZE) & PMD_MASK; 253 continue; 254 } 255 256 pt_dir = pte_offset_kernel(pm_dir, address); 257 if (pte_none(*pt_dir)) { 258 void *new_page; 259 260 new_page = vmemmap_alloc_block(PAGE_SIZE, node); 261 if (!new_page) 262 goto out; 263 pte_val(*pt_dir) = 264 __pa(new_page) | pgprot_val(PAGE_KERNEL); 265 } 266 address += PAGE_SIZE; 267 } 268 ret = 0; 269 out: 270 return ret; 271 } 272 273 void vmemmap_free(unsigned long start, unsigned long end) 274 { 275 } 276 277 /* 278 * Add memory segment to the segment list if it doesn't overlap with 279 * an already present segment. 280 */ 281 static int insert_memory_segment(struct memory_segment *seg) 282 { 283 struct memory_segment *tmp; 284 285 if (seg->start + seg->size > VMEM_MAX_PHYS || 286 seg->start + seg->size < seg->start) 287 return -ERANGE; 288 289 list_for_each_entry(tmp, &mem_segs, list) { 290 if (seg->start >= tmp->start + tmp->size) 291 continue; 292 if (seg->start + seg->size <= tmp->start) 293 continue; 294 return -ENOSPC; 295 } 296 list_add(&seg->list, &mem_segs); 297 return 0; 298 } 299 300 /* 301 * Remove memory segment from the segment list. 302 */ 303 static void remove_memory_segment(struct memory_segment *seg) 304 { 305 list_del(&seg->list); 306 } 307 308 static void __remove_shared_memory(struct memory_segment *seg) 309 { 310 remove_memory_segment(seg); 311 vmem_remove_range(seg->start, seg->size); 312 } 313 314 int vmem_remove_mapping(unsigned long start, unsigned long size) 315 { 316 struct memory_segment *seg; 317 int ret; 318 319 mutex_lock(&vmem_mutex); 320 321 ret = -ENOENT; 322 list_for_each_entry(seg, &mem_segs, list) { 323 if (seg->start == start && seg->size == size) 324 break; 325 } 326 327 if (seg->start != start || seg->size != size) 328 goto out; 329 330 ret = 0; 331 __remove_shared_memory(seg); 332 kfree(seg); 333 out: 334 mutex_unlock(&vmem_mutex); 335 return ret; 336 } 337 338 int vmem_add_mapping(unsigned long start, unsigned long size) 339 { 340 struct memory_segment *seg; 341 int ret; 342 343 mutex_lock(&vmem_mutex); 344 ret = -ENOMEM; 345 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 346 if (!seg) 347 goto out; 348 seg->start = start; 349 seg->size = size; 350 351 ret = insert_memory_segment(seg); 352 if (ret) 353 goto out_free; 354 355 ret = vmem_add_mem(start, size); 356 if (ret) 357 goto out_remove; 358 goto out; 359 360 out_remove: 361 __remove_shared_memory(seg); 362 out_free: 363 kfree(seg); 364 out: 365 mutex_unlock(&vmem_mutex); 366 return ret; 367 } 368 369 /* 370 * map whole physical memory to virtual memory (identity mapping) 371 * we reserve enough space in the vmalloc area for vmemmap to hotplug 372 * additional memory segments. 373 */ 374 void __init vmem_map_init(void) 375 { 376 unsigned long size = _eshared - _stext; 377 struct memblock_region *reg; 378 379 for_each_memblock(memory, reg) 380 vmem_add_mem(reg->base, reg->size); 381 set_memory_ro((unsigned long)_stext, size >> PAGE_SHIFT); 382 pr_info("Write protected kernel read-only data: %luk\n", size >> 10); 383 } 384 385 /* 386 * Convert memblock.memory to a memory segment list so there is a single 387 * list that contains all memory segments. 388 */ 389 static int __init vmem_convert_memory_chunk(void) 390 { 391 struct memblock_region *reg; 392 struct memory_segment *seg; 393 394 mutex_lock(&vmem_mutex); 395 for_each_memblock(memory, reg) { 396 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 397 if (!seg) 398 panic("Out of memory...\n"); 399 seg->start = reg->base; 400 seg->size = reg->size; 401 insert_memory_segment(seg); 402 } 403 mutex_unlock(&vmem_mutex); 404 return 0; 405 } 406 407 core_initcall(vmem_convert_memory_chunk); 408