1 /* 2 * Copyright IBM Corp. 2006 3 * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com> 4 */ 5 6 #include <linux/bootmem.h> 7 #include <linux/pfn.h> 8 #include <linux/mm.h> 9 #include <linux/module.h> 10 #include <linux/list.h> 11 #include <linux/hugetlb.h> 12 #include <linux/slab.h> 13 #include <linux/memblock.h> 14 #include <asm/cacheflush.h> 15 #include <asm/pgalloc.h> 16 #include <asm/pgtable.h> 17 #include <asm/setup.h> 18 #include <asm/tlbflush.h> 19 #include <asm/sections.h> 20 21 static DEFINE_MUTEX(vmem_mutex); 22 23 struct memory_segment { 24 struct list_head list; 25 unsigned long start; 26 unsigned long size; 27 }; 28 29 static LIST_HEAD(mem_segs); 30 31 static void __ref *vmem_alloc_pages(unsigned int order) 32 { 33 unsigned long size = PAGE_SIZE << order; 34 35 if (slab_is_available()) 36 return (void *)__get_free_pages(GFP_KERNEL, order); 37 return (void *) memblock_alloc(size, size); 38 } 39 40 static inline pud_t *vmem_pud_alloc(void) 41 { 42 pud_t *pud = NULL; 43 44 pud = vmem_alloc_pages(2); 45 if (!pud) 46 return NULL; 47 clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4); 48 return pud; 49 } 50 51 pmd_t *vmem_pmd_alloc(void) 52 { 53 pmd_t *pmd = NULL; 54 55 pmd = vmem_alloc_pages(2); 56 if (!pmd) 57 return NULL; 58 clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4); 59 return pmd; 60 } 61 62 pte_t __ref *vmem_pte_alloc(void) 63 { 64 unsigned long size = PTRS_PER_PTE * sizeof(pte_t); 65 pte_t *pte; 66 67 if (slab_is_available()) 68 pte = (pte_t *) page_table_alloc(&init_mm); 69 else 70 pte = (pte_t *) memblock_alloc(size, size); 71 if (!pte) 72 return NULL; 73 clear_table((unsigned long *) pte, _PAGE_INVALID, size); 74 return pte; 75 } 76 77 /* 78 * Add a physical memory range to the 1:1 mapping. 79 */ 80 static int vmem_add_mem(unsigned long start, unsigned long size) 81 { 82 unsigned long pages4k, pages1m, pages2g; 83 unsigned long end = start + size; 84 unsigned long address = start; 85 pgd_t *pg_dir; 86 pud_t *pu_dir; 87 pmd_t *pm_dir; 88 pte_t *pt_dir; 89 int ret = -ENOMEM; 90 91 pages4k = pages1m = pages2g = 0; 92 while (address < end) { 93 pg_dir = pgd_offset_k(address); 94 if (pgd_none(*pg_dir)) { 95 pu_dir = vmem_pud_alloc(); 96 if (!pu_dir) 97 goto out; 98 pgd_populate(&init_mm, pg_dir, pu_dir); 99 } 100 pu_dir = pud_offset(pg_dir, address); 101 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address && 102 !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) && 103 !debug_pagealloc_enabled()) { 104 pud_val(*pu_dir) = address | pgprot_val(REGION3_KERNEL); 105 address += PUD_SIZE; 106 pages2g++; 107 continue; 108 } 109 if (pud_none(*pu_dir)) { 110 pm_dir = vmem_pmd_alloc(); 111 if (!pm_dir) 112 goto out; 113 pud_populate(&init_mm, pu_dir, pm_dir); 114 } 115 pm_dir = pmd_offset(pu_dir, address); 116 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address && 117 !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) && 118 !debug_pagealloc_enabled()) { 119 pmd_val(*pm_dir) = address | pgprot_val(SEGMENT_KERNEL); 120 address += PMD_SIZE; 121 pages1m++; 122 continue; 123 } 124 if (pmd_none(*pm_dir)) { 125 pt_dir = vmem_pte_alloc(); 126 if (!pt_dir) 127 goto out; 128 pmd_populate(&init_mm, pm_dir, pt_dir); 129 } 130 131 pt_dir = pte_offset_kernel(pm_dir, address); 132 pte_val(*pt_dir) = address | pgprot_val(PAGE_KERNEL); 133 address += PAGE_SIZE; 134 pages4k++; 135 } 136 ret = 0; 137 out: 138 update_page_count(PG_DIRECT_MAP_4K, pages4k); 139 update_page_count(PG_DIRECT_MAP_1M, pages1m); 140 update_page_count(PG_DIRECT_MAP_2G, pages2g); 141 return ret; 142 } 143 144 /* 145 * Remove a physical memory range from the 1:1 mapping. 146 * Currently only invalidates page table entries. 147 */ 148 static void vmem_remove_range(unsigned long start, unsigned long size) 149 { 150 unsigned long pages4k, pages1m, pages2g; 151 unsigned long end = start + size; 152 unsigned long address = start; 153 pgd_t *pg_dir; 154 pud_t *pu_dir; 155 pmd_t *pm_dir; 156 pte_t *pt_dir; 157 158 pages4k = pages1m = pages2g = 0; 159 while (address < end) { 160 pg_dir = pgd_offset_k(address); 161 if (pgd_none(*pg_dir)) { 162 address += PGDIR_SIZE; 163 continue; 164 } 165 pu_dir = pud_offset(pg_dir, address); 166 if (pud_none(*pu_dir)) { 167 address += PUD_SIZE; 168 continue; 169 } 170 if (pud_large(*pu_dir)) { 171 pud_clear(pu_dir); 172 address += PUD_SIZE; 173 pages2g++; 174 continue; 175 } 176 pm_dir = pmd_offset(pu_dir, address); 177 if (pmd_none(*pm_dir)) { 178 address += PMD_SIZE; 179 continue; 180 } 181 if (pmd_large(*pm_dir)) { 182 pmd_clear(pm_dir); 183 address += PMD_SIZE; 184 pages1m++; 185 continue; 186 } 187 pt_dir = pte_offset_kernel(pm_dir, address); 188 pte_clear(&init_mm, address, pt_dir); 189 address += PAGE_SIZE; 190 pages4k++; 191 } 192 flush_tlb_kernel_range(start, end); 193 update_page_count(PG_DIRECT_MAP_4K, -pages4k); 194 update_page_count(PG_DIRECT_MAP_1M, -pages1m); 195 update_page_count(PG_DIRECT_MAP_2G, -pages2g); 196 } 197 198 /* 199 * Add a backed mem_map array to the virtual mem_map array. 200 */ 201 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 202 { 203 unsigned long address = start; 204 pgd_t *pg_dir; 205 pud_t *pu_dir; 206 pmd_t *pm_dir; 207 pte_t *pt_dir; 208 int ret = -ENOMEM; 209 210 for (address = start; address < end;) { 211 pg_dir = pgd_offset_k(address); 212 if (pgd_none(*pg_dir)) { 213 pu_dir = vmem_pud_alloc(); 214 if (!pu_dir) 215 goto out; 216 pgd_populate(&init_mm, pg_dir, pu_dir); 217 } 218 219 pu_dir = pud_offset(pg_dir, address); 220 if (pud_none(*pu_dir)) { 221 pm_dir = vmem_pmd_alloc(); 222 if (!pm_dir) 223 goto out; 224 pud_populate(&init_mm, pu_dir, pm_dir); 225 } 226 227 pm_dir = pmd_offset(pu_dir, address); 228 if (pmd_none(*pm_dir)) { 229 /* Use 1MB frames for vmemmap if available. We always 230 * use large frames even if they are only partially 231 * used. 232 * Otherwise we would have also page tables since 233 * vmemmap_populate gets called for each section 234 * separately. */ 235 if (MACHINE_HAS_EDAT1) { 236 void *new_page; 237 238 new_page = vmemmap_alloc_block(PMD_SIZE, node); 239 if (!new_page) 240 goto out; 241 pmd_val(*pm_dir) = __pa(new_page) | 242 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE; 243 address = (address + PMD_SIZE) & PMD_MASK; 244 continue; 245 } 246 pt_dir = vmem_pte_alloc(); 247 if (!pt_dir) 248 goto out; 249 pmd_populate(&init_mm, pm_dir, pt_dir); 250 } else if (pmd_large(*pm_dir)) { 251 address = (address + PMD_SIZE) & PMD_MASK; 252 continue; 253 } 254 255 pt_dir = pte_offset_kernel(pm_dir, address); 256 if (pte_none(*pt_dir)) { 257 void *new_page; 258 259 new_page = vmemmap_alloc_block(PAGE_SIZE, node); 260 if (!new_page) 261 goto out; 262 pte_val(*pt_dir) = 263 __pa(new_page) | pgprot_val(PAGE_KERNEL); 264 } 265 address += PAGE_SIZE; 266 } 267 ret = 0; 268 out: 269 return ret; 270 } 271 272 void vmemmap_free(unsigned long start, unsigned long end) 273 { 274 } 275 276 /* 277 * Add memory segment to the segment list if it doesn't overlap with 278 * an already present segment. 279 */ 280 static int insert_memory_segment(struct memory_segment *seg) 281 { 282 struct memory_segment *tmp; 283 284 if (seg->start + seg->size > VMEM_MAX_PHYS || 285 seg->start + seg->size < seg->start) 286 return -ERANGE; 287 288 list_for_each_entry(tmp, &mem_segs, list) { 289 if (seg->start >= tmp->start + tmp->size) 290 continue; 291 if (seg->start + seg->size <= tmp->start) 292 continue; 293 return -ENOSPC; 294 } 295 list_add(&seg->list, &mem_segs); 296 return 0; 297 } 298 299 /* 300 * Remove memory segment from the segment list. 301 */ 302 static void remove_memory_segment(struct memory_segment *seg) 303 { 304 list_del(&seg->list); 305 } 306 307 static void __remove_shared_memory(struct memory_segment *seg) 308 { 309 remove_memory_segment(seg); 310 vmem_remove_range(seg->start, seg->size); 311 } 312 313 int vmem_remove_mapping(unsigned long start, unsigned long size) 314 { 315 struct memory_segment *seg; 316 int ret; 317 318 mutex_lock(&vmem_mutex); 319 320 ret = -ENOENT; 321 list_for_each_entry(seg, &mem_segs, list) { 322 if (seg->start == start && seg->size == size) 323 break; 324 } 325 326 if (seg->start != start || seg->size != size) 327 goto out; 328 329 ret = 0; 330 __remove_shared_memory(seg); 331 kfree(seg); 332 out: 333 mutex_unlock(&vmem_mutex); 334 return ret; 335 } 336 337 int vmem_add_mapping(unsigned long start, unsigned long size) 338 { 339 struct memory_segment *seg; 340 int ret; 341 342 mutex_lock(&vmem_mutex); 343 ret = -ENOMEM; 344 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 345 if (!seg) 346 goto out; 347 seg->start = start; 348 seg->size = size; 349 350 ret = insert_memory_segment(seg); 351 if (ret) 352 goto out_free; 353 354 ret = vmem_add_mem(start, size); 355 if (ret) 356 goto out_remove; 357 goto out; 358 359 out_remove: 360 __remove_shared_memory(seg); 361 out_free: 362 kfree(seg); 363 out: 364 mutex_unlock(&vmem_mutex); 365 return ret; 366 } 367 368 /* 369 * map whole physical memory to virtual memory (identity mapping) 370 * we reserve enough space in the vmalloc area for vmemmap to hotplug 371 * additional memory segments. 372 */ 373 void __init vmem_map_init(void) 374 { 375 unsigned long size = _eshared - _stext; 376 struct memblock_region *reg; 377 378 for_each_memblock(memory, reg) 379 vmem_add_mem(reg->base, reg->size); 380 set_memory_ro((unsigned long)_stext, size >> PAGE_SHIFT); 381 pr_info("Write protected kernel read-only data: %luk\n", size >> 10); 382 } 383 384 /* 385 * Convert memblock.memory to a memory segment list so there is a single 386 * list that contains all memory segments. 387 */ 388 static int __init vmem_convert_memory_chunk(void) 389 { 390 struct memblock_region *reg; 391 struct memory_segment *seg; 392 393 mutex_lock(&vmem_mutex); 394 for_each_memblock(memory, reg) { 395 seg = kzalloc(sizeof(*seg), GFP_KERNEL); 396 if (!seg) 397 panic("Out of memory...\n"); 398 seg->start = reg->base; 399 seg->size = reg->size; 400 insert_memory_segment(seg); 401 } 402 mutex_unlock(&vmem_mutex); 403 return 0; 404 } 405 406 core_initcall(vmem_convert_memory_chunk); 407