1 /* 2 * sparse memory mappings. 3 */ 4 #include <linux/mm.h> 5 #include <linux/mmzone.h> 6 #include <linux/bootmem.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/spinlock.h> 10 #include <linux/vmalloc.h> 11 #include "internal.h" 12 #include <asm/dma.h> 13 #include <asm/pgalloc.h> 14 #include <asm/pgtable.h> 15 16 /* 17 * Permanent SPARSEMEM data: 18 * 19 * 1) mem_section - memory sections, mem_map's for valid memory 20 */ 21 #ifdef CONFIG_SPARSEMEM_EXTREME 22 struct mem_section *mem_section[NR_SECTION_ROOTS] 23 ____cacheline_internodealigned_in_smp; 24 #else 25 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 26 ____cacheline_internodealigned_in_smp; 27 #endif 28 EXPORT_SYMBOL(mem_section); 29 30 #ifdef NODE_NOT_IN_PAGE_FLAGS 31 /* 32 * If we did not store the node number in the page then we have to 33 * do a lookup in the section_to_node_table in order to find which 34 * node the page belongs to. 35 */ 36 #if MAX_NUMNODES <= 256 37 static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 38 #else 39 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 40 #endif 41 42 int page_to_nid(struct page *page) 43 { 44 return section_to_node_table[page_to_section(page)]; 45 } 46 EXPORT_SYMBOL(page_to_nid); 47 48 static void set_section_nid(unsigned long section_nr, int nid) 49 { 50 section_to_node_table[section_nr] = nid; 51 } 52 #else /* !NODE_NOT_IN_PAGE_FLAGS */ 53 static inline void set_section_nid(unsigned long section_nr, int nid) 54 { 55 } 56 #endif 57 58 #ifdef CONFIG_SPARSEMEM_EXTREME 59 static struct mem_section noinline __init_refok *sparse_index_alloc(int nid) 60 { 61 struct mem_section *section = NULL; 62 unsigned long array_size = SECTIONS_PER_ROOT * 63 sizeof(struct mem_section); 64 65 if (slab_is_available()) 66 section = kmalloc_node(array_size, GFP_KERNEL, nid); 67 else 68 section = alloc_bootmem_node(NODE_DATA(nid), array_size); 69 70 if (section) 71 memset(section, 0, array_size); 72 73 return section; 74 } 75 76 static int __meminit sparse_index_init(unsigned long section_nr, int nid) 77 { 78 static DEFINE_SPINLOCK(index_init_lock); 79 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 80 struct mem_section *section; 81 int ret = 0; 82 83 if (mem_section[root]) 84 return -EEXIST; 85 86 section = sparse_index_alloc(nid); 87 if (!section) 88 return -ENOMEM; 89 /* 90 * This lock keeps two different sections from 91 * reallocating for the same index 92 */ 93 spin_lock(&index_init_lock); 94 95 if (mem_section[root]) { 96 ret = -EEXIST; 97 goto out; 98 } 99 100 mem_section[root] = section; 101 out: 102 spin_unlock(&index_init_lock); 103 return ret; 104 } 105 #else /* !SPARSEMEM_EXTREME */ 106 static inline int sparse_index_init(unsigned long section_nr, int nid) 107 { 108 return 0; 109 } 110 #endif 111 112 /* 113 * Although written for the SPARSEMEM_EXTREME case, this happens 114 * to also work for the flat array case because 115 * NR_SECTION_ROOTS==NR_MEM_SECTIONS. 116 */ 117 int __section_nr(struct mem_section* ms) 118 { 119 unsigned long root_nr; 120 struct mem_section* root; 121 122 for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { 123 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); 124 if (!root) 125 continue; 126 127 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) 128 break; 129 } 130 131 return (root_nr * SECTIONS_PER_ROOT) + (ms - root); 132 } 133 134 /* 135 * During early boot, before section_mem_map is used for an actual 136 * mem_map, we use section_mem_map to store the section's NUMA 137 * node. This keeps us from having to use another data structure. The 138 * node information is cleared just before we store the real mem_map. 139 */ 140 static inline unsigned long sparse_encode_early_nid(int nid) 141 { 142 return (nid << SECTION_NID_SHIFT); 143 } 144 145 static inline int sparse_early_nid(struct mem_section *section) 146 { 147 return (section->section_mem_map >> SECTION_NID_SHIFT); 148 } 149 150 /* Validate the physical addressing limitations of the model */ 151 void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn, 152 unsigned long *end_pfn) 153 { 154 unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 155 156 /* 157 * Sanity checks - do not allow an architecture to pass 158 * in larger pfns than the maximum scope of sparsemem: 159 */ 160 if (*start_pfn > max_sparsemem_pfn) { 161 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 162 "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 163 *start_pfn, *end_pfn, max_sparsemem_pfn); 164 WARN_ON_ONCE(1); 165 *start_pfn = max_sparsemem_pfn; 166 *end_pfn = max_sparsemem_pfn; 167 } 168 169 if (*end_pfn > max_sparsemem_pfn) { 170 mminit_dprintk(MMINIT_WARNING, "pfnvalidation", 171 "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n", 172 *start_pfn, *end_pfn, max_sparsemem_pfn); 173 WARN_ON_ONCE(1); 174 *end_pfn = max_sparsemem_pfn; 175 } 176 } 177 178 /* Record a memory area against a node. */ 179 void __init memory_present(int nid, unsigned long start, unsigned long end) 180 { 181 unsigned long pfn; 182 183 start &= PAGE_SECTION_MASK; 184 mminit_validate_memmodel_limits(&start, &end); 185 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 186 unsigned long section = pfn_to_section_nr(pfn); 187 struct mem_section *ms; 188 189 sparse_index_init(section, nid); 190 set_section_nid(section, nid); 191 192 ms = __nr_to_section(section); 193 if (!ms->section_mem_map) 194 ms->section_mem_map = sparse_encode_early_nid(nid) | 195 SECTION_MARKED_PRESENT; 196 } 197 } 198 199 /* 200 * Only used by the i386 NUMA architecures, but relatively 201 * generic code. 202 */ 203 unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, 204 unsigned long end_pfn) 205 { 206 unsigned long pfn; 207 unsigned long nr_pages = 0; 208 209 mminit_validate_memmodel_limits(&start_pfn, &end_pfn); 210 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 211 if (nid != early_pfn_to_nid(pfn)) 212 continue; 213 214 if (pfn_present(pfn)) 215 nr_pages += PAGES_PER_SECTION; 216 } 217 218 return nr_pages * sizeof(struct page); 219 } 220 221 /* 222 * Subtle, we encode the real pfn into the mem_map such that 223 * the identity pfn - section_mem_map will return the actual 224 * physical page frame number. 225 */ 226 static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 227 { 228 return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 229 } 230 231 /* 232 * Decode mem_map from the coded memmap 233 */ 234 struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 235 { 236 /* mask off the extra low bits of information */ 237 coded_mem_map &= SECTION_MAP_MASK; 238 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 239 } 240 241 static int __meminit sparse_init_one_section(struct mem_section *ms, 242 unsigned long pnum, struct page *mem_map, 243 unsigned long *pageblock_bitmap) 244 { 245 if (!present_section(ms)) 246 return -EINVAL; 247 248 ms->section_mem_map &= ~SECTION_MAP_MASK; 249 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | 250 SECTION_HAS_MEM_MAP; 251 ms->pageblock_flags = pageblock_bitmap; 252 253 return 1; 254 } 255 256 unsigned long usemap_size(void) 257 { 258 unsigned long size_bytes; 259 size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8; 260 size_bytes = roundup(size_bytes, sizeof(unsigned long)); 261 return size_bytes; 262 } 263 264 #ifdef CONFIG_MEMORY_HOTPLUG 265 static unsigned long *__kmalloc_section_usemap(void) 266 { 267 return kmalloc(usemap_size(), GFP_KERNEL); 268 } 269 #endif /* CONFIG_MEMORY_HOTPLUG */ 270 271 #ifdef CONFIG_MEMORY_HOTREMOVE 272 static unsigned long * __init 273 sparse_early_usemap_alloc_pgdat_section(struct pglist_data *pgdat) 274 { 275 unsigned long section_nr; 276 277 /* 278 * A page may contain usemaps for other sections preventing the 279 * page being freed and making a section unremovable while 280 * other sections referencing the usemap retmain active. Similarly, 281 * a pgdat can prevent a section being removed. If section A 282 * contains a pgdat and section B contains the usemap, both 283 * sections become inter-dependent. This allocates usemaps 284 * from the same section as the pgdat where possible to avoid 285 * this problem. 286 */ 287 section_nr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT); 288 return alloc_bootmem_section(usemap_size(), section_nr); 289 } 290 291 static void __init check_usemap_section_nr(int nid, unsigned long *usemap) 292 { 293 unsigned long usemap_snr, pgdat_snr; 294 static unsigned long old_usemap_snr = NR_MEM_SECTIONS; 295 static unsigned long old_pgdat_snr = NR_MEM_SECTIONS; 296 struct pglist_data *pgdat = NODE_DATA(nid); 297 int usemap_nid; 298 299 usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT); 300 pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT); 301 if (usemap_snr == pgdat_snr) 302 return; 303 304 if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr) 305 /* skip redundant message */ 306 return; 307 308 old_usemap_snr = usemap_snr; 309 old_pgdat_snr = pgdat_snr; 310 311 usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr)); 312 if (usemap_nid != nid) { 313 printk(KERN_INFO 314 "node %d must be removed before remove section %ld\n", 315 nid, usemap_snr); 316 return; 317 } 318 /* 319 * There is a circular dependency. 320 * Some platforms allow un-removable section because they will just 321 * gather other removable sections for dynamic partitioning. 322 * Just notify un-removable section's number here. 323 */ 324 printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr, 325 pgdat_snr, nid); 326 printk(KERN_CONT 327 " have a circular dependency on usemap and pgdat allocations\n"); 328 } 329 #else 330 static unsigned long * __init 331 sparse_early_usemap_alloc_pgdat_section(struct pglist_data *pgdat) 332 { 333 return NULL; 334 } 335 336 static void __init check_usemap_section_nr(int nid, unsigned long *usemap) 337 { 338 } 339 #endif /* CONFIG_MEMORY_HOTREMOVE */ 340 341 static unsigned long *__init sparse_early_usemap_alloc(unsigned long pnum) 342 { 343 unsigned long *usemap; 344 struct mem_section *ms = __nr_to_section(pnum); 345 int nid = sparse_early_nid(ms); 346 347 usemap = sparse_early_usemap_alloc_pgdat_section(NODE_DATA(nid)); 348 if (usemap) 349 return usemap; 350 351 usemap = alloc_bootmem_node(NODE_DATA(nid), usemap_size()); 352 if (usemap) { 353 check_usemap_section_nr(nid, usemap); 354 return usemap; 355 } 356 357 /* Stupid: suppress gcc warning for SPARSEMEM && !NUMA */ 358 nid = 0; 359 360 printk(KERN_WARNING "%s: allocation failed\n", __func__); 361 return NULL; 362 } 363 364 #ifndef CONFIG_SPARSEMEM_VMEMMAP 365 struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid) 366 { 367 struct page *map; 368 369 map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); 370 if (map) 371 return map; 372 373 map = alloc_bootmem_pages_node(NODE_DATA(nid), 374 PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION)); 375 return map; 376 } 377 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 378 379 static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum) 380 { 381 struct page *map; 382 struct mem_section *ms = __nr_to_section(pnum); 383 int nid = sparse_early_nid(ms); 384 385 map = sparse_mem_map_populate(pnum, nid); 386 if (map) 387 return map; 388 389 printk(KERN_ERR "%s: sparsemem memory map backing failed " 390 "some memory will not be available.\n", __func__); 391 ms->section_mem_map = 0; 392 return NULL; 393 } 394 395 void __attribute__((weak)) __meminit vmemmap_populate_print_last(void) 396 { 397 } 398 /* 399 * Allocate the accumulated non-linear sections, allocate a mem_map 400 * for each and record the physical to section mapping. 401 */ 402 void __init sparse_init(void) 403 { 404 unsigned long pnum; 405 struct page *map; 406 unsigned long *usemap; 407 unsigned long **usemap_map; 408 int size; 409 410 /* 411 * map is using big page (aka 2M in x86 64 bit) 412 * usemap is less one page (aka 24 bytes) 413 * so alloc 2M (with 2M align) and 24 bytes in turn will 414 * make next 2M slip to one more 2M later. 415 * then in big system, the memory will have a lot of holes... 416 * here try to allocate 2M pages continously. 417 * 418 * powerpc need to call sparse_init_one_section right after each 419 * sparse_early_mem_map_alloc, so allocate usemap_map at first. 420 */ 421 size = sizeof(unsigned long *) * NR_MEM_SECTIONS; 422 usemap_map = alloc_bootmem(size); 423 if (!usemap_map) 424 panic("can not allocate usemap_map\n"); 425 426 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 427 if (!present_section_nr(pnum)) 428 continue; 429 usemap_map[pnum] = sparse_early_usemap_alloc(pnum); 430 } 431 432 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 433 if (!present_section_nr(pnum)) 434 continue; 435 436 usemap = usemap_map[pnum]; 437 if (!usemap) 438 continue; 439 440 map = sparse_early_mem_map_alloc(pnum); 441 if (!map) 442 continue; 443 444 sparse_init_one_section(__nr_to_section(pnum), pnum, map, 445 usemap); 446 } 447 448 vmemmap_populate_print_last(); 449 450 free_bootmem(__pa(usemap_map), size); 451 } 452 453 #ifdef CONFIG_MEMORY_HOTPLUG 454 #ifdef CONFIG_SPARSEMEM_VMEMMAP 455 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 456 unsigned long nr_pages) 457 { 458 /* This will make the necessary allocations eventually. */ 459 return sparse_mem_map_populate(pnum, nid); 460 } 461 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 462 { 463 return; /* XXX: Not implemented yet */ 464 } 465 static void free_map_bootmem(struct page *page, unsigned long nr_pages) 466 { 467 } 468 #else 469 static struct page *__kmalloc_section_memmap(unsigned long nr_pages) 470 { 471 struct page *page, *ret; 472 unsigned long memmap_size = sizeof(struct page) * nr_pages; 473 474 page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); 475 if (page) 476 goto got_map_page; 477 478 ret = vmalloc(memmap_size); 479 if (ret) 480 goto got_map_ptr; 481 482 return NULL; 483 got_map_page: 484 ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); 485 got_map_ptr: 486 memset(ret, 0, memmap_size); 487 488 return ret; 489 } 490 491 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 492 unsigned long nr_pages) 493 { 494 return __kmalloc_section_memmap(nr_pages); 495 } 496 497 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 498 { 499 if (is_vmalloc_addr(memmap)) 500 vfree(memmap); 501 else 502 free_pages((unsigned long)memmap, 503 get_order(sizeof(struct page) * nr_pages)); 504 } 505 506 static void free_map_bootmem(struct page *page, unsigned long nr_pages) 507 { 508 unsigned long maps_section_nr, removing_section_nr, i; 509 int magic; 510 511 for (i = 0; i < nr_pages; i++, page++) { 512 magic = atomic_read(&page->_mapcount); 513 514 BUG_ON(magic == NODE_INFO); 515 516 maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 517 removing_section_nr = page->private; 518 519 /* 520 * When this function is called, the removing section is 521 * logical offlined state. This means all pages are isolated 522 * from page allocator. If removing section's memmap is placed 523 * on the same section, it must not be freed. 524 * If it is freed, page allocator may allocate it which will 525 * be removed physically soon. 526 */ 527 if (maps_section_nr != removing_section_nr) 528 put_page_bootmem(page); 529 } 530 } 531 #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 532 533 static void free_section_usemap(struct page *memmap, unsigned long *usemap) 534 { 535 struct page *usemap_page; 536 unsigned long nr_pages; 537 538 if (!usemap) 539 return; 540 541 usemap_page = virt_to_page(usemap); 542 /* 543 * Check to see if allocation came from hot-plug-add 544 */ 545 if (PageSlab(usemap_page)) { 546 kfree(usemap); 547 if (memmap) 548 __kfree_section_memmap(memmap, PAGES_PER_SECTION); 549 return; 550 } 551 552 /* 553 * The usemap came from bootmem. This is packed with other usemaps 554 * on the section which has pgdat at boot time. Just keep it as is now. 555 */ 556 557 if (memmap) { 558 struct page *memmap_page; 559 memmap_page = virt_to_page(memmap); 560 561 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 562 >> PAGE_SHIFT; 563 564 free_map_bootmem(memmap_page, nr_pages); 565 } 566 } 567 568 /* 569 * returns the number of sections whose mem_maps were properly 570 * set. If this is <=0, then that means that the passed-in 571 * map was not consumed and must be freed. 572 */ 573 int sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 574 int nr_pages) 575 { 576 unsigned long section_nr = pfn_to_section_nr(start_pfn); 577 struct pglist_data *pgdat = zone->zone_pgdat; 578 struct mem_section *ms; 579 struct page *memmap; 580 unsigned long *usemap; 581 unsigned long flags; 582 int ret; 583 584 /* 585 * no locking for this, because it does its own 586 * plus, it does a kmalloc 587 */ 588 ret = sparse_index_init(section_nr, pgdat->node_id); 589 if (ret < 0 && ret != -EEXIST) 590 return ret; 591 memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages); 592 if (!memmap) 593 return -ENOMEM; 594 usemap = __kmalloc_section_usemap(); 595 if (!usemap) { 596 __kfree_section_memmap(memmap, nr_pages); 597 return -ENOMEM; 598 } 599 600 pgdat_resize_lock(pgdat, &flags); 601 602 ms = __pfn_to_section(start_pfn); 603 if (ms->section_mem_map & SECTION_MARKED_PRESENT) { 604 ret = -EEXIST; 605 goto out; 606 } 607 608 ms->section_mem_map |= SECTION_MARKED_PRESENT; 609 610 ret = sparse_init_one_section(ms, section_nr, memmap, usemap); 611 612 out: 613 pgdat_resize_unlock(pgdat, &flags); 614 if (ret <= 0) { 615 kfree(usemap); 616 __kfree_section_memmap(memmap, nr_pages); 617 } 618 return ret; 619 } 620 621 void sparse_remove_one_section(struct zone *zone, struct mem_section *ms) 622 { 623 struct page *memmap = NULL; 624 unsigned long *usemap = NULL; 625 626 if (ms->section_mem_map) { 627 usemap = ms->pageblock_flags; 628 memmap = sparse_decode_mem_map(ms->section_mem_map, 629 __section_nr(ms)); 630 ms->section_mem_map = 0; 631 ms->pageblock_flags = NULL; 632 } 633 634 free_section_usemap(memmap, usemap); 635 } 636 #endif 637