1 /* 2 * sparse memory mappings. 3 */ 4 #include <linux/mm.h> 5 #include <linux/mmzone.h> 6 #include <linux/bootmem.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/spinlock.h> 10 #include <linux/vmalloc.h> 11 #include "internal.h" 12 #include <asm/dma.h> 13 #include <asm/pgalloc.h> 14 #include <asm/pgtable.h> 15 16 /* 17 * Permanent SPARSEMEM data: 18 * 19 * 1) mem_section - memory sections, mem_map's for valid memory 20 */ 21 #ifdef CONFIG_SPARSEMEM_EXTREME 22 struct mem_section *mem_section[NR_SECTION_ROOTS] 23 ____cacheline_internodealigned_in_smp; 24 #else 25 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT] 26 ____cacheline_internodealigned_in_smp; 27 #endif 28 EXPORT_SYMBOL(mem_section); 29 30 #ifdef NODE_NOT_IN_PAGE_FLAGS 31 /* 32 * If we did not store the node number in the page then we have to 33 * do a lookup in the section_to_node_table in order to find which 34 * node the page belongs to. 35 */ 36 #if MAX_NUMNODES <= 256 37 static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 38 #else 39 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned; 40 #endif 41 42 int page_to_nid(struct page *page) 43 { 44 return section_to_node_table[page_to_section(page)]; 45 } 46 EXPORT_SYMBOL(page_to_nid); 47 48 static void set_section_nid(unsigned long section_nr, int nid) 49 { 50 section_to_node_table[section_nr] = nid; 51 } 52 #else /* !NODE_NOT_IN_PAGE_FLAGS */ 53 static inline void set_section_nid(unsigned long section_nr, int nid) 54 { 55 } 56 #endif 57 58 #ifdef CONFIG_SPARSEMEM_EXTREME 59 static struct mem_section noinline __init_refok *sparse_index_alloc(int nid) 60 { 61 struct mem_section *section = NULL; 62 unsigned long array_size = SECTIONS_PER_ROOT * 63 sizeof(struct mem_section); 64 65 if (slab_is_available()) 66 section = kmalloc_node(array_size, GFP_KERNEL, nid); 67 else 68 section = alloc_bootmem_node(NODE_DATA(nid), array_size); 69 70 if (section) 71 memset(section, 0, array_size); 72 73 return section; 74 } 75 76 static int __meminit sparse_index_init(unsigned long section_nr, int nid) 77 { 78 static DEFINE_SPINLOCK(index_init_lock); 79 unsigned long root = SECTION_NR_TO_ROOT(section_nr); 80 struct mem_section *section; 81 int ret = 0; 82 83 if (mem_section[root]) 84 return -EEXIST; 85 86 section = sparse_index_alloc(nid); 87 if (!section) 88 return -ENOMEM; 89 /* 90 * This lock keeps two different sections from 91 * reallocating for the same index 92 */ 93 spin_lock(&index_init_lock); 94 95 if (mem_section[root]) { 96 ret = -EEXIST; 97 goto out; 98 } 99 100 mem_section[root] = section; 101 out: 102 spin_unlock(&index_init_lock); 103 return ret; 104 } 105 #else /* !SPARSEMEM_EXTREME */ 106 static inline int sparse_index_init(unsigned long section_nr, int nid) 107 { 108 return 0; 109 } 110 #endif 111 112 /* 113 * Although written for the SPARSEMEM_EXTREME case, this happens 114 * to also work for the flat array case because 115 * NR_SECTION_ROOTS==NR_MEM_SECTIONS. 116 */ 117 int __section_nr(struct mem_section* ms) 118 { 119 unsigned long root_nr; 120 struct mem_section* root; 121 122 for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) { 123 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT); 124 if (!root) 125 continue; 126 127 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT))) 128 break; 129 } 130 131 return (root_nr * SECTIONS_PER_ROOT) + (ms - root); 132 } 133 134 /* 135 * During early boot, before section_mem_map is used for an actual 136 * mem_map, we use section_mem_map to store the section's NUMA 137 * node. This keeps us from having to use another data structure. The 138 * node information is cleared just before we store the real mem_map. 139 */ 140 static inline unsigned long sparse_encode_early_nid(int nid) 141 { 142 return (nid << SECTION_NID_SHIFT); 143 } 144 145 static inline int sparse_early_nid(struct mem_section *section) 146 { 147 return (section->section_mem_map >> SECTION_NID_SHIFT); 148 } 149 150 /* Record a memory area against a node. */ 151 void __init memory_present(int nid, unsigned long start, unsigned long end) 152 { 153 unsigned long max_arch_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT); 154 unsigned long pfn; 155 156 /* 157 * Sanity checks - do not allow an architecture to pass 158 * in larger pfns than the maximum scope of sparsemem: 159 */ 160 if (start >= max_arch_pfn) 161 return; 162 if (end >= max_arch_pfn) 163 end = max_arch_pfn; 164 165 start &= PAGE_SECTION_MASK; 166 for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) { 167 unsigned long section = pfn_to_section_nr(pfn); 168 struct mem_section *ms; 169 170 sparse_index_init(section, nid); 171 set_section_nid(section, nid); 172 173 ms = __nr_to_section(section); 174 if (!ms->section_mem_map) 175 ms->section_mem_map = sparse_encode_early_nid(nid) | 176 SECTION_MARKED_PRESENT; 177 } 178 } 179 180 /* 181 * Only used by the i386 NUMA architecures, but relatively 182 * generic code. 183 */ 184 unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn, 185 unsigned long end_pfn) 186 { 187 unsigned long pfn; 188 unsigned long nr_pages = 0; 189 190 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 191 if (nid != early_pfn_to_nid(pfn)) 192 continue; 193 194 if (pfn_present(pfn)) 195 nr_pages += PAGES_PER_SECTION; 196 } 197 198 return nr_pages * sizeof(struct page); 199 } 200 201 /* 202 * Subtle, we encode the real pfn into the mem_map such that 203 * the identity pfn - section_mem_map will return the actual 204 * physical page frame number. 205 */ 206 static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum) 207 { 208 return (unsigned long)(mem_map - (section_nr_to_pfn(pnum))); 209 } 210 211 /* 212 * Decode mem_map from the coded memmap 213 */ 214 struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum) 215 { 216 /* mask off the extra low bits of information */ 217 coded_mem_map &= SECTION_MAP_MASK; 218 return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum); 219 } 220 221 static int __meminit sparse_init_one_section(struct mem_section *ms, 222 unsigned long pnum, struct page *mem_map, 223 unsigned long *pageblock_bitmap) 224 { 225 if (!present_section(ms)) 226 return -EINVAL; 227 228 ms->section_mem_map &= ~SECTION_MAP_MASK; 229 ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) | 230 SECTION_HAS_MEM_MAP; 231 ms->pageblock_flags = pageblock_bitmap; 232 233 return 1; 234 } 235 236 unsigned long usemap_size(void) 237 { 238 unsigned long size_bytes; 239 size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8; 240 size_bytes = roundup(size_bytes, sizeof(unsigned long)); 241 return size_bytes; 242 } 243 244 #ifdef CONFIG_MEMORY_HOTPLUG 245 static unsigned long *__kmalloc_section_usemap(void) 246 { 247 return kmalloc(usemap_size(), GFP_KERNEL); 248 } 249 #endif /* CONFIG_MEMORY_HOTPLUG */ 250 251 static unsigned long *__init sparse_early_usemap_alloc(unsigned long pnum) 252 { 253 unsigned long *usemap; 254 struct mem_section *ms = __nr_to_section(pnum); 255 int nid = sparse_early_nid(ms); 256 257 usemap = alloc_bootmem_node(NODE_DATA(nid), usemap_size()); 258 if (usemap) 259 return usemap; 260 261 /* Stupid: suppress gcc warning for SPARSEMEM && !NUMA */ 262 nid = 0; 263 264 printk(KERN_WARNING "%s: allocation failed\n", __func__); 265 return NULL; 266 } 267 268 #ifndef CONFIG_SPARSEMEM_VMEMMAP 269 struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid) 270 { 271 struct page *map; 272 273 map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION); 274 if (map) 275 return map; 276 277 map = alloc_bootmem_pages_node(NODE_DATA(nid), 278 PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION)); 279 return map; 280 } 281 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 282 283 struct page __init *sparse_early_mem_map_alloc(unsigned long pnum) 284 { 285 struct page *map; 286 struct mem_section *ms = __nr_to_section(pnum); 287 int nid = sparse_early_nid(ms); 288 289 map = sparse_mem_map_populate(pnum, nid); 290 if (map) 291 return map; 292 293 printk(KERN_ERR "%s: sparsemem memory map backing failed " 294 "some memory will not be available.\n", __func__); 295 ms->section_mem_map = 0; 296 return NULL; 297 } 298 299 void __attribute__((weak)) __meminit vmemmap_populate_print_last(void) 300 { 301 } 302 /* 303 * Allocate the accumulated non-linear sections, allocate a mem_map 304 * for each and record the physical to section mapping. 305 */ 306 void __init sparse_init(void) 307 { 308 unsigned long pnum; 309 struct page *map; 310 unsigned long *usemap; 311 unsigned long **usemap_map; 312 int size; 313 314 /* 315 * map is using big page (aka 2M in x86 64 bit) 316 * usemap is less one page (aka 24 bytes) 317 * so alloc 2M (with 2M align) and 24 bytes in turn will 318 * make next 2M slip to one more 2M later. 319 * then in big system, the memory will have a lot of holes... 320 * here try to allocate 2M pages continously. 321 * 322 * powerpc need to call sparse_init_one_section right after each 323 * sparse_early_mem_map_alloc, so allocate usemap_map at first. 324 */ 325 size = sizeof(unsigned long *) * NR_MEM_SECTIONS; 326 usemap_map = alloc_bootmem(size); 327 if (!usemap_map) 328 panic("can not allocate usemap_map\n"); 329 330 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 331 if (!present_section_nr(pnum)) 332 continue; 333 usemap_map[pnum] = sparse_early_usemap_alloc(pnum); 334 } 335 336 for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) { 337 if (!present_section_nr(pnum)) 338 continue; 339 340 usemap = usemap_map[pnum]; 341 if (!usemap) 342 continue; 343 344 map = sparse_early_mem_map_alloc(pnum); 345 if (!map) 346 continue; 347 348 sparse_init_one_section(__nr_to_section(pnum), pnum, map, 349 usemap); 350 } 351 352 vmemmap_populate_print_last(); 353 354 free_bootmem(__pa(usemap_map), size); 355 } 356 357 #ifdef CONFIG_MEMORY_HOTPLUG 358 #ifdef CONFIG_SPARSEMEM_VMEMMAP 359 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 360 unsigned long nr_pages) 361 { 362 /* This will make the necessary allocations eventually. */ 363 return sparse_mem_map_populate(pnum, nid); 364 } 365 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 366 { 367 return; /* XXX: Not implemented yet */ 368 } 369 static void free_map_bootmem(struct page *page, unsigned long nr_pages) 370 { 371 } 372 #else 373 static struct page *__kmalloc_section_memmap(unsigned long nr_pages) 374 { 375 struct page *page, *ret; 376 unsigned long memmap_size = sizeof(struct page) * nr_pages; 377 378 page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size)); 379 if (page) 380 goto got_map_page; 381 382 ret = vmalloc(memmap_size); 383 if (ret) 384 goto got_map_ptr; 385 386 return NULL; 387 got_map_page: 388 ret = (struct page *)pfn_to_kaddr(page_to_pfn(page)); 389 got_map_ptr: 390 memset(ret, 0, memmap_size); 391 392 return ret; 393 } 394 395 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid, 396 unsigned long nr_pages) 397 { 398 return __kmalloc_section_memmap(nr_pages); 399 } 400 401 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages) 402 { 403 if (is_vmalloc_addr(memmap)) 404 vfree(memmap); 405 else 406 free_pages((unsigned long)memmap, 407 get_order(sizeof(struct page) * nr_pages)); 408 } 409 410 static void free_map_bootmem(struct page *page, unsigned long nr_pages) 411 { 412 unsigned long maps_section_nr, removing_section_nr, i; 413 int magic; 414 415 for (i = 0; i < nr_pages; i++, page++) { 416 magic = atomic_read(&page->_mapcount); 417 418 BUG_ON(magic == NODE_INFO); 419 420 maps_section_nr = pfn_to_section_nr(page_to_pfn(page)); 421 removing_section_nr = page->private; 422 423 /* 424 * When this function is called, the removing section is 425 * logical offlined state. This means all pages are isolated 426 * from page allocator. If removing section's memmap is placed 427 * on the same section, it must not be freed. 428 * If it is freed, page allocator may allocate it which will 429 * be removed physically soon. 430 */ 431 if (maps_section_nr != removing_section_nr) 432 put_page_bootmem(page); 433 } 434 } 435 #endif /* CONFIG_SPARSEMEM_VMEMMAP */ 436 437 static void free_section_usemap(struct page *memmap, unsigned long *usemap) 438 { 439 struct page *usemap_page; 440 unsigned long nr_pages; 441 442 if (!usemap) 443 return; 444 445 usemap_page = virt_to_page(usemap); 446 /* 447 * Check to see if allocation came from hot-plug-add 448 */ 449 if (PageSlab(usemap_page)) { 450 kfree(usemap); 451 if (memmap) 452 __kfree_section_memmap(memmap, PAGES_PER_SECTION); 453 return; 454 } 455 456 /* 457 * The usemap came from bootmem. This is packed with other usemaps 458 * on the section which has pgdat at boot time. Just keep it as is now. 459 */ 460 461 if (memmap) { 462 struct page *memmap_page; 463 memmap_page = virt_to_page(memmap); 464 465 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) 466 >> PAGE_SHIFT; 467 468 free_map_bootmem(memmap_page, nr_pages); 469 } 470 } 471 472 /* 473 * returns the number of sections whose mem_maps were properly 474 * set. If this is <=0, then that means that the passed-in 475 * map was not consumed and must be freed. 476 */ 477 int sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 478 int nr_pages) 479 { 480 unsigned long section_nr = pfn_to_section_nr(start_pfn); 481 struct pglist_data *pgdat = zone->zone_pgdat; 482 struct mem_section *ms; 483 struct page *memmap; 484 unsigned long *usemap; 485 unsigned long flags; 486 int ret; 487 488 /* 489 * no locking for this, because it does its own 490 * plus, it does a kmalloc 491 */ 492 ret = sparse_index_init(section_nr, pgdat->node_id); 493 if (ret < 0 && ret != -EEXIST) 494 return ret; 495 memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages); 496 if (!memmap) 497 return -ENOMEM; 498 usemap = __kmalloc_section_usemap(); 499 if (!usemap) { 500 __kfree_section_memmap(memmap, nr_pages); 501 return -ENOMEM; 502 } 503 504 pgdat_resize_lock(pgdat, &flags); 505 506 ms = __pfn_to_section(start_pfn); 507 if (ms->section_mem_map & SECTION_MARKED_PRESENT) { 508 ret = -EEXIST; 509 goto out; 510 } 511 512 ms->section_mem_map |= SECTION_MARKED_PRESENT; 513 514 ret = sparse_init_one_section(ms, section_nr, memmap, usemap); 515 516 out: 517 pgdat_resize_unlock(pgdat, &flags); 518 if (ret <= 0) { 519 kfree(usemap); 520 __kfree_section_memmap(memmap, nr_pages); 521 } 522 return ret; 523 } 524 525 void sparse_remove_one_section(struct zone *zone, struct mem_section *ms) 526 { 527 struct page *memmap = NULL; 528 unsigned long *usemap = NULL; 529 530 if (ms->section_mem_map) { 531 usemap = ms->pageblock_flags; 532 memmap = sparse_decode_mem_map(ms->section_mem_map, 533 __section_nr(ms)); 534 ms->section_mem_map = 0; 535 ms->pageblock_flags = NULL; 536 } 537 538 free_section_usemap(memmap, usemap); 539 } 540 #endif 541