1 /* 2 * linux/mm/memory_hotplug.c 3 * 4 * Copyright (C) 5 */ 6 7 #include <linux/stddef.h> 8 #include <linux/mm.h> 9 #include <linux/swap.h> 10 #include <linux/interrupt.h> 11 #include <linux/pagemap.h> 12 #include <linux/compiler.h> 13 #include <linux/export.h> 14 #include <linux/pagevec.h> 15 #include <linux/writeback.h> 16 #include <linux/slab.h> 17 #include <linux/sysctl.h> 18 #include <linux/cpu.h> 19 #include <linux/memory.h> 20 #include <linux/memory_hotplug.h> 21 #include <linux/highmem.h> 22 #include <linux/vmalloc.h> 23 #include <linux/ioport.h> 24 #include <linux/delay.h> 25 #include <linux/migrate.h> 26 #include <linux/page-isolation.h> 27 #include <linux/pfn.h> 28 #include <linux/suspend.h> 29 #include <linux/mm_inline.h> 30 #include <linux/firmware-map.h> 31 #include <linux/stop_machine.h> 32 #include <linux/hugetlb.h> 33 #include <linux/memblock.h> 34 35 #include <asm/tlbflush.h> 36 37 #include "internal.h" 38 39 /* 40 * online_page_callback contains pointer to current page onlining function. 41 * Initially it is generic_online_page(). If it is required it could be 42 * changed by calling set_online_page_callback() for callback registration 43 * and restore_online_page_callback() for generic callback restore. 44 */ 45 46 static void generic_online_page(struct page *page); 47 48 static online_page_callback_t online_page_callback = generic_online_page; 49 static DEFINE_MUTEX(online_page_callback_lock); 50 51 /* The same as the cpu_hotplug lock, but for memory hotplug. */ 52 static struct { 53 struct task_struct *active_writer; 54 struct mutex lock; /* Synchronizes accesses to refcount, */ 55 /* 56 * Also blocks the new readers during 57 * an ongoing mem hotplug operation. 58 */ 59 int refcount; 60 61 #ifdef CONFIG_DEBUG_LOCK_ALLOC 62 struct lockdep_map dep_map; 63 #endif 64 } mem_hotplug = { 65 .active_writer = NULL, 66 .lock = __MUTEX_INITIALIZER(mem_hotplug.lock), 67 .refcount = 0, 68 #ifdef CONFIG_DEBUG_LOCK_ALLOC 69 .dep_map = {.name = "mem_hotplug.lock" }, 70 #endif 71 }; 72 73 /* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */ 74 #define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map) 75 #define memhp_lock_acquire() lock_map_acquire(&mem_hotplug.dep_map) 76 #define memhp_lock_release() lock_map_release(&mem_hotplug.dep_map) 77 78 void get_online_mems(void) 79 { 80 might_sleep(); 81 if (mem_hotplug.active_writer == current) 82 return; 83 memhp_lock_acquire_read(); 84 mutex_lock(&mem_hotplug.lock); 85 mem_hotplug.refcount++; 86 mutex_unlock(&mem_hotplug.lock); 87 88 } 89 90 void put_online_mems(void) 91 { 92 if (mem_hotplug.active_writer == current) 93 return; 94 mutex_lock(&mem_hotplug.lock); 95 96 if (WARN_ON(!mem_hotplug.refcount)) 97 mem_hotplug.refcount++; /* try to fix things up */ 98 99 if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer)) 100 wake_up_process(mem_hotplug.active_writer); 101 mutex_unlock(&mem_hotplug.lock); 102 memhp_lock_release(); 103 104 } 105 106 static void mem_hotplug_begin(void) 107 { 108 mem_hotplug.active_writer = current; 109 110 memhp_lock_acquire(); 111 for (;;) { 112 mutex_lock(&mem_hotplug.lock); 113 if (likely(!mem_hotplug.refcount)) 114 break; 115 __set_current_state(TASK_UNINTERRUPTIBLE); 116 mutex_unlock(&mem_hotplug.lock); 117 schedule(); 118 } 119 } 120 121 static void mem_hotplug_done(void) 122 { 123 mem_hotplug.active_writer = NULL; 124 mutex_unlock(&mem_hotplug.lock); 125 memhp_lock_release(); 126 } 127 128 /* add this memory to iomem resource */ 129 static struct resource *register_memory_resource(u64 start, u64 size) 130 { 131 struct resource *res; 132 res = kzalloc(sizeof(struct resource), GFP_KERNEL); 133 BUG_ON(!res); 134 135 res->name = "System RAM"; 136 res->start = start; 137 res->end = start + size - 1; 138 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; 139 if (request_resource(&iomem_resource, res) < 0) { 140 pr_debug("System RAM resource %pR cannot be added\n", res); 141 kfree(res); 142 res = NULL; 143 } 144 return res; 145 } 146 147 static void release_memory_resource(struct resource *res) 148 { 149 if (!res) 150 return; 151 release_resource(res); 152 kfree(res); 153 return; 154 } 155 156 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE 157 void get_page_bootmem(unsigned long info, struct page *page, 158 unsigned long type) 159 { 160 page->lru.next = (struct list_head *) type; 161 SetPagePrivate(page); 162 set_page_private(page, info); 163 atomic_inc(&page->_count); 164 } 165 166 void put_page_bootmem(struct page *page) 167 { 168 unsigned long type; 169 170 type = (unsigned long) page->lru.next; 171 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || 172 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); 173 174 if (atomic_dec_return(&page->_count) == 1) { 175 ClearPagePrivate(page); 176 set_page_private(page, 0); 177 INIT_LIST_HEAD(&page->lru); 178 free_reserved_page(page); 179 } 180 } 181 182 #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE 183 #ifndef CONFIG_SPARSEMEM_VMEMMAP 184 static void register_page_bootmem_info_section(unsigned long start_pfn) 185 { 186 unsigned long *usemap, mapsize, section_nr, i; 187 struct mem_section *ms; 188 struct page *page, *memmap; 189 190 section_nr = pfn_to_section_nr(start_pfn); 191 ms = __nr_to_section(section_nr); 192 193 /* Get section's memmap address */ 194 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 195 196 /* 197 * Get page for the memmap's phys address 198 * XXX: need more consideration for sparse_vmemmap... 199 */ 200 page = virt_to_page(memmap); 201 mapsize = sizeof(struct page) * PAGES_PER_SECTION; 202 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; 203 204 /* remember memmap's page */ 205 for (i = 0; i < mapsize; i++, page++) 206 get_page_bootmem(section_nr, page, SECTION_INFO); 207 208 usemap = __nr_to_section(section_nr)->pageblock_flags; 209 page = virt_to_page(usemap); 210 211 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 212 213 for (i = 0; i < mapsize; i++, page++) 214 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 215 216 } 217 #else /* CONFIG_SPARSEMEM_VMEMMAP */ 218 static void register_page_bootmem_info_section(unsigned long start_pfn) 219 { 220 unsigned long *usemap, mapsize, section_nr, i; 221 struct mem_section *ms; 222 struct page *page, *memmap; 223 224 if (!pfn_valid(start_pfn)) 225 return; 226 227 section_nr = pfn_to_section_nr(start_pfn); 228 ms = __nr_to_section(section_nr); 229 230 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 231 232 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); 233 234 usemap = __nr_to_section(section_nr)->pageblock_flags; 235 page = virt_to_page(usemap); 236 237 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 238 239 for (i = 0; i < mapsize; i++, page++) 240 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 241 } 242 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 243 244 void register_page_bootmem_info_node(struct pglist_data *pgdat) 245 { 246 unsigned long i, pfn, end_pfn, nr_pages; 247 int node = pgdat->node_id; 248 struct page *page; 249 struct zone *zone; 250 251 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; 252 page = virt_to_page(pgdat); 253 254 for (i = 0; i < nr_pages; i++, page++) 255 get_page_bootmem(node, page, NODE_INFO); 256 257 zone = &pgdat->node_zones[0]; 258 for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) { 259 if (zone_is_initialized(zone)) { 260 nr_pages = zone->wait_table_hash_nr_entries 261 * sizeof(wait_queue_head_t); 262 nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT; 263 page = virt_to_page(zone->wait_table); 264 265 for (i = 0; i < nr_pages; i++, page++) 266 get_page_bootmem(node, page, NODE_INFO); 267 } 268 } 269 270 pfn = pgdat->node_start_pfn; 271 end_pfn = pgdat_end_pfn(pgdat); 272 273 /* register section info */ 274 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 275 /* 276 * Some platforms can assign the same pfn to multiple nodes - on 277 * node0 as well as nodeN. To avoid registering a pfn against 278 * multiple nodes we check that this pfn does not already 279 * reside in some other nodes. 280 */ 281 if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node)) 282 register_page_bootmem_info_section(pfn); 283 } 284 } 285 #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ 286 287 static void grow_zone_span(struct zone *zone, unsigned long start_pfn, 288 unsigned long end_pfn) 289 { 290 unsigned long old_zone_end_pfn; 291 292 zone_span_writelock(zone); 293 294 old_zone_end_pfn = zone_end_pfn(zone); 295 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) 296 zone->zone_start_pfn = start_pfn; 297 298 zone->spanned_pages = max(old_zone_end_pfn, end_pfn) - 299 zone->zone_start_pfn; 300 301 zone_span_writeunlock(zone); 302 } 303 304 static void resize_zone(struct zone *zone, unsigned long start_pfn, 305 unsigned long end_pfn) 306 { 307 zone_span_writelock(zone); 308 309 if (end_pfn - start_pfn) { 310 zone->zone_start_pfn = start_pfn; 311 zone->spanned_pages = end_pfn - start_pfn; 312 } else { 313 /* 314 * make it consist as free_area_init_core(), 315 * if spanned_pages = 0, then keep start_pfn = 0 316 */ 317 zone->zone_start_pfn = 0; 318 zone->spanned_pages = 0; 319 } 320 321 zone_span_writeunlock(zone); 322 } 323 324 static void fix_zone_id(struct zone *zone, unsigned long start_pfn, 325 unsigned long end_pfn) 326 { 327 enum zone_type zid = zone_idx(zone); 328 int nid = zone->zone_pgdat->node_id; 329 unsigned long pfn; 330 331 for (pfn = start_pfn; pfn < end_pfn; pfn++) 332 set_page_links(pfn_to_page(pfn), zid, nid, pfn); 333 } 334 335 /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or 336 * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */ 337 static int __ref ensure_zone_is_initialized(struct zone *zone, 338 unsigned long start_pfn, unsigned long num_pages) 339 { 340 if (!zone_is_initialized(zone)) 341 return init_currently_empty_zone(zone, start_pfn, num_pages, 342 MEMMAP_HOTPLUG); 343 return 0; 344 } 345 346 static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2, 347 unsigned long start_pfn, unsigned long end_pfn) 348 { 349 int ret; 350 unsigned long flags; 351 unsigned long z1_start_pfn; 352 353 ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn); 354 if (ret) 355 return ret; 356 357 pgdat_resize_lock(z1->zone_pgdat, &flags); 358 359 /* can't move pfns which are higher than @z2 */ 360 if (end_pfn > zone_end_pfn(z2)) 361 goto out_fail; 362 /* the move out part must be at the left most of @z2 */ 363 if (start_pfn > z2->zone_start_pfn) 364 goto out_fail; 365 /* must included/overlap */ 366 if (end_pfn <= z2->zone_start_pfn) 367 goto out_fail; 368 369 /* use start_pfn for z1's start_pfn if z1 is empty */ 370 if (!zone_is_empty(z1)) 371 z1_start_pfn = z1->zone_start_pfn; 372 else 373 z1_start_pfn = start_pfn; 374 375 resize_zone(z1, z1_start_pfn, end_pfn); 376 resize_zone(z2, end_pfn, zone_end_pfn(z2)); 377 378 pgdat_resize_unlock(z1->zone_pgdat, &flags); 379 380 fix_zone_id(z1, start_pfn, end_pfn); 381 382 return 0; 383 out_fail: 384 pgdat_resize_unlock(z1->zone_pgdat, &flags); 385 return -1; 386 } 387 388 static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2, 389 unsigned long start_pfn, unsigned long end_pfn) 390 { 391 int ret; 392 unsigned long flags; 393 unsigned long z2_end_pfn; 394 395 ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn); 396 if (ret) 397 return ret; 398 399 pgdat_resize_lock(z1->zone_pgdat, &flags); 400 401 /* can't move pfns which are lower than @z1 */ 402 if (z1->zone_start_pfn > start_pfn) 403 goto out_fail; 404 /* the move out part mast at the right most of @z1 */ 405 if (zone_end_pfn(z1) > end_pfn) 406 goto out_fail; 407 /* must included/overlap */ 408 if (start_pfn >= zone_end_pfn(z1)) 409 goto out_fail; 410 411 /* use end_pfn for z2's end_pfn if z2 is empty */ 412 if (!zone_is_empty(z2)) 413 z2_end_pfn = zone_end_pfn(z2); 414 else 415 z2_end_pfn = end_pfn; 416 417 resize_zone(z1, z1->zone_start_pfn, start_pfn); 418 resize_zone(z2, start_pfn, z2_end_pfn); 419 420 pgdat_resize_unlock(z1->zone_pgdat, &flags); 421 422 fix_zone_id(z2, start_pfn, end_pfn); 423 424 return 0; 425 out_fail: 426 pgdat_resize_unlock(z1->zone_pgdat, &flags); 427 return -1; 428 } 429 430 static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn, 431 unsigned long end_pfn) 432 { 433 unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat); 434 435 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) 436 pgdat->node_start_pfn = start_pfn; 437 438 pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) - 439 pgdat->node_start_pfn; 440 } 441 442 static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn) 443 { 444 struct pglist_data *pgdat = zone->zone_pgdat; 445 int nr_pages = PAGES_PER_SECTION; 446 int nid = pgdat->node_id; 447 int zone_type; 448 unsigned long flags; 449 int ret; 450 451 zone_type = zone - pgdat->node_zones; 452 ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages); 453 if (ret) 454 return ret; 455 456 pgdat_resize_lock(zone->zone_pgdat, &flags); 457 grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages); 458 grow_pgdat_span(zone->zone_pgdat, phys_start_pfn, 459 phys_start_pfn + nr_pages); 460 pgdat_resize_unlock(zone->zone_pgdat, &flags); 461 memmap_init_zone(nr_pages, nid, zone_type, 462 phys_start_pfn, MEMMAP_HOTPLUG); 463 return 0; 464 } 465 466 static int __meminit __add_section(int nid, struct zone *zone, 467 unsigned long phys_start_pfn) 468 { 469 int ret; 470 471 if (pfn_valid(phys_start_pfn)) 472 return -EEXIST; 473 474 ret = sparse_add_one_section(zone, phys_start_pfn); 475 476 if (ret < 0) 477 return ret; 478 479 ret = __add_zone(zone, phys_start_pfn); 480 481 if (ret < 0) 482 return ret; 483 484 return register_new_memory(nid, __pfn_to_section(phys_start_pfn)); 485 } 486 487 /* 488 * Reasonably generic function for adding memory. It is 489 * expected that archs that support memory hotplug will 490 * call this function after deciding the zone to which to 491 * add the new pages. 492 */ 493 int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn, 494 unsigned long nr_pages) 495 { 496 unsigned long i; 497 int err = 0; 498 int start_sec, end_sec; 499 /* during initialize mem_map, align hot-added range to section */ 500 start_sec = pfn_to_section_nr(phys_start_pfn); 501 end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1); 502 503 for (i = start_sec; i <= end_sec; i++) { 504 err = __add_section(nid, zone, i << PFN_SECTION_SHIFT); 505 506 /* 507 * EEXIST is finally dealt with by ioresource collision 508 * check. see add_memory() => register_memory_resource() 509 * Warning will be printed if there is collision. 510 */ 511 if (err && (err != -EEXIST)) 512 break; 513 err = 0; 514 } 515 516 return err; 517 } 518 EXPORT_SYMBOL_GPL(__add_pages); 519 520 #ifdef CONFIG_MEMORY_HOTREMOVE 521 /* find the smallest valid pfn in the range [start_pfn, end_pfn) */ 522 static int find_smallest_section_pfn(int nid, struct zone *zone, 523 unsigned long start_pfn, 524 unsigned long end_pfn) 525 { 526 struct mem_section *ms; 527 528 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) { 529 ms = __pfn_to_section(start_pfn); 530 531 if (unlikely(!valid_section(ms))) 532 continue; 533 534 if (unlikely(pfn_to_nid(start_pfn) != nid)) 535 continue; 536 537 if (zone && zone != page_zone(pfn_to_page(start_pfn))) 538 continue; 539 540 return start_pfn; 541 } 542 543 return 0; 544 } 545 546 /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ 547 static int find_biggest_section_pfn(int nid, struct zone *zone, 548 unsigned long start_pfn, 549 unsigned long end_pfn) 550 { 551 struct mem_section *ms; 552 unsigned long pfn; 553 554 /* pfn is the end pfn of a memory section. */ 555 pfn = end_pfn - 1; 556 for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) { 557 ms = __pfn_to_section(pfn); 558 559 if (unlikely(!valid_section(ms))) 560 continue; 561 562 if (unlikely(pfn_to_nid(pfn) != nid)) 563 continue; 564 565 if (zone && zone != page_zone(pfn_to_page(pfn))) 566 continue; 567 568 return pfn; 569 } 570 571 return 0; 572 } 573 574 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, 575 unsigned long end_pfn) 576 { 577 unsigned long zone_start_pfn = zone->zone_start_pfn; 578 unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */ 579 unsigned long zone_end_pfn = z; 580 unsigned long pfn; 581 struct mem_section *ms; 582 int nid = zone_to_nid(zone); 583 584 zone_span_writelock(zone); 585 if (zone_start_pfn == start_pfn) { 586 /* 587 * If the section is smallest section in the zone, it need 588 * shrink zone->zone_start_pfn and zone->zone_spanned_pages. 589 * In this case, we find second smallest valid mem_section 590 * for shrinking zone. 591 */ 592 pfn = find_smallest_section_pfn(nid, zone, end_pfn, 593 zone_end_pfn); 594 if (pfn) { 595 zone->zone_start_pfn = pfn; 596 zone->spanned_pages = zone_end_pfn - pfn; 597 } 598 } else if (zone_end_pfn == end_pfn) { 599 /* 600 * If the section is biggest section in the zone, it need 601 * shrink zone->spanned_pages. 602 * In this case, we find second biggest valid mem_section for 603 * shrinking zone. 604 */ 605 pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn, 606 start_pfn); 607 if (pfn) 608 zone->spanned_pages = pfn - zone_start_pfn + 1; 609 } 610 611 /* 612 * The section is not biggest or smallest mem_section in the zone, it 613 * only creates a hole in the zone. So in this case, we need not 614 * change the zone. But perhaps, the zone has only hole data. Thus 615 * it check the zone has only hole or not. 616 */ 617 pfn = zone_start_pfn; 618 for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) { 619 ms = __pfn_to_section(pfn); 620 621 if (unlikely(!valid_section(ms))) 622 continue; 623 624 if (page_zone(pfn_to_page(pfn)) != zone) 625 continue; 626 627 /* If the section is current section, it continues the loop */ 628 if (start_pfn == pfn) 629 continue; 630 631 /* If we find valid section, we have nothing to do */ 632 zone_span_writeunlock(zone); 633 return; 634 } 635 636 /* The zone has no valid section */ 637 zone->zone_start_pfn = 0; 638 zone->spanned_pages = 0; 639 zone_span_writeunlock(zone); 640 } 641 642 static void shrink_pgdat_span(struct pglist_data *pgdat, 643 unsigned long start_pfn, unsigned long end_pfn) 644 { 645 unsigned long pgdat_start_pfn = pgdat->node_start_pfn; 646 unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */ 647 unsigned long pgdat_end_pfn = p; 648 unsigned long pfn; 649 struct mem_section *ms; 650 int nid = pgdat->node_id; 651 652 if (pgdat_start_pfn == start_pfn) { 653 /* 654 * If the section is smallest section in the pgdat, it need 655 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages. 656 * In this case, we find second smallest valid mem_section 657 * for shrinking zone. 658 */ 659 pfn = find_smallest_section_pfn(nid, NULL, end_pfn, 660 pgdat_end_pfn); 661 if (pfn) { 662 pgdat->node_start_pfn = pfn; 663 pgdat->node_spanned_pages = pgdat_end_pfn - pfn; 664 } 665 } else if (pgdat_end_pfn == end_pfn) { 666 /* 667 * If the section is biggest section in the pgdat, it need 668 * shrink pgdat->node_spanned_pages. 669 * In this case, we find second biggest valid mem_section for 670 * shrinking zone. 671 */ 672 pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn, 673 start_pfn); 674 if (pfn) 675 pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1; 676 } 677 678 /* 679 * If the section is not biggest or smallest mem_section in the pgdat, 680 * it only creates a hole in the pgdat. So in this case, we need not 681 * change the pgdat. 682 * But perhaps, the pgdat has only hole data. Thus it check the pgdat 683 * has only hole or not. 684 */ 685 pfn = pgdat_start_pfn; 686 for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) { 687 ms = __pfn_to_section(pfn); 688 689 if (unlikely(!valid_section(ms))) 690 continue; 691 692 if (pfn_to_nid(pfn) != nid) 693 continue; 694 695 /* If the section is current section, it continues the loop */ 696 if (start_pfn == pfn) 697 continue; 698 699 /* If we find valid section, we have nothing to do */ 700 return; 701 } 702 703 /* The pgdat has no valid section */ 704 pgdat->node_start_pfn = 0; 705 pgdat->node_spanned_pages = 0; 706 } 707 708 static void __remove_zone(struct zone *zone, unsigned long start_pfn) 709 { 710 struct pglist_data *pgdat = zone->zone_pgdat; 711 int nr_pages = PAGES_PER_SECTION; 712 int zone_type; 713 unsigned long flags; 714 715 zone_type = zone - pgdat->node_zones; 716 717 pgdat_resize_lock(zone->zone_pgdat, &flags); 718 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); 719 shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages); 720 pgdat_resize_unlock(zone->zone_pgdat, &flags); 721 } 722 723 static int __remove_section(struct zone *zone, struct mem_section *ms) 724 { 725 unsigned long start_pfn; 726 int scn_nr; 727 int ret = -EINVAL; 728 729 if (!valid_section(ms)) 730 return ret; 731 732 ret = unregister_memory_section(ms); 733 if (ret) 734 return ret; 735 736 scn_nr = __section_nr(ms); 737 start_pfn = section_nr_to_pfn(scn_nr); 738 __remove_zone(zone, start_pfn); 739 740 sparse_remove_one_section(zone, ms); 741 return 0; 742 } 743 744 /** 745 * __remove_pages() - remove sections of pages from a zone 746 * @zone: zone from which pages need to be removed 747 * @phys_start_pfn: starting pageframe (must be aligned to start of a section) 748 * @nr_pages: number of pages to remove (must be multiple of section size) 749 * 750 * Generic helper function to remove section mappings and sysfs entries 751 * for the section of the memory we are removing. Caller needs to make 752 * sure that pages are marked reserved and zones are adjust properly by 753 * calling offline_pages(). 754 */ 755 int __remove_pages(struct zone *zone, unsigned long phys_start_pfn, 756 unsigned long nr_pages) 757 { 758 unsigned long i; 759 int sections_to_remove; 760 resource_size_t start, size; 761 int ret = 0; 762 763 /* 764 * We can only remove entire sections 765 */ 766 BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK); 767 BUG_ON(nr_pages % PAGES_PER_SECTION); 768 769 start = phys_start_pfn << PAGE_SHIFT; 770 size = nr_pages * PAGE_SIZE; 771 ret = release_mem_region_adjustable(&iomem_resource, start, size); 772 if (ret) { 773 resource_size_t endres = start + size - 1; 774 775 pr_warn("Unable to release resource <%pa-%pa> (%d)\n", 776 &start, &endres, ret); 777 } 778 779 sections_to_remove = nr_pages / PAGES_PER_SECTION; 780 for (i = 0; i < sections_to_remove; i++) { 781 unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION; 782 ret = __remove_section(zone, __pfn_to_section(pfn)); 783 if (ret) 784 break; 785 } 786 return ret; 787 } 788 EXPORT_SYMBOL_GPL(__remove_pages); 789 #endif /* CONFIG_MEMORY_HOTREMOVE */ 790 791 int set_online_page_callback(online_page_callback_t callback) 792 { 793 int rc = -EINVAL; 794 795 get_online_mems(); 796 mutex_lock(&online_page_callback_lock); 797 798 if (online_page_callback == generic_online_page) { 799 online_page_callback = callback; 800 rc = 0; 801 } 802 803 mutex_unlock(&online_page_callback_lock); 804 put_online_mems(); 805 806 return rc; 807 } 808 EXPORT_SYMBOL_GPL(set_online_page_callback); 809 810 int restore_online_page_callback(online_page_callback_t callback) 811 { 812 int rc = -EINVAL; 813 814 get_online_mems(); 815 mutex_lock(&online_page_callback_lock); 816 817 if (online_page_callback == callback) { 818 online_page_callback = generic_online_page; 819 rc = 0; 820 } 821 822 mutex_unlock(&online_page_callback_lock); 823 put_online_mems(); 824 825 return rc; 826 } 827 EXPORT_SYMBOL_GPL(restore_online_page_callback); 828 829 void __online_page_set_limits(struct page *page) 830 { 831 } 832 EXPORT_SYMBOL_GPL(__online_page_set_limits); 833 834 void __online_page_increment_counters(struct page *page) 835 { 836 adjust_managed_page_count(page, 1); 837 } 838 EXPORT_SYMBOL_GPL(__online_page_increment_counters); 839 840 void __online_page_free(struct page *page) 841 { 842 __free_reserved_page(page); 843 } 844 EXPORT_SYMBOL_GPL(__online_page_free); 845 846 static void generic_online_page(struct page *page) 847 { 848 __online_page_set_limits(page); 849 __online_page_increment_counters(page); 850 __online_page_free(page); 851 } 852 853 static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, 854 void *arg) 855 { 856 unsigned long i; 857 unsigned long onlined_pages = *(unsigned long *)arg; 858 struct page *page; 859 if (PageReserved(pfn_to_page(start_pfn))) 860 for (i = 0; i < nr_pages; i++) { 861 page = pfn_to_page(start_pfn + i); 862 (*online_page_callback)(page); 863 onlined_pages++; 864 } 865 *(unsigned long *)arg = onlined_pages; 866 return 0; 867 } 868 869 #ifdef CONFIG_MOVABLE_NODE 870 /* 871 * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have 872 * normal memory. 873 */ 874 static bool can_online_high_movable(struct zone *zone) 875 { 876 return true; 877 } 878 #else /* CONFIG_MOVABLE_NODE */ 879 /* ensure every online node has NORMAL memory */ 880 static bool can_online_high_movable(struct zone *zone) 881 { 882 return node_state(zone_to_nid(zone), N_NORMAL_MEMORY); 883 } 884 #endif /* CONFIG_MOVABLE_NODE */ 885 886 /* check which state of node_states will be changed when online memory */ 887 static void node_states_check_changes_online(unsigned long nr_pages, 888 struct zone *zone, struct memory_notify *arg) 889 { 890 int nid = zone_to_nid(zone); 891 enum zone_type zone_last = ZONE_NORMAL; 892 893 /* 894 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] 895 * contains nodes which have zones of 0...ZONE_NORMAL, 896 * set zone_last to ZONE_NORMAL. 897 * 898 * If we don't have HIGHMEM nor movable node, 899 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of 900 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. 901 */ 902 if (N_MEMORY == N_NORMAL_MEMORY) 903 zone_last = ZONE_MOVABLE; 904 905 /* 906 * if the memory to be online is in a zone of 0...zone_last, and 907 * the zones of 0...zone_last don't have memory before online, we will 908 * need to set the node to node_states[N_NORMAL_MEMORY] after 909 * the memory is online. 910 */ 911 if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY)) 912 arg->status_change_nid_normal = nid; 913 else 914 arg->status_change_nid_normal = -1; 915 916 #ifdef CONFIG_HIGHMEM 917 /* 918 * If we have movable node, node_states[N_HIGH_MEMORY] 919 * contains nodes which have zones of 0...ZONE_HIGHMEM, 920 * set zone_last to ZONE_HIGHMEM. 921 * 922 * If we don't have movable node, node_states[N_NORMAL_MEMORY] 923 * contains nodes which have zones of 0...ZONE_MOVABLE, 924 * set zone_last to ZONE_MOVABLE. 925 */ 926 zone_last = ZONE_HIGHMEM; 927 if (N_MEMORY == N_HIGH_MEMORY) 928 zone_last = ZONE_MOVABLE; 929 930 if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY)) 931 arg->status_change_nid_high = nid; 932 else 933 arg->status_change_nid_high = -1; 934 #else 935 arg->status_change_nid_high = arg->status_change_nid_normal; 936 #endif 937 938 /* 939 * if the node don't have memory befor online, we will need to 940 * set the node to node_states[N_MEMORY] after the memory 941 * is online. 942 */ 943 if (!node_state(nid, N_MEMORY)) 944 arg->status_change_nid = nid; 945 else 946 arg->status_change_nid = -1; 947 } 948 949 static void node_states_set_node(int node, struct memory_notify *arg) 950 { 951 if (arg->status_change_nid_normal >= 0) 952 node_set_state(node, N_NORMAL_MEMORY); 953 954 if (arg->status_change_nid_high >= 0) 955 node_set_state(node, N_HIGH_MEMORY); 956 957 node_set_state(node, N_MEMORY); 958 } 959 960 961 int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type) 962 { 963 unsigned long flags; 964 unsigned long onlined_pages = 0; 965 struct zone *zone; 966 int need_zonelists_rebuild = 0; 967 int nid; 968 int ret; 969 struct memory_notify arg; 970 971 mem_hotplug_begin(); 972 /* 973 * This doesn't need a lock to do pfn_to_page(). 974 * The section can't be removed here because of the 975 * memory_block->state_mutex. 976 */ 977 zone = page_zone(pfn_to_page(pfn)); 978 979 ret = -EINVAL; 980 if ((zone_idx(zone) > ZONE_NORMAL || online_type == ONLINE_MOVABLE) && 981 !can_online_high_movable(zone)) 982 goto out; 983 984 if (online_type == ONLINE_KERNEL && zone_idx(zone) == ZONE_MOVABLE) { 985 if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages)) 986 goto out; 987 } 988 if (online_type == ONLINE_MOVABLE && zone_idx(zone) == ZONE_MOVABLE - 1) { 989 if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages)) 990 goto out; 991 } 992 993 /* Previous code may changed the zone of the pfn range */ 994 zone = page_zone(pfn_to_page(pfn)); 995 996 arg.start_pfn = pfn; 997 arg.nr_pages = nr_pages; 998 node_states_check_changes_online(nr_pages, zone, &arg); 999 1000 nid = pfn_to_nid(pfn); 1001 1002 ret = memory_notify(MEM_GOING_ONLINE, &arg); 1003 ret = notifier_to_errno(ret); 1004 if (ret) { 1005 memory_notify(MEM_CANCEL_ONLINE, &arg); 1006 goto out; 1007 } 1008 /* 1009 * If this zone is not populated, then it is not in zonelist. 1010 * This means the page allocator ignores this zone. 1011 * So, zonelist must be updated after online. 1012 */ 1013 mutex_lock(&zonelists_mutex); 1014 if (!populated_zone(zone)) { 1015 need_zonelists_rebuild = 1; 1016 build_all_zonelists(NULL, zone); 1017 } 1018 1019 ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages, 1020 online_pages_range); 1021 if (ret) { 1022 if (need_zonelists_rebuild) 1023 zone_pcp_reset(zone); 1024 mutex_unlock(&zonelists_mutex); 1025 printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n", 1026 (unsigned long long) pfn << PAGE_SHIFT, 1027 (((unsigned long long) pfn + nr_pages) 1028 << PAGE_SHIFT) - 1); 1029 memory_notify(MEM_CANCEL_ONLINE, &arg); 1030 goto out; 1031 } 1032 1033 zone->present_pages += onlined_pages; 1034 1035 pgdat_resize_lock(zone->zone_pgdat, &flags); 1036 zone->zone_pgdat->node_present_pages += onlined_pages; 1037 pgdat_resize_unlock(zone->zone_pgdat, &flags); 1038 1039 if (onlined_pages) { 1040 node_states_set_node(zone_to_nid(zone), &arg); 1041 if (need_zonelists_rebuild) 1042 build_all_zonelists(NULL, NULL); 1043 else 1044 zone_pcp_update(zone); 1045 } 1046 1047 mutex_unlock(&zonelists_mutex); 1048 1049 init_per_zone_wmark_min(); 1050 1051 if (onlined_pages) 1052 kswapd_run(zone_to_nid(zone)); 1053 1054 vm_total_pages = nr_free_pagecache_pages(); 1055 1056 writeback_set_ratelimit(); 1057 1058 if (onlined_pages) 1059 memory_notify(MEM_ONLINE, &arg); 1060 out: 1061 mem_hotplug_done(); 1062 return ret; 1063 } 1064 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ 1065 1066 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 1067 static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start) 1068 { 1069 struct pglist_data *pgdat; 1070 unsigned long zones_size[MAX_NR_ZONES] = {0}; 1071 unsigned long zholes_size[MAX_NR_ZONES] = {0}; 1072 unsigned long start_pfn = PFN_DOWN(start); 1073 1074 pgdat = NODE_DATA(nid); 1075 if (!pgdat) { 1076 pgdat = arch_alloc_nodedata(nid); 1077 if (!pgdat) 1078 return NULL; 1079 1080 arch_refresh_nodedata(nid, pgdat); 1081 } 1082 1083 /* we can use NODE_DATA(nid) from here */ 1084 1085 /* init node's zones as empty zones, we don't have any present pages.*/ 1086 free_area_init_node(nid, zones_size, start_pfn, zholes_size); 1087 1088 /* 1089 * The node we allocated has no zone fallback lists. For avoiding 1090 * to access not-initialized zonelist, build here. 1091 */ 1092 mutex_lock(&zonelists_mutex); 1093 build_all_zonelists(pgdat, NULL); 1094 mutex_unlock(&zonelists_mutex); 1095 1096 return pgdat; 1097 } 1098 1099 static void rollback_node_hotadd(int nid, pg_data_t *pgdat) 1100 { 1101 arch_refresh_nodedata(nid, NULL); 1102 arch_free_nodedata(pgdat); 1103 return; 1104 } 1105 1106 1107 /** 1108 * try_online_node - online a node if offlined 1109 * 1110 * called by cpu_up() to online a node without onlined memory. 1111 */ 1112 int try_online_node(int nid) 1113 { 1114 pg_data_t *pgdat; 1115 int ret; 1116 1117 if (node_online(nid)) 1118 return 0; 1119 1120 mem_hotplug_begin(); 1121 pgdat = hotadd_new_pgdat(nid, 0); 1122 if (!pgdat) { 1123 pr_err("Cannot online node %d due to NULL pgdat\n", nid); 1124 ret = -ENOMEM; 1125 goto out; 1126 } 1127 node_set_online(nid); 1128 ret = register_one_node(nid); 1129 BUG_ON(ret); 1130 1131 if (pgdat->node_zonelists->_zonerefs->zone == NULL) { 1132 mutex_lock(&zonelists_mutex); 1133 build_all_zonelists(NULL, NULL); 1134 mutex_unlock(&zonelists_mutex); 1135 } 1136 1137 out: 1138 mem_hotplug_done(); 1139 return ret; 1140 } 1141 1142 static int check_hotplug_memory_range(u64 start, u64 size) 1143 { 1144 u64 start_pfn = PFN_DOWN(start); 1145 u64 nr_pages = size >> PAGE_SHIFT; 1146 1147 /* Memory range must be aligned with section */ 1148 if ((start_pfn & ~PAGE_SECTION_MASK) || 1149 (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) { 1150 pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n", 1151 (unsigned long long)start, 1152 (unsigned long long)size); 1153 return -EINVAL; 1154 } 1155 1156 return 0; 1157 } 1158 1159 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 1160 int __ref add_memory(int nid, u64 start, u64 size) 1161 { 1162 pg_data_t *pgdat = NULL; 1163 bool new_pgdat; 1164 bool new_node; 1165 struct resource *res; 1166 int ret; 1167 1168 ret = check_hotplug_memory_range(start, size); 1169 if (ret) 1170 return ret; 1171 1172 res = register_memory_resource(start, size); 1173 ret = -EEXIST; 1174 if (!res) 1175 return ret; 1176 1177 { /* Stupid hack to suppress address-never-null warning */ 1178 void *p = NODE_DATA(nid); 1179 new_pgdat = !p; 1180 } 1181 1182 mem_hotplug_begin(); 1183 1184 new_node = !node_online(nid); 1185 if (new_node) { 1186 pgdat = hotadd_new_pgdat(nid, start); 1187 ret = -ENOMEM; 1188 if (!pgdat) 1189 goto error; 1190 } 1191 1192 /* call arch's memory hotadd */ 1193 ret = arch_add_memory(nid, start, size); 1194 1195 if (ret < 0) 1196 goto error; 1197 1198 /* we online node here. we can't roll back from here. */ 1199 node_set_online(nid); 1200 1201 if (new_node) { 1202 ret = register_one_node(nid); 1203 /* 1204 * If sysfs file of new node can't create, cpu on the node 1205 * can't be hot-added. There is no rollback way now. 1206 * So, check by BUG_ON() to catch it reluctantly.. 1207 */ 1208 BUG_ON(ret); 1209 } 1210 1211 /* create new memmap entry */ 1212 firmware_map_add_hotplug(start, start + size, "System RAM"); 1213 1214 goto out; 1215 1216 error: 1217 /* rollback pgdat allocation and others */ 1218 if (new_pgdat) 1219 rollback_node_hotadd(nid, pgdat); 1220 release_memory_resource(res); 1221 1222 out: 1223 mem_hotplug_done(); 1224 return ret; 1225 } 1226 EXPORT_SYMBOL_GPL(add_memory); 1227 1228 #ifdef CONFIG_MEMORY_HOTREMOVE 1229 /* 1230 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy 1231 * set and the size of the free page is given by page_order(). Using this, 1232 * the function determines if the pageblock contains only free pages. 1233 * Due to buddy contraints, a free page at least the size of a pageblock will 1234 * be located at the start of the pageblock 1235 */ 1236 static inline int pageblock_free(struct page *page) 1237 { 1238 return PageBuddy(page) && page_order(page) >= pageblock_order; 1239 } 1240 1241 /* Return the start of the next active pageblock after a given page */ 1242 static struct page *next_active_pageblock(struct page *page) 1243 { 1244 /* Ensure the starting page is pageblock-aligned */ 1245 BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1)); 1246 1247 /* If the entire pageblock is free, move to the end of free page */ 1248 if (pageblock_free(page)) { 1249 int order; 1250 /* be careful. we don't have locks, page_order can be changed.*/ 1251 order = page_order(page); 1252 if ((order < MAX_ORDER) && (order >= pageblock_order)) 1253 return page + (1 << order); 1254 } 1255 1256 return page + pageblock_nr_pages; 1257 } 1258 1259 /* Checks if this range of memory is likely to be hot-removable. */ 1260 int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) 1261 { 1262 struct page *page = pfn_to_page(start_pfn); 1263 struct page *end_page = page + nr_pages; 1264 1265 /* Check the starting page of each pageblock within the range */ 1266 for (; page < end_page; page = next_active_pageblock(page)) { 1267 if (!is_pageblock_removable_nolock(page)) 1268 return 0; 1269 cond_resched(); 1270 } 1271 1272 /* All pageblocks in the memory block are likely to be hot-removable */ 1273 return 1; 1274 } 1275 1276 /* 1277 * Confirm all pages in a range [start, end) is belongs to the same zone. 1278 */ 1279 static int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn) 1280 { 1281 unsigned long pfn; 1282 struct zone *zone = NULL; 1283 struct page *page; 1284 int i; 1285 for (pfn = start_pfn; 1286 pfn < end_pfn; 1287 pfn += MAX_ORDER_NR_PAGES) { 1288 i = 0; 1289 /* This is just a CONFIG_HOLES_IN_ZONE check.*/ 1290 while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i)) 1291 i++; 1292 if (i == MAX_ORDER_NR_PAGES) 1293 continue; 1294 page = pfn_to_page(pfn + i); 1295 if (zone && page_zone(page) != zone) 1296 return 0; 1297 zone = page_zone(page); 1298 } 1299 return 1; 1300 } 1301 1302 /* 1303 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages 1304 * and hugepages). We scan pfn because it's much easier than scanning over 1305 * linked list. This function returns the pfn of the first found movable 1306 * page if it's found, otherwise 0. 1307 */ 1308 static unsigned long scan_movable_pages(unsigned long start, unsigned long end) 1309 { 1310 unsigned long pfn; 1311 struct page *page; 1312 for (pfn = start; pfn < end; pfn++) { 1313 if (pfn_valid(pfn)) { 1314 page = pfn_to_page(pfn); 1315 if (PageLRU(page)) 1316 return pfn; 1317 if (PageHuge(page)) { 1318 if (is_hugepage_active(page)) 1319 return pfn; 1320 else 1321 pfn = round_up(pfn + 1, 1322 1 << compound_order(page)) - 1; 1323 } 1324 } 1325 } 1326 return 0; 1327 } 1328 1329 #define NR_OFFLINE_AT_ONCE_PAGES (256) 1330 static int 1331 do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) 1332 { 1333 unsigned long pfn; 1334 struct page *page; 1335 int move_pages = NR_OFFLINE_AT_ONCE_PAGES; 1336 int not_managed = 0; 1337 int ret = 0; 1338 LIST_HEAD(source); 1339 1340 for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) { 1341 if (!pfn_valid(pfn)) 1342 continue; 1343 page = pfn_to_page(pfn); 1344 1345 if (PageHuge(page)) { 1346 struct page *head = compound_head(page); 1347 pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1; 1348 if (compound_order(head) > PFN_SECTION_SHIFT) { 1349 ret = -EBUSY; 1350 break; 1351 } 1352 if (isolate_huge_page(page, &source)) 1353 move_pages -= 1 << compound_order(head); 1354 continue; 1355 } 1356 1357 if (!get_page_unless_zero(page)) 1358 continue; 1359 /* 1360 * We can skip free pages. And we can only deal with pages on 1361 * LRU. 1362 */ 1363 ret = isolate_lru_page(page); 1364 if (!ret) { /* Success */ 1365 put_page(page); 1366 list_add_tail(&page->lru, &source); 1367 move_pages--; 1368 inc_zone_page_state(page, NR_ISOLATED_ANON + 1369 page_is_file_cache(page)); 1370 1371 } else { 1372 #ifdef CONFIG_DEBUG_VM 1373 printk(KERN_ALERT "removing pfn %lx from LRU failed\n", 1374 pfn); 1375 dump_page(page, "failed to remove from LRU"); 1376 #endif 1377 put_page(page); 1378 /* Because we don't have big zone->lock. we should 1379 check this again here. */ 1380 if (page_count(page)) { 1381 not_managed++; 1382 ret = -EBUSY; 1383 break; 1384 } 1385 } 1386 } 1387 if (!list_empty(&source)) { 1388 if (not_managed) { 1389 putback_movable_pages(&source); 1390 goto out; 1391 } 1392 1393 /* 1394 * alloc_migrate_target should be improooooved!! 1395 * migrate_pages returns # of failed pages. 1396 */ 1397 ret = migrate_pages(&source, alloc_migrate_target, NULL, 0, 1398 MIGRATE_SYNC, MR_MEMORY_HOTPLUG); 1399 if (ret) 1400 putback_movable_pages(&source); 1401 } 1402 out: 1403 return ret; 1404 } 1405 1406 /* 1407 * remove from free_area[] and mark all as Reserved. 1408 */ 1409 static int 1410 offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages, 1411 void *data) 1412 { 1413 __offline_isolated_pages(start, start + nr_pages); 1414 return 0; 1415 } 1416 1417 static void 1418 offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 1419 { 1420 walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL, 1421 offline_isolated_pages_cb); 1422 } 1423 1424 /* 1425 * Check all pages in range, recoreded as memory resource, are isolated. 1426 */ 1427 static int 1428 check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages, 1429 void *data) 1430 { 1431 int ret; 1432 long offlined = *(long *)data; 1433 ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true); 1434 offlined = nr_pages; 1435 if (!ret) 1436 *(long *)data += offlined; 1437 return ret; 1438 } 1439 1440 static long 1441 check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) 1442 { 1443 long offlined = 0; 1444 int ret; 1445 1446 ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined, 1447 check_pages_isolated_cb); 1448 if (ret < 0) 1449 offlined = (long)ret; 1450 return offlined; 1451 } 1452 1453 #ifdef CONFIG_MOVABLE_NODE 1454 /* 1455 * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have 1456 * normal memory. 1457 */ 1458 static bool can_offline_normal(struct zone *zone, unsigned long nr_pages) 1459 { 1460 return true; 1461 } 1462 #else /* CONFIG_MOVABLE_NODE */ 1463 /* ensure the node has NORMAL memory if it is still online */ 1464 static bool can_offline_normal(struct zone *zone, unsigned long nr_pages) 1465 { 1466 struct pglist_data *pgdat = zone->zone_pgdat; 1467 unsigned long present_pages = 0; 1468 enum zone_type zt; 1469 1470 for (zt = 0; zt <= ZONE_NORMAL; zt++) 1471 present_pages += pgdat->node_zones[zt].present_pages; 1472 1473 if (present_pages > nr_pages) 1474 return true; 1475 1476 present_pages = 0; 1477 for (; zt <= ZONE_MOVABLE; zt++) 1478 present_pages += pgdat->node_zones[zt].present_pages; 1479 1480 /* 1481 * we can't offline the last normal memory until all 1482 * higher memory is offlined. 1483 */ 1484 return present_pages == 0; 1485 } 1486 #endif /* CONFIG_MOVABLE_NODE */ 1487 1488 static int __init cmdline_parse_movable_node(char *p) 1489 { 1490 #ifdef CONFIG_MOVABLE_NODE 1491 /* 1492 * Memory used by the kernel cannot be hot-removed because Linux 1493 * cannot migrate the kernel pages. When memory hotplug is 1494 * enabled, we should prevent memblock from allocating memory 1495 * for the kernel. 1496 * 1497 * ACPI SRAT records all hotpluggable memory ranges. But before 1498 * SRAT is parsed, we don't know about it. 1499 * 1500 * The kernel image is loaded into memory at very early time. We 1501 * cannot prevent this anyway. So on NUMA system, we set any 1502 * node the kernel resides in as un-hotpluggable. 1503 * 1504 * Since on modern servers, one node could have double-digit 1505 * gigabytes memory, we can assume the memory around the kernel 1506 * image is also un-hotpluggable. So before SRAT is parsed, just 1507 * allocate memory near the kernel image to try the best to keep 1508 * the kernel away from hotpluggable memory. 1509 */ 1510 memblock_set_bottom_up(true); 1511 movable_node_enabled = true; 1512 #else 1513 pr_warn("movable_node option not supported\n"); 1514 #endif 1515 return 0; 1516 } 1517 early_param("movable_node", cmdline_parse_movable_node); 1518 1519 /* check which state of node_states will be changed when offline memory */ 1520 static void node_states_check_changes_offline(unsigned long nr_pages, 1521 struct zone *zone, struct memory_notify *arg) 1522 { 1523 struct pglist_data *pgdat = zone->zone_pgdat; 1524 unsigned long present_pages = 0; 1525 enum zone_type zt, zone_last = ZONE_NORMAL; 1526 1527 /* 1528 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY] 1529 * contains nodes which have zones of 0...ZONE_NORMAL, 1530 * set zone_last to ZONE_NORMAL. 1531 * 1532 * If we don't have HIGHMEM nor movable node, 1533 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of 1534 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE. 1535 */ 1536 if (N_MEMORY == N_NORMAL_MEMORY) 1537 zone_last = ZONE_MOVABLE; 1538 1539 /* 1540 * check whether node_states[N_NORMAL_MEMORY] will be changed. 1541 * If the memory to be offline is in a zone of 0...zone_last, 1542 * and it is the last present memory, 0...zone_last will 1543 * become empty after offline , thus we can determind we will 1544 * need to clear the node from node_states[N_NORMAL_MEMORY]. 1545 */ 1546 for (zt = 0; zt <= zone_last; zt++) 1547 present_pages += pgdat->node_zones[zt].present_pages; 1548 if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) 1549 arg->status_change_nid_normal = zone_to_nid(zone); 1550 else 1551 arg->status_change_nid_normal = -1; 1552 1553 #ifdef CONFIG_HIGHMEM 1554 /* 1555 * If we have movable node, node_states[N_HIGH_MEMORY] 1556 * contains nodes which have zones of 0...ZONE_HIGHMEM, 1557 * set zone_last to ZONE_HIGHMEM. 1558 * 1559 * If we don't have movable node, node_states[N_NORMAL_MEMORY] 1560 * contains nodes which have zones of 0...ZONE_MOVABLE, 1561 * set zone_last to ZONE_MOVABLE. 1562 */ 1563 zone_last = ZONE_HIGHMEM; 1564 if (N_MEMORY == N_HIGH_MEMORY) 1565 zone_last = ZONE_MOVABLE; 1566 1567 for (; zt <= zone_last; zt++) 1568 present_pages += pgdat->node_zones[zt].present_pages; 1569 if (zone_idx(zone) <= zone_last && nr_pages >= present_pages) 1570 arg->status_change_nid_high = zone_to_nid(zone); 1571 else 1572 arg->status_change_nid_high = -1; 1573 #else 1574 arg->status_change_nid_high = arg->status_change_nid_normal; 1575 #endif 1576 1577 /* 1578 * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE 1579 */ 1580 zone_last = ZONE_MOVABLE; 1581 1582 /* 1583 * check whether node_states[N_HIGH_MEMORY] will be changed 1584 * If we try to offline the last present @nr_pages from the node, 1585 * we can determind we will need to clear the node from 1586 * node_states[N_HIGH_MEMORY]. 1587 */ 1588 for (; zt <= zone_last; zt++) 1589 present_pages += pgdat->node_zones[zt].present_pages; 1590 if (nr_pages >= present_pages) 1591 arg->status_change_nid = zone_to_nid(zone); 1592 else 1593 arg->status_change_nid = -1; 1594 } 1595 1596 static void node_states_clear_node(int node, struct memory_notify *arg) 1597 { 1598 if (arg->status_change_nid_normal >= 0) 1599 node_clear_state(node, N_NORMAL_MEMORY); 1600 1601 if ((N_MEMORY != N_NORMAL_MEMORY) && 1602 (arg->status_change_nid_high >= 0)) 1603 node_clear_state(node, N_HIGH_MEMORY); 1604 1605 if ((N_MEMORY != N_HIGH_MEMORY) && 1606 (arg->status_change_nid >= 0)) 1607 node_clear_state(node, N_MEMORY); 1608 } 1609 1610 static int __ref __offline_pages(unsigned long start_pfn, 1611 unsigned long end_pfn, unsigned long timeout) 1612 { 1613 unsigned long pfn, nr_pages, expire; 1614 long offlined_pages; 1615 int ret, drain, retry_max, node; 1616 unsigned long flags; 1617 struct zone *zone; 1618 struct memory_notify arg; 1619 1620 /* at least, alignment against pageblock is necessary */ 1621 if (!IS_ALIGNED(start_pfn, pageblock_nr_pages)) 1622 return -EINVAL; 1623 if (!IS_ALIGNED(end_pfn, pageblock_nr_pages)) 1624 return -EINVAL; 1625 /* This makes hotplug much easier...and readable. 1626 we assume this for now. .*/ 1627 if (!test_pages_in_a_zone(start_pfn, end_pfn)) 1628 return -EINVAL; 1629 1630 mem_hotplug_begin(); 1631 1632 zone = page_zone(pfn_to_page(start_pfn)); 1633 node = zone_to_nid(zone); 1634 nr_pages = end_pfn - start_pfn; 1635 1636 ret = -EINVAL; 1637 if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages)) 1638 goto out; 1639 1640 /* set above range as isolated */ 1641 ret = start_isolate_page_range(start_pfn, end_pfn, 1642 MIGRATE_MOVABLE, true); 1643 if (ret) 1644 goto out; 1645 1646 arg.start_pfn = start_pfn; 1647 arg.nr_pages = nr_pages; 1648 node_states_check_changes_offline(nr_pages, zone, &arg); 1649 1650 ret = memory_notify(MEM_GOING_OFFLINE, &arg); 1651 ret = notifier_to_errno(ret); 1652 if (ret) 1653 goto failed_removal; 1654 1655 pfn = start_pfn; 1656 expire = jiffies + timeout; 1657 drain = 0; 1658 retry_max = 5; 1659 repeat: 1660 /* start memory hot removal */ 1661 ret = -EAGAIN; 1662 if (time_after(jiffies, expire)) 1663 goto failed_removal; 1664 ret = -EINTR; 1665 if (signal_pending(current)) 1666 goto failed_removal; 1667 ret = 0; 1668 if (drain) { 1669 lru_add_drain_all(); 1670 cond_resched(); 1671 drain_all_pages(); 1672 } 1673 1674 pfn = scan_movable_pages(start_pfn, end_pfn); 1675 if (pfn) { /* We have movable pages */ 1676 ret = do_migrate_range(pfn, end_pfn); 1677 if (!ret) { 1678 drain = 1; 1679 goto repeat; 1680 } else { 1681 if (ret < 0) 1682 if (--retry_max == 0) 1683 goto failed_removal; 1684 yield(); 1685 drain = 1; 1686 goto repeat; 1687 } 1688 } 1689 /* drain all zone's lru pagevec, this is asynchronous... */ 1690 lru_add_drain_all(); 1691 yield(); 1692 /* drain pcp pages, this is synchronous. */ 1693 drain_all_pages(); 1694 /* 1695 * dissolve free hugepages in the memory block before doing offlining 1696 * actually in order to make hugetlbfs's object counting consistent. 1697 */ 1698 dissolve_free_huge_pages(start_pfn, end_pfn); 1699 /* check again */ 1700 offlined_pages = check_pages_isolated(start_pfn, end_pfn); 1701 if (offlined_pages < 0) { 1702 ret = -EBUSY; 1703 goto failed_removal; 1704 } 1705 printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages); 1706 /* Ok, all of our target is isolated. 1707 We cannot do rollback at this point. */ 1708 offline_isolated_pages(start_pfn, end_pfn); 1709 /* reset pagetype flags and makes migrate type to be MOVABLE */ 1710 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1711 /* removal success */ 1712 adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages); 1713 zone->present_pages -= offlined_pages; 1714 1715 pgdat_resize_lock(zone->zone_pgdat, &flags); 1716 zone->zone_pgdat->node_present_pages -= offlined_pages; 1717 pgdat_resize_unlock(zone->zone_pgdat, &flags); 1718 1719 init_per_zone_wmark_min(); 1720 1721 if (!populated_zone(zone)) { 1722 zone_pcp_reset(zone); 1723 mutex_lock(&zonelists_mutex); 1724 build_all_zonelists(NULL, NULL); 1725 mutex_unlock(&zonelists_mutex); 1726 } else 1727 zone_pcp_update(zone); 1728 1729 node_states_clear_node(node, &arg); 1730 if (arg.status_change_nid >= 0) 1731 kswapd_stop(node); 1732 1733 vm_total_pages = nr_free_pagecache_pages(); 1734 writeback_set_ratelimit(); 1735 1736 memory_notify(MEM_OFFLINE, &arg); 1737 mem_hotplug_done(); 1738 return 0; 1739 1740 failed_removal: 1741 printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n", 1742 (unsigned long long) start_pfn << PAGE_SHIFT, 1743 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1); 1744 memory_notify(MEM_CANCEL_OFFLINE, &arg); 1745 /* pushback to free area */ 1746 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1747 1748 out: 1749 mem_hotplug_done(); 1750 return ret; 1751 } 1752 1753 int offline_pages(unsigned long start_pfn, unsigned long nr_pages) 1754 { 1755 return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ); 1756 } 1757 #endif /* CONFIG_MEMORY_HOTREMOVE */ 1758 1759 /** 1760 * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn) 1761 * @start_pfn: start pfn of the memory range 1762 * @end_pfn: end pfn of the memory range 1763 * @arg: argument passed to func 1764 * @func: callback for each memory section walked 1765 * 1766 * This function walks through all present mem sections in range 1767 * [start_pfn, end_pfn) and call func on each mem section. 1768 * 1769 * Returns the return value of func. 1770 */ 1771 int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn, 1772 void *arg, int (*func)(struct memory_block *, void *)) 1773 { 1774 struct memory_block *mem = NULL; 1775 struct mem_section *section; 1776 unsigned long pfn, section_nr; 1777 int ret; 1778 1779 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1780 section_nr = pfn_to_section_nr(pfn); 1781 if (!present_section_nr(section_nr)) 1782 continue; 1783 1784 section = __nr_to_section(section_nr); 1785 /* same memblock? */ 1786 if (mem) 1787 if ((section_nr >= mem->start_section_nr) && 1788 (section_nr <= mem->end_section_nr)) 1789 continue; 1790 1791 mem = find_memory_block_hinted(section, mem); 1792 if (!mem) 1793 continue; 1794 1795 ret = func(mem, arg); 1796 if (ret) { 1797 kobject_put(&mem->dev.kobj); 1798 return ret; 1799 } 1800 } 1801 1802 if (mem) 1803 kobject_put(&mem->dev.kobj); 1804 1805 return 0; 1806 } 1807 1808 #ifdef CONFIG_MEMORY_HOTREMOVE 1809 static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) 1810 { 1811 int ret = !is_memblock_offlined(mem); 1812 1813 if (unlikely(ret)) { 1814 phys_addr_t beginpa, endpa; 1815 1816 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); 1817 endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1; 1818 pr_warn("removing memory fails, because memory " 1819 "[%pa-%pa] is onlined\n", 1820 &beginpa, &endpa); 1821 } 1822 1823 return ret; 1824 } 1825 1826 static int check_cpu_on_node(pg_data_t *pgdat) 1827 { 1828 int cpu; 1829 1830 for_each_present_cpu(cpu) { 1831 if (cpu_to_node(cpu) == pgdat->node_id) 1832 /* 1833 * the cpu on this node isn't removed, and we can't 1834 * offline this node. 1835 */ 1836 return -EBUSY; 1837 } 1838 1839 return 0; 1840 } 1841 1842 static void unmap_cpu_on_node(pg_data_t *pgdat) 1843 { 1844 #ifdef CONFIG_ACPI_NUMA 1845 int cpu; 1846 1847 for_each_possible_cpu(cpu) 1848 if (cpu_to_node(cpu) == pgdat->node_id) 1849 numa_clear_node(cpu); 1850 #endif 1851 } 1852 1853 static int check_and_unmap_cpu_on_node(pg_data_t *pgdat) 1854 { 1855 int ret; 1856 1857 ret = check_cpu_on_node(pgdat); 1858 if (ret) 1859 return ret; 1860 1861 /* 1862 * the node will be offlined when we come here, so we can clear 1863 * the cpu_to_node() now. 1864 */ 1865 1866 unmap_cpu_on_node(pgdat); 1867 return 0; 1868 } 1869 1870 /** 1871 * try_offline_node 1872 * 1873 * Offline a node if all memory sections and cpus of the node are removed. 1874 * 1875 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1876 * and online/offline operations before this call. 1877 */ 1878 void try_offline_node(int nid) 1879 { 1880 pg_data_t *pgdat = NODE_DATA(nid); 1881 unsigned long start_pfn = pgdat->node_start_pfn; 1882 unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages; 1883 unsigned long pfn; 1884 struct page *pgdat_page = virt_to_page(pgdat); 1885 int i; 1886 1887 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1888 unsigned long section_nr = pfn_to_section_nr(pfn); 1889 1890 if (!present_section_nr(section_nr)) 1891 continue; 1892 1893 if (pfn_to_nid(pfn) != nid) 1894 continue; 1895 1896 /* 1897 * some memory sections of this node are not removed, and we 1898 * can't offline node now. 1899 */ 1900 return; 1901 } 1902 1903 if (check_and_unmap_cpu_on_node(pgdat)) 1904 return; 1905 1906 /* 1907 * all memory/cpu of this node are removed, we can offline this 1908 * node now. 1909 */ 1910 node_set_offline(nid); 1911 unregister_one_node(nid); 1912 1913 if (!PageSlab(pgdat_page) && !PageCompound(pgdat_page)) 1914 /* node data is allocated from boot memory */ 1915 return; 1916 1917 /* free waittable in each zone */ 1918 for (i = 0; i < MAX_NR_ZONES; i++) { 1919 struct zone *zone = pgdat->node_zones + i; 1920 1921 /* 1922 * wait_table may be allocated from boot memory, 1923 * here only free if it's allocated by vmalloc. 1924 */ 1925 if (is_vmalloc_addr(zone->wait_table)) 1926 vfree(zone->wait_table); 1927 } 1928 1929 /* 1930 * Since there is no way to guarentee the address of pgdat/zone is not 1931 * on stack of any kernel threads or used by other kernel objects 1932 * without reference counting or other symchronizing method, do not 1933 * reset node_data and free pgdat here. Just reset it to 0 and reuse 1934 * the memory when the node is online again. 1935 */ 1936 memset(pgdat, 0, sizeof(*pgdat)); 1937 } 1938 EXPORT_SYMBOL(try_offline_node); 1939 1940 /** 1941 * remove_memory 1942 * 1943 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1944 * and online/offline operations before this call, as required by 1945 * try_offline_node(). 1946 */ 1947 void __ref remove_memory(int nid, u64 start, u64 size) 1948 { 1949 int ret; 1950 1951 BUG_ON(check_hotplug_memory_range(start, size)); 1952 1953 mem_hotplug_begin(); 1954 1955 /* 1956 * All memory blocks must be offlined before removing memory. Check 1957 * whether all memory blocks in question are offline and trigger a BUG() 1958 * if this is not the case. 1959 */ 1960 ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL, 1961 check_memblock_offlined_cb); 1962 if (ret) 1963 BUG(); 1964 1965 /* remove memmap entry */ 1966 firmware_map_remove(start, start + size, "System RAM"); 1967 1968 arch_remove_memory(start, size); 1969 1970 try_offline_node(nid); 1971 1972 mem_hotplug_done(); 1973 } 1974 EXPORT_SYMBOL_GPL(remove_memory); 1975 #endif /* CONFIG_MEMORY_HOTREMOVE */ 1976