1 /* 2 * linux/mm/memory_hotplug.c 3 * 4 * Copyright (C) 5 */ 6 7 #include <linux/stddef.h> 8 #include <linux/mm.h> 9 #include <linux/sched/signal.h> 10 #include <linux/swap.h> 11 #include <linux/interrupt.h> 12 #include <linux/pagemap.h> 13 #include <linux/compiler.h> 14 #include <linux/export.h> 15 #include <linux/pagevec.h> 16 #include <linux/writeback.h> 17 #include <linux/slab.h> 18 #include <linux/sysctl.h> 19 #include <linux/cpu.h> 20 #include <linux/memory.h> 21 #include <linux/memremap.h> 22 #include <linux/memory_hotplug.h> 23 #include <linux/highmem.h> 24 #include <linux/vmalloc.h> 25 #include <linux/ioport.h> 26 #include <linux/delay.h> 27 #include <linux/migrate.h> 28 #include <linux/page-isolation.h> 29 #include <linux/pfn.h> 30 #include <linux/suspend.h> 31 #include <linux/mm_inline.h> 32 #include <linux/firmware-map.h> 33 #include <linux/stop_machine.h> 34 #include <linux/hugetlb.h> 35 #include <linux/memblock.h> 36 #include <linux/compaction.h> 37 #include <linux/rmap.h> 38 39 #include <asm/tlbflush.h> 40 41 #include "internal.h" 42 43 /* 44 * online_page_callback contains pointer to current page onlining function. 45 * Initially it is generic_online_page(). If it is required it could be 46 * changed by calling set_online_page_callback() for callback registration 47 * and restore_online_page_callback() for generic callback restore. 48 */ 49 50 static void generic_online_page(struct page *page, unsigned int order); 51 52 static online_page_callback_t online_page_callback = generic_online_page; 53 static DEFINE_MUTEX(online_page_callback_lock); 54 55 DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock); 56 57 void get_online_mems(void) 58 { 59 percpu_down_read(&mem_hotplug_lock); 60 } 61 62 void put_online_mems(void) 63 { 64 percpu_up_read(&mem_hotplug_lock); 65 } 66 67 bool movable_node_enabled = false; 68 69 #ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE 70 bool memhp_auto_online; 71 #else 72 bool memhp_auto_online = true; 73 #endif 74 EXPORT_SYMBOL_GPL(memhp_auto_online); 75 76 static int __init setup_memhp_default_state(char *str) 77 { 78 if (!strcmp(str, "online")) 79 memhp_auto_online = true; 80 else if (!strcmp(str, "offline")) 81 memhp_auto_online = false; 82 83 return 1; 84 } 85 __setup("memhp_default_state=", setup_memhp_default_state); 86 87 void mem_hotplug_begin(void) 88 { 89 cpus_read_lock(); 90 percpu_down_write(&mem_hotplug_lock); 91 } 92 93 void mem_hotplug_done(void) 94 { 95 percpu_up_write(&mem_hotplug_lock); 96 cpus_read_unlock(); 97 } 98 99 /* add this memory to iomem resource */ 100 static struct resource *register_memory_resource(u64 start, u64 size) 101 { 102 struct resource *res, *conflict; 103 res = kzalloc(sizeof(struct resource), GFP_KERNEL); 104 if (!res) 105 return ERR_PTR(-ENOMEM); 106 107 res->name = "System RAM"; 108 res->start = start; 109 res->end = start + size - 1; 110 res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY; 111 conflict = request_resource_conflict(&iomem_resource, res); 112 if (conflict) { 113 if (conflict->desc == IORES_DESC_DEVICE_PRIVATE_MEMORY) { 114 pr_debug("Device unaddressable memory block " 115 "memory hotplug at %#010llx !\n", 116 (unsigned long long)start); 117 } 118 pr_debug("System RAM resource %pR cannot be added\n", res); 119 kfree(res); 120 return ERR_PTR(-EEXIST); 121 } 122 return res; 123 } 124 125 static void release_memory_resource(struct resource *res) 126 { 127 if (!res) 128 return; 129 release_resource(res); 130 kfree(res); 131 return; 132 } 133 134 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE 135 void get_page_bootmem(unsigned long info, struct page *page, 136 unsigned long type) 137 { 138 page->freelist = (void *)type; 139 SetPagePrivate(page); 140 set_page_private(page, info); 141 page_ref_inc(page); 142 } 143 144 void put_page_bootmem(struct page *page) 145 { 146 unsigned long type; 147 148 type = (unsigned long) page->freelist; 149 BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE || 150 type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE); 151 152 if (page_ref_dec_return(page) == 1) { 153 page->freelist = NULL; 154 ClearPagePrivate(page); 155 set_page_private(page, 0); 156 INIT_LIST_HEAD(&page->lru); 157 free_reserved_page(page); 158 } 159 } 160 161 #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE 162 #ifndef CONFIG_SPARSEMEM_VMEMMAP 163 static void register_page_bootmem_info_section(unsigned long start_pfn) 164 { 165 unsigned long *usemap, mapsize, section_nr, i; 166 struct mem_section *ms; 167 struct page *page, *memmap; 168 169 section_nr = pfn_to_section_nr(start_pfn); 170 ms = __nr_to_section(section_nr); 171 172 /* Get section's memmap address */ 173 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 174 175 /* 176 * Get page for the memmap's phys address 177 * XXX: need more consideration for sparse_vmemmap... 178 */ 179 page = virt_to_page(memmap); 180 mapsize = sizeof(struct page) * PAGES_PER_SECTION; 181 mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT; 182 183 /* remember memmap's page */ 184 for (i = 0; i < mapsize; i++, page++) 185 get_page_bootmem(section_nr, page, SECTION_INFO); 186 187 usemap = ms->pageblock_flags; 188 page = virt_to_page(usemap); 189 190 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 191 192 for (i = 0; i < mapsize; i++, page++) 193 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 194 195 } 196 #else /* CONFIG_SPARSEMEM_VMEMMAP */ 197 static void register_page_bootmem_info_section(unsigned long start_pfn) 198 { 199 unsigned long *usemap, mapsize, section_nr, i; 200 struct mem_section *ms; 201 struct page *page, *memmap; 202 203 section_nr = pfn_to_section_nr(start_pfn); 204 ms = __nr_to_section(section_nr); 205 206 memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr); 207 208 register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION); 209 210 usemap = ms->pageblock_flags; 211 page = virt_to_page(usemap); 212 213 mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT; 214 215 for (i = 0; i < mapsize; i++, page++) 216 get_page_bootmem(section_nr, page, MIX_SECTION_INFO); 217 } 218 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */ 219 220 void __init register_page_bootmem_info_node(struct pglist_data *pgdat) 221 { 222 unsigned long i, pfn, end_pfn, nr_pages; 223 int node = pgdat->node_id; 224 struct page *page; 225 226 nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT; 227 page = virt_to_page(pgdat); 228 229 for (i = 0; i < nr_pages; i++, page++) 230 get_page_bootmem(node, page, NODE_INFO); 231 232 pfn = pgdat->node_start_pfn; 233 end_pfn = pgdat_end_pfn(pgdat); 234 235 /* register section info */ 236 for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 237 /* 238 * Some platforms can assign the same pfn to multiple nodes - on 239 * node0 as well as nodeN. To avoid registering a pfn against 240 * multiple nodes we check that this pfn does not already 241 * reside in some other nodes. 242 */ 243 if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node)) 244 register_page_bootmem_info_section(pfn); 245 } 246 } 247 #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */ 248 249 static int __meminit __add_section(int nid, unsigned long phys_start_pfn, 250 struct vmem_altmap *altmap, bool want_memblock) 251 { 252 int ret; 253 254 if (pfn_valid(phys_start_pfn)) 255 return -EEXIST; 256 257 ret = sparse_add_one_section(nid, phys_start_pfn, altmap); 258 if (ret < 0) 259 return ret; 260 261 if (!want_memblock) 262 return 0; 263 264 return hotplug_memory_register(nid, __pfn_to_section(phys_start_pfn)); 265 } 266 267 /* 268 * Reasonably generic function for adding memory. It is 269 * expected that archs that support memory hotplug will 270 * call this function after deciding the zone to which to 271 * add the new pages. 272 */ 273 int __ref __add_pages(int nid, unsigned long phys_start_pfn, 274 unsigned long nr_pages, struct vmem_altmap *altmap, 275 bool want_memblock) 276 { 277 unsigned long i; 278 int err = 0; 279 int start_sec, end_sec; 280 281 /* during initialize mem_map, align hot-added range to section */ 282 start_sec = pfn_to_section_nr(phys_start_pfn); 283 end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1); 284 285 if (altmap) { 286 /* 287 * Validate altmap is within bounds of the total request 288 */ 289 if (altmap->base_pfn != phys_start_pfn 290 || vmem_altmap_offset(altmap) > nr_pages) { 291 pr_warn_once("memory add fail, invalid altmap\n"); 292 err = -EINVAL; 293 goto out; 294 } 295 altmap->alloc = 0; 296 } 297 298 for (i = start_sec; i <= end_sec; i++) { 299 err = __add_section(nid, section_nr_to_pfn(i), altmap, 300 want_memblock); 301 302 /* 303 * EEXIST is finally dealt with by ioresource collision 304 * check. see add_memory() => register_memory_resource() 305 * Warning will be printed if there is collision. 306 */ 307 if (err && (err != -EEXIST)) 308 break; 309 err = 0; 310 cond_resched(); 311 } 312 vmemmap_populate_print_last(); 313 out: 314 return err; 315 } 316 317 #ifdef CONFIG_MEMORY_HOTREMOVE 318 /* find the smallest valid pfn in the range [start_pfn, end_pfn) */ 319 static unsigned long find_smallest_section_pfn(int nid, struct zone *zone, 320 unsigned long start_pfn, 321 unsigned long end_pfn) 322 { 323 struct mem_section *ms; 324 325 for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) { 326 ms = __pfn_to_section(start_pfn); 327 328 if (unlikely(!valid_section(ms))) 329 continue; 330 331 if (unlikely(pfn_to_nid(start_pfn) != nid)) 332 continue; 333 334 if (zone && zone != page_zone(pfn_to_page(start_pfn))) 335 continue; 336 337 return start_pfn; 338 } 339 340 return 0; 341 } 342 343 /* find the biggest valid pfn in the range [start_pfn, end_pfn). */ 344 static unsigned long find_biggest_section_pfn(int nid, struct zone *zone, 345 unsigned long start_pfn, 346 unsigned long end_pfn) 347 { 348 struct mem_section *ms; 349 unsigned long pfn; 350 351 /* pfn is the end pfn of a memory section. */ 352 pfn = end_pfn - 1; 353 for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) { 354 ms = __pfn_to_section(pfn); 355 356 if (unlikely(!valid_section(ms))) 357 continue; 358 359 if (unlikely(pfn_to_nid(pfn) != nid)) 360 continue; 361 362 if (zone && zone != page_zone(pfn_to_page(pfn))) 363 continue; 364 365 return pfn; 366 } 367 368 return 0; 369 } 370 371 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn, 372 unsigned long end_pfn) 373 { 374 unsigned long zone_start_pfn = zone->zone_start_pfn; 375 unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */ 376 unsigned long zone_end_pfn = z; 377 unsigned long pfn; 378 struct mem_section *ms; 379 int nid = zone_to_nid(zone); 380 381 zone_span_writelock(zone); 382 if (zone_start_pfn == start_pfn) { 383 /* 384 * If the section is smallest section in the zone, it need 385 * shrink zone->zone_start_pfn and zone->zone_spanned_pages. 386 * In this case, we find second smallest valid mem_section 387 * for shrinking zone. 388 */ 389 pfn = find_smallest_section_pfn(nid, zone, end_pfn, 390 zone_end_pfn); 391 if (pfn) { 392 zone->zone_start_pfn = pfn; 393 zone->spanned_pages = zone_end_pfn - pfn; 394 } 395 } else if (zone_end_pfn == end_pfn) { 396 /* 397 * If the section is biggest section in the zone, it need 398 * shrink zone->spanned_pages. 399 * In this case, we find second biggest valid mem_section for 400 * shrinking zone. 401 */ 402 pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn, 403 start_pfn); 404 if (pfn) 405 zone->spanned_pages = pfn - zone_start_pfn + 1; 406 } 407 408 /* 409 * The section is not biggest or smallest mem_section in the zone, it 410 * only creates a hole in the zone. So in this case, we need not 411 * change the zone. But perhaps, the zone has only hole data. Thus 412 * it check the zone has only hole or not. 413 */ 414 pfn = zone_start_pfn; 415 for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) { 416 ms = __pfn_to_section(pfn); 417 418 if (unlikely(!valid_section(ms))) 419 continue; 420 421 if (page_zone(pfn_to_page(pfn)) != zone) 422 continue; 423 424 /* If the section is current section, it continues the loop */ 425 if (start_pfn == pfn) 426 continue; 427 428 /* If we find valid section, we have nothing to do */ 429 zone_span_writeunlock(zone); 430 return; 431 } 432 433 /* The zone has no valid section */ 434 zone->zone_start_pfn = 0; 435 zone->spanned_pages = 0; 436 zone_span_writeunlock(zone); 437 } 438 439 static void shrink_pgdat_span(struct pglist_data *pgdat, 440 unsigned long start_pfn, unsigned long end_pfn) 441 { 442 unsigned long pgdat_start_pfn = pgdat->node_start_pfn; 443 unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */ 444 unsigned long pgdat_end_pfn = p; 445 unsigned long pfn; 446 struct mem_section *ms; 447 int nid = pgdat->node_id; 448 449 if (pgdat_start_pfn == start_pfn) { 450 /* 451 * If the section is smallest section in the pgdat, it need 452 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages. 453 * In this case, we find second smallest valid mem_section 454 * for shrinking zone. 455 */ 456 pfn = find_smallest_section_pfn(nid, NULL, end_pfn, 457 pgdat_end_pfn); 458 if (pfn) { 459 pgdat->node_start_pfn = pfn; 460 pgdat->node_spanned_pages = pgdat_end_pfn - pfn; 461 } 462 } else if (pgdat_end_pfn == end_pfn) { 463 /* 464 * If the section is biggest section in the pgdat, it need 465 * shrink pgdat->node_spanned_pages. 466 * In this case, we find second biggest valid mem_section for 467 * shrinking zone. 468 */ 469 pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn, 470 start_pfn); 471 if (pfn) 472 pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1; 473 } 474 475 /* 476 * If the section is not biggest or smallest mem_section in the pgdat, 477 * it only creates a hole in the pgdat. So in this case, we need not 478 * change the pgdat. 479 * But perhaps, the pgdat has only hole data. Thus it check the pgdat 480 * has only hole or not. 481 */ 482 pfn = pgdat_start_pfn; 483 for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) { 484 ms = __pfn_to_section(pfn); 485 486 if (unlikely(!valid_section(ms))) 487 continue; 488 489 if (pfn_to_nid(pfn) != nid) 490 continue; 491 492 /* If the section is current section, it continues the loop */ 493 if (start_pfn == pfn) 494 continue; 495 496 /* If we find valid section, we have nothing to do */ 497 return; 498 } 499 500 /* The pgdat has no valid section */ 501 pgdat->node_start_pfn = 0; 502 pgdat->node_spanned_pages = 0; 503 } 504 505 static void __remove_zone(struct zone *zone, unsigned long start_pfn) 506 { 507 struct pglist_data *pgdat = zone->zone_pgdat; 508 int nr_pages = PAGES_PER_SECTION; 509 unsigned long flags; 510 511 pgdat_resize_lock(zone->zone_pgdat, &flags); 512 shrink_zone_span(zone, start_pfn, start_pfn + nr_pages); 513 shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages); 514 pgdat_resize_unlock(zone->zone_pgdat, &flags); 515 } 516 517 static int __remove_section(struct zone *zone, struct mem_section *ms, 518 unsigned long map_offset, struct vmem_altmap *altmap) 519 { 520 unsigned long start_pfn; 521 int scn_nr; 522 int ret = -EINVAL; 523 524 if (!valid_section(ms)) 525 return ret; 526 527 ret = unregister_memory_section(ms); 528 if (ret) 529 return ret; 530 531 scn_nr = __section_nr(ms); 532 start_pfn = section_nr_to_pfn((unsigned long)scn_nr); 533 __remove_zone(zone, start_pfn); 534 535 sparse_remove_one_section(zone, ms, map_offset, altmap); 536 return 0; 537 } 538 539 /** 540 * __remove_pages() - remove sections of pages from a zone 541 * @zone: zone from which pages need to be removed 542 * @phys_start_pfn: starting pageframe (must be aligned to start of a section) 543 * @nr_pages: number of pages to remove (must be multiple of section size) 544 * @altmap: alternative device page map or %NULL if default memmap is used 545 * 546 * Generic helper function to remove section mappings and sysfs entries 547 * for the section of the memory we are removing. Caller needs to make 548 * sure that pages are marked reserved and zones are adjust properly by 549 * calling offline_pages(). 550 */ 551 int __remove_pages(struct zone *zone, unsigned long phys_start_pfn, 552 unsigned long nr_pages, struct vmem_altmap *altmap) 553 { 554 unsigned long i; 555 unsigned long map_offset = 0; 556 int sections_to_remove, ret = 0; 557 558 /* In the ZONE_DEVICE case device driver owns the memory region */ 559 if (is_dev_zone(zone)) { 560 if (altmap) 561 map_offset = vmem_altmap_offset(altmap); 562 } else { 563 resource_size_t start, size; 564 565 start = phys_start_pfn << PAGE_SHIFT; 566 size = nr_pages * PAGE_SIZE; 567 568 ret = release_mem_region_adjustable(&iomem_resource, start, 569 size); 570 if (ret) { 571 resource_size_t endres = start + size - 1; 572 573 pr_warn("Unable to release resource <%pa-%pa> (%d)\n", 574 &start, &endres, ret); 575 } 576 } 577 578 clear_zone_contiguous(zone); 579 580 /* 581 * We can only remove entire sections 582 */ 583 BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK); 584 BUG_ON(nr_pages % PAGES_PER_SECTION); 585 586 sections_to_remove = nr_pages / PAGES_PER_SECTION; 587 for (i = 0; i < sections_to_remove; i++) { 588 unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION; 589 590 cond_resched(); 591 ret = __remove_section(zone, __pfn_to_section(pfn), map_offset, 592 altmap); 593 map_offset = 0; 594 if (ret) 595 break; 596 } 597 598 set_zone_contiguous(zone); 599 600 return ret; 601 } 602 #endif /* CONFIG_MEMORY_HOTREMOVE */ 603 604 int set_online_page_callback(online_page_callback_t callback) 605 { 606 int rc = -EINVAL; 607 608 get_online_mems(); 609 mutex_lock(&online_page_callback_lock); 610 611 if (online_page_callback == generic_online_page) { 612 online_page_callback = callback; 613 rc = 0; 614 } 615 616 mutex_unlock(&online_page_callback_lock); 617 put_online_mems(); 618 619 return rc; 620 } 621 EXPORT_SYMBOL_GPL(set_online_page_callback); 622 623 int restore_online_page_callback(online_page_callback_t callback) 624 { 625 int rc = -EINVAL; 626 627 get_online_mems(); 628 mutex_lock(&online_page_callback_lock); 629 630 if (online_page_callback == callback) { 631 online_page_callback = generic_online_page; 632 rc = 0; 633 } 634 635 mutex_unlock(&online_page_callback_lock); 636 put_online_mems(); 637 638 return rc; 639 } 640 EXPORT_SYMBOL_GPL(restore_online_page_callback); 641 642 void __online_page_set_limits(struct page *page) 643 { 644 } 645 EXPORT_SYMBOL_GPL(__online_page_set_limits); 646 647 void __online_page_increment_counters(struct page *page) 648 { 649 adjust_managed_page_count(page, 1); 650 } 651 EXPORT_SYMBOL_GPL(__online_page_increment_counters); 652 653 void __online_page_free(struct page *page) 654 { 655 __free_reserved_page(page); 656 } 657 EXPORT_SYMBOL_GPL(__online_page_free); 658 659 static void generic_online_page(struct page *page, unsigned int order) 660 { 661 kernel_map_pages(page, 1 << order, 1); 662 __free_pages_core(page, order); 663 totalram_pages_add(1UL << order); 664 #ifdef CONFIG_HIGHMEM 665 if (PageHighMem(page)) 666 totalhigh_pages_add(1UL << order); 667 #endif 668 } 669 670 static int online_pages_blocks(unsigned long start, unsigned long nr_pages) 671 { 672 unsigned long end = start + nr_pages; 673 int order, onlined_pages = 0; 674 675 while (start < end) { 676 order = min(MAX_ORDER - 1, 677 get_order(PFN_PHYS(end) - PFN_PHYS(start))); 678 (*online_page_callback)(pfn_to_page(start), order); 679 680 onlined_pages += (1UL << order); 681 start += (1UL << order); 682 } 683 return onlined_pages; 684 } 685 686 static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages, 687 void *arg) 688 { 689 unsigned long onlined_pages = *(unsigned long *)arg; 690 691 if (PageReserved(pfn_to_page(start_pfn))) 692 onlined_pages += online_pages_blocks(start_pfn, nr_pages); 693 694 online_mem_sections(start_pfn, start_pfn + nr_pages); 695 696 *(unsigned long *)arg = onlined_pages; 697 return 0; 698 } 699 700 /* check which state of node_states will be changed when online memory */ 701 static void node_states_check_changes_online(unsigned long nr_pages, 702 struct zone *zone, struct memory_notify *arg) 703 { 704 int nid = zone_to_nid(zone); 705 706 arg->status_change_nid = NUMA_NO_NODE; 707 arg->status_change_nid_normal = NUMA_NO_NODE; 708 arg->status_change_nid_high = NUMA_NO_NODE; 709 710 if (!node_state(nid, N_MEMORY)) 711 arg->status_change_nid = nid; 712 if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY)) 713 arg->status_change_nid_normal = nid; 714 #ifdef CONFIG_HIGHMEM 715 if (zone_idx(zone) <= N_HIGH_MEMORY && !node_state(nid, N_HIGH_MEMORY)) 716 arg->status_change_nid_high = nid; 717 #endif 718 } 719 720 static void node_states_set_node(int node, struct memory_notify *arg) 721 { 722 if (arg->status_change_nid_normal >= 0) 723 node_set_state(node, N_NORMAL_MEMORY); 724 725 if (arg->status_change_nid_high >= 0) 726 node_set_state(node, N_HIGH_MEMORY); 727 728 if (arg->status_change_nid >= 0) 729 node_set_state(node, N_MEMORY); 730 } 731 732 static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn, 733 unsigned long nr_pages) 734 { 735 unsigned long old_end_pfn = zone_end_pfn(zone); 736 737 if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn) 738 zone->zone_start_pfn = start_pfn; 739 740 zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn; 741 } 742 743 static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn, 744 unsigned long nr_pages) 745 { 746 unsigned long old_end_pfn = pgdat_end_pfn(pgdat); 747 748 if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn) 749 pgdat->node_start_pfn = start_pfn; 750 751 pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn; 752 } 753 754 void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn, 755 unsigned long nr_pages, struct vmem_altmap *altmap) 756 { 757 struct pglist_data *pgdat = zone->zone_pgdat; 758 int nid = pgdat->node_id; 759 unsigned long flags; 760 761 clear_zone_contiguous(zone); 762 763 /* TODO Huh pgdat is irqsave while zone is not. It used to be like that before */ 764 pgdat_resize_lock(pgdat, &flags); 765 zone_span_writelock(zone); 766 if (zone_is_empty(zone)) 767 init_currently_empty_zone(zone, start_pfn, nr_pages); 768 resize_zone_range(zone, start_pfn, nr_pages); 769 zone_span_writeunlock(zone); 770 resize_pgdat_range(pgdat, start_pfn, nr_pages); 771 pgdat_resize_unlock(pgdat, &flags); 772 773 /* 774 * TODO now we have a visible range of pages which are not associated 775 * with their zone properly. Not nice but set_pfnblock_flags_mask 776 * expects the zone spans the pfn range. All the pages in the range 777 * are reserved so nobody should be touching them so we should be safe 778 */ 779 memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn, 780 MEMMAP_HOTPLUG, altmap); 781 782 set_zone_contiguous(zone); 783 } 784 785 /* 786 * Returns a default kernel memory zone for the given pfn range. 787 * If no kernel zone covers this pfn range it will automatically go 788 * to the ZONE_NORMAL. 789 */ 790 static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn, 791 unsigned long nr_pages) 792 { 793 struct pglist_data *pgdat = NODE_DATA(nid); 794 int zid; 795 796 for (zid = 0; zid <= ZONE_NORMAL; zid++) { 797 struct zone *zone = &pgdat->node_zones[zid]; 798 799 if (zone_intersects(zone, start_pfn, nr_pages)) 800 return zone; 801 } 802 803 return &pgdat->node_zones[ZONE_NORMAL]; 804 } 805 806 static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn, 807 unsigned long nr_pages) 808 { 809 struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn, 810 nr_pages); 811 struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; 812 bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages); 813 bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages); 814 815 /* 816 * We inherit the existing zone in a simple case where zones do not 817 * overlap in the given range 818 */ 819 if (in_kernel ^ in_movable) 820 return (in_kernel) ? kernel_zone : movable_zone; 821 822 /* 823 * If the range doesn't belong to any zone or two zones overlap in the 824 * given range then we use movable zone only if movable_node is 825 * enabled because we always online to a kernel zone by default. 826 */ 827 return movable_node_enabled ? movable_zone : kernel_zone; 828 } 829 830 struct zone * zone_for_pfn_range(int online_type, int nid, unsigned start_pfn, 831 unsigned long nr_pages) 832 { 833 if (online_type == MMOP_ONLINE_KERNEL) 834 return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages); 835 836 if (online_type == MMOP_ONLINE_MOVABLE) 837 return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE]; 838 839 return default_zone_for_pfn(nid, start_pfn, nr_pages); 840 } 841 842 /* 843 * Associates the given pfn range with the given node and the zone appropriate 844 * for the given online type. 845 */ 846 static struct zone * __meminit move_pfn_range(int online_type, int nid, 847 unsigned long start_pfn, unsigned long nr_pages) 848 { 849 struct zone *zone; 850 851 zone = zone_for_pfn_range(online_type, nid, start_pfn, nr_pages); 852 move_pfn_range_to_zone(zone, start_pfn, nr_pages, NULL); 853 return zone; 854 } 855 856 int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type) 857 { 858 unsigned long flags; 859 unsigned long onlined_pages = 0; 860 struct zone *zone; 861 int need_zonelists_rebuild = 0; 862 int nid; 863 int ret; 864 struct memory_notify arg; 865 struct memory_block *mem; 866 867 mem_hotplug_begin(); 868 869 /* 870 * We can't use pfn_to_nid() because nid might be stored in struct page 871 * which is not yet initialized. Instead, we find nid from memory block. 872 */ 873 mem = find_memory_block(__pfn_to_section(pfn)); 874 nid = mem->nid; 875 876 /* associate pfn range with the zone */ 877 zone = move_pfn_range(online_type, nid, pfn, nr_pages); 878 879 arg.start_pfn = pfn; 880 arg.nr_pages = nr_pages; 881 node_states_check_changes_online(nr_pages, zone, &arg); 882 883 ret = memory_notify(MEM_GOING_ONLINE, &arg); 884 ret = notifier_to_errno(ret); 885 if (ret) 886 goto failed_addition; 887 888 /* 889 * If this zone is not populated, then it is not in zonelist. 890 * This means the page allocator ignores this zone. 891 * So, zonelist must be updated after online. 892 */ 893 if (!populated_zone(zone)) { 894 need_zonelists_rebuild = 1; 895 setup_zone_pageset(zone); 896 } 897 898 ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages, 899 online_pages_range); 900 if (ret) { 901 if (need_zonelists_rebuild) 902 zone_pcp_reset(zone); 903 goto failed_addition; 904 } 905 906 zone->present_pages += onlined_pages; 907 908 pgdat_resize_lock(zone->zone_pgdat, &flags); 909 zone->zone_pgdat->node_present_pages += onlined_pages; 910 pgdat_resize_unlock(zone->zone_pgdat, &flags); 911 912 if (onlined_pages) { 913 node_states_set_node(nid, &arg); 914 if (need_zonelists_rebuild) 915 build_all_zonelists(NULL); 916 else 917 zone_pcp_update(zone); 918 } 919 920 init_per_zone_wmark_min(); 921 922 if (onlined_pages) { 923 kswapd_run(nid); 924 kcompactd_run(nid); 925 } 926 927 vm_total_pages = nr_free_pagecache_pages(); 928 929 writeback_set_ratelimit(); 930 931 if (onlined_pages) 932 memory_notify(MEM_ONLINE, &arg); 933 mem_hotplug_done(); 934 return 0; 935 936 failed_addition: 937 pr_debug("online_pages [mem %#010llx-%#010llx] failed\n", 938 (unsigned long long) pfn << PAGE_SHIFT, 939 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1); 940 memory_notify(MEM_CANCEL_ONLINE, &arg); 941 mem_hotplug_done(); 942 return ret; 943 } 944 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */ 945 946 static void reset_node_present_pages(pg_data_t *pgdat) 947 { 948 struct zone *z; 949 950 for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++) 951 z->present_pages = 0; 952 953 pgdat->node_present_pages = 0; 954 } 955 956 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */ 957 static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start) 958 { 959 struct pglist_data *pgdat; 960 unsigned long start_pfn = PFN_DOWN(start); 961 962 pgdat = NODE_DATA(nid); 963 if (!pgdat) { 964 pgdat = arch_alloc_nodedata(nid); 965 if (!pgdat) 966 return NULL; 967 968 arch_refresh_nodedata(nid, pgdat); 969 } else { 970 /* 971 * Reset the nr_zones, order and classzone_idx before reuse. 972 * Note that kswapd will init kswapd_classzone_idx properly 973 * when it starts in the near future. 974 */ 975 pgdat->nr_zones = 0; 976 pgdat->kswapd_order = 0; 977 pgdat->kswapd_classzone_idx = 0; 978 } 979 980 /* we can use NODE_DATA(nid) from here */ 981 982 pgdat->node_id = nid; 983 pgdat->node_start_pfn = start_pfn; 984 985 /* init node's zones as empty zones, we don't have any present pages.*/ 986 free_area_init_core_hotplug(nid); 987 pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); 988 989 /* 990 * The node we allocated has no zone fallback lists. For avoiding 991 * to access not-initialized zonelist, build here. 992 */ 993 build_all_zonelists(pgdat); 994 995 /* 996 * When memory is hot-added, all the memory is in offline state. So 997 * clear all zones' present_pages because they will be updated in 998 * online_pages() and offline_pages(). 999 */ 1000 reset_node_managed_pages(pgdat); 1001 reset_node_present_pages(pgdat); 1002 1003 return pgdat; 1004 } 1005 1006 static void rollback_node_hotadd(int nid) 1007 { 1008 pg_data_t *pgdat = NODE_DATA(nid); 1009 1010 arch_refresh_nodedata(nid, NULL); 1011 free_percpu(pgdat->per_cpu_nodestats); 1012 arch_free_nodedata(pgdat); 1013 return; 1014 } 1015 1016 1017 /** 1018 * try_online_node - online a node if offlined 1019 * @nid: the node ID 1020 * @start: start addr of the node 1021 * @set_node_online: Whether we want to online the node 1022 * called by cpu_up() to online a node without onlined memory. 1023 * 1024 * Returns: 1025 * 1 -> a new node has been allocated 1026 * 0 -> the node is already online 1027 * -ENOMEM -> the node could not be allocated 1028 */ 1029 static int __try_online_node(int nid, u64 start, bool set_node_online) 1030 { 1031 pg_data_t *pgdat; 1032 int ret = 1; 1033 1034 if (node_online(nid)) 1035 return 0; 1036 1037 pgdat = hotadd_new_pgdat(nid, start); 1038 if (!pgdat) { 1039 pr_err("Cannot online node %d due to NULL pgdat\n", nid); 1040 ret = -ENOMEM; 1041 goto out; 1042 } 1043 1044 if (set_node_online) { 1045 node_set_online(nid); 1046 ret = register_one_node(nid); 1047 BUG_ON(ret); 1048 } 1049 out: 1050 return ret; 1051 } 1052 1053 /* 1054 * Users of this function always want to online/register the node 1055 */ 1056 int try_online_node(int nid) 1057 { 1058 int ret; 1059 1060 mem_hotplug_begin(); 1061 ret = __try_online_node(nid, 0, true); 1062 mem_hotplug_done(); 1063 return ret; 1064 } 1065 1066 static int check_hotplug_memory_range(u64 start, u64 size) 1067 { 1068 unsigned long block_sz = memory_block_size_bytes(); 1069 u64 block_nr_pages = block_sz >> PAGE_SHIFT; 1070 u64 nr_pages = size >> PAGE_SHIFT; 1071 u64 start_pfn = PFN_DOWN(start); 1072 1073 /* memory range must be block size aligned */ 1074 if (!nr_pages || !IS_ALIGNED(start_pfn, block_nr_pages) || 1075 !IS_ALIGNED(nr_pages, block_nr_pages)) { 1076 pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx", 1077 block_sz, start, size); 1078 return -EINVAL; 1079 } 1080 1081 return 0; 1082 } 1083 1084 static int online_memory_block(struct memory_block *mem, void *arg) 1085 { 1086 return device_online(&mem->dev); 1087 } 1088 1089 /* 1090 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1091 * and online/offline operations (triggered e.g. by sysfs). 1092 * 1093 * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG 1094 */ 1095 int __ref add_memory_resource(int nid, struct resource *res) 1096 { 1097 u64 start, size; 1098 bool new_node = false; 1099 int ret; 1100 1101 start = res->start; 1102 size = resource_size(res); 1103 1104 ret = check_hotplug_memory_range(start, size); 1105 if (ret) 1106 return ret; 1107 1108 mem_hotplug_begin(); 1109 1110 /* 1111 * Add new range to memblock so that when hotadd_new_pgdat() is called 1112 * to allocate new pgdat, get_pfn_range_for_nid() will be able to find 1113 * this new range and calculate total pages correctly. The range will 1114 * be removed at hot-remove time. 1115 */ 1116 memblock_add_node(start, size, nid); 1117 1118 ret = __try_online_node(nid, start, false); 1119 if (ret < 0) 1120 goto error; 1121 new_node = ret; 1122 1123 /* call arch's memory hotadd */ 1124 ret = arch_add_memory(nid, start, size, NULL, true); 1125 if (ret < 0) 1126 goto error; 1127 1128 if (new_node) { 1129 /* If sysfs file of new node can't be created, cpu on the node 1130 * can't be hot-added. There is no rollback way now. 1131 * So, check by BUG_ON() to catch it reluctantly.. 1132 * We online node here. We can't roll back from here. 1133 */ 1134 node_set_online(nid); 1135 ret = __register_one_node(nid); 1136 BUG_ON(ret); 1137 } 1138 1139 /* link memory sections under this node.*/ 1140 ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1)); 1141 BUG_ON(ret); 1142 1143 /* create new memmap entry */ 1144 firmware_map_add_hotplug(start, start + size, "System RAM"); 1145 1146 /* device_online() will take the lock when calling online_pages() */ 1147 mem_hotplug_done(); 1148 1149 /* online pages if requested */ 1150 if (memhp_auto_online) 1151 walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), 1152 NULL, online_memory_block); 1153 1154 return ret; 1155 error: 1156 /* rollback pgdat allocation and others */ 1157 if (new_node) 1158 rollback_node_hotadd(nid); 1159 memblock_remove(start, size); 1160 mem_hotplug_done(); 1161 return ret; 1162 } 1163 1164 /* requires device_hotplug_lock, see add_memory_resource() */ 1165 int __ref __add_memory(int nid, u64 start, u64 size) 1166 { 1167 struct resource *res; 1168 int ret; 1169 1170 res = register_memory_resource(start, size); 1171 if (IS_ERR(res)) 1172 return PTR_ERR(res); 1173 1174 ret = add_memory_resource(nid, res); 1175 if (ret < 0) 1176 release_memory_resource(res); 1177 return ret; 1178 } 1179 1180 int add_memory(int nid, u64 start, u64 size) 1181 { 1182 int rc; 1183 1184 lock_device_hotplug(); 1185 rc = __add_memory(nid, start, size); 1186 unlock_device_hotplug(); 1187 1188 return rc; 1189 } 1190 EXPORT_SYMBOL_GPL(add_memory); 1191 1192 #ifdef CONFIG_MEMORY_HOTREMOVE 1193 /* 1194 * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy 1195 * set and the size of the free page is given by page_order(). Using this, 1196 * the function determines if the pageblock contains only free pages. 1197 * Due to buddy contraints, a free page at least the size of a pageblock will 1198 * be located at the start of the pageblock 1199 */ 1200 static inline int pageblock_free(struct page *page) 1201 { 1202 return PageBuddy(page) && page_order(page) >= pageblock_order; 1203 } 1204 1205 /* Return the pfn of the start of the next active pageblock after a given pfn */ 1206 static unsigned long next_active_pageblock(unsigned long pfn) 1207 { 1208 struct page *page = pfn_to_page(pfn); 1209 1210 /* Ensure the starting page is pageblock-aligned */ 1211 BUG_ON(pfn & (pageblock_nr_pages - 1)); 1212 1213 /* If the entire pageblock is free, move to the end of free page */ 1214 if (pageblock_free(page)) { 1215 int order; 1216 /* be careful. we don't have locks, page_order can be changed.*/ 1217 order = page_order(page); 1218 if ((order < MAX_ORDER) && (order >= pageblock_order)) 1219 return pfn + (1 << order); 1220 } 1221 1222 return pfn + pageblock_nr_pages; 1223 } 1224 1225 static bool is_pageblock_removable_nolock(unsigned long pfn) 1226 { 1227 struct page *page = pfn_to_page(pfn); 1228 struct zone *zone; 1229 1230 /* 1231 * We have to be careful here because we are iterating over memory 1232 * sections which are not zone aware so we might end up outside of 1233 * the zone but still within the section. 1234 * We have to take care about the node as well. If the node is offline 1235 * its NODE_DATA will be NULL - see page_zone. 1236 */ 1237 if (!node_online(page_to_nid(page))) 1238 return false; 1239 1240 zone = page_zone(page); 1241 pfn = page_to_pfn(page); 1242 if (!zone_spans_pfn(zone, pfn)) 1243 return false; 1244 1245 return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE, SKIP_HWPOISON); 1246 } 1247 1248 /* Checks if this range of memory is likely to be hot-removable. */ 1249 bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages) 1250 { 1251 unsigned long end_pfn, pfn; 1252 1253 end_pfn = min(start_pfn + nr_pages, 1254 zone_end_pfn(page_zone(pfn_to_page(start_pfn)))); 1255 1256 /* Check the starting page of each pageblock within the range */ 1257 for (pfn = start_pfn; pfn < end_pfn; pfn = next_active_pageblock(pfn)) { 1258 if (!is_pageblock_removable_nolock(pfn)) 1259 return false; 1260 cond_resched(); 1261 } 1262 1263 /* All pageblocks in the memory block are likely to be hot-removable */ 1264 return true; 1265 } 1266 1267 /* 1268 * Confirm all pages in a range [start, end) belong to the same zone. 1269 * When true, return its valid [start, end). 1270 */ 1271 int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn, 1272 unsigned long *valid_start, unsigned long *valid_end) 1273 { 1274 unsigned long pfn, sec_end_pfn; 1275 unsigned long start, end; 1276 struct zone *zone = NULL; 1277 struct page *page; 1278 int i; 1279 for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1); 1280 pfn < end_pfn; 1281 pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) { 1282 /* Make sure the memory section is present first */ 1283 if (!present_section_nr(pfn_to_section_nr(pfn))) 1284 continue; 1285 for (; pfn < sec_end_pfn && pfn < end_pfn; 1286 pfn += MAX_ORDER_NR_PAGES) { 1287 i = 0; 1288 /* This is just a CONFIG_HOLES_IN_ZONE check.*/ 1289 while ((i < MAX_ORDER_NR_PAGES) && 1290 !pfn_valid_within(pfn + i)) 1291 i++; 1292 if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn) 1293 continue; 1294 /* Check if we got outside of the zone */ 1295 if (zone && !zone_spans_pfn(zone, pfn + i)) 1296 return 0; 1297 page = pfn_to_page(pfn + i); 1298 if (zone && page_zone(page) != zone) 1299 return 0; 1300 if (!zone) 1301 start = pfn + i; 1302 zone = page_zone(page); 1303 end = pfn + MAX_ORDER_NR_PAGES; 1304 } 1305 } 1306 1307 if (zone) { 1308 *valid_start = start; 1309 *valid_end = min(end, end_pfn); 1310 return 1; 1311 } else { 1312 return 0; 1313 } 1314 } 1315 1316 /* 1317 * Scan pfn range [start,end) to find movable/migratable pages (LRU pages, 1318 * non-lru movable pages and hugepages). We scan pfn because it's much 1319 * easier than scanning over linked list. This function returns the pfn 1320 * of the first found movable page if it's found, otherwise 0. 1321 */ 1322 static unsigned long scan_movable_pages(unsigned long start, unsigned long end) 1323 { 1324 unsigned long pfn; 1325 1326 for (pfn = start; pfn < end; pfn++) { 1327 struct page *page, *head; 1328 unsigned long skip; 1329 1330 if (!pfn_valid(pfn)) 1331 continue; 1332 page = pfn_to_page(pfn); 1333 if (PageLRU(page)) 1334 return pfn; 1335 if (__PageMovable(page)) 1336 return pfn; 1337 1338 if (!PageHuge(page)) 1339 continue; 1340 head = compound_head(page); 1341 if (hugepage_migration_supported(page_hstate(head)) && 1342 page_huge_active(head)) 1343 return pfn; 1344 skip = (1 << compound_order(head)) - (page - head); 1345 pfn += skip - 1; 1346 } 1347 return 0; 1348 } 1349 1350 static struct page *new_node_page(struct page *page, unsigned long private) 1351 { 1352 int nid = page_to_nid(page); 1353 nodemask_t nmask = node_states[N_MEMORY]; 1354 1355 /* 1356 * try to allocate from a different node but reuse this node if there 1357 * are no other online nodes to be used (e.g. we are offlining a part 1358 * of the only existing node) 1359 */ 1360 node_clear(nid, nmask); 1361 if (nodes_empty(nmask)) 1362 node_set(nid, nmask); 1363 1364 return new_page_nodemask(page, nid, &nmask); 1365 } 1366 1367 static int 1368 do_migrate_range(unsigned long start_pfn, unsigned long end_pfn) 1369 { 1370 unsigned long pfn; 1371 struct page *page; 1372 int ret = 0; 1373 LIST_HEAD(source); 1374 1375 for (pfn = start_pfn; pfn < end_pfn; pfn++) { 1376 if (!pfn_valid(pfn)) 1377 continue; 1378 page = pfn_to_page(pfn); 1379 1380 if (PageHuge(page)) { 1381 struct page *head = compound_head(page); 1382 if (compound_order(head) > PFN_SECTION_SHIFT) { 1383 ret = -EBUSY; 1384 break; 1385 } 1386 pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1; 1387 isolate_huge_page(head, &source); 1388 continue; 1389 } else if (PageTransHuge(page)) 1390 pfn = page_to_pfn(compound_head(page)) 1391 + hpage_nr_pages(page) - 1; 1392 1393 /* 1394 * HWPoison pages have elevated reference counts so the migration would 1395 * fail on them. It also doesn't make any sense to migrate them in the 1396 * first place. Still try to unmap such a page in case it is still mapped 1397 * (e.g. current hwpoison implementation doesn't unmap KSM pages but keep 1398 * the unmap as the catch all safety net). 1399 */ 1400 if (PageHWPoison(page)) { 1401 if (WARN_ON(PageLRU(page))) 1402 isolate_lru_page(page); 1403 if (page_mapped(page)) 1404 try_to_unmap(page, TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS); 1405 continue; 1406 } 1407 1408 if (!get_page_unless_zero(page)) 1409 continue; 1410 /* 1411 * We can skip free pages. And we can deal with pages on 1412 * LRU and non-lru movable pages. 1413 */ 1414 if (PageLRU(page)) 1415 ret = isolate_lru_page(page); 1416 else 1417 ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE); 1418 if (!ret) { /* Success */ 1419 list_add_tail(&page->lru, &source); 1420 if (!__PageMovable(page)) 1421 inc_node_page_state(page, NR_ISOLATED_ANON + 1422 page_is_file_cache(page)); 1423 1424 } else { 1425 pr_warn("failed to isolate pfn %lx\n", pfn); 1426 dump_page(page, "isolation failed"); 1427 } 1428 put_page(page); 1429 } 1430 if (!list_empty(&source)) { 1431 /* Allocate a new page from the nearest neighbor node */ 1432 ret = migrate_pages(&source, new_node_page, NULL, 0, 1433 MIGRATE_SYNC, MR_MEMORY_HOTPLUG); 1434 if (ret) { 1435 list_for_each_entry(page, &source, lru) { 1436 pr_warn("migrating pfn %lx failed ret:%d ", 1437 page_to_pfn(page), ret); 1438 dump_page(page, "migration failure"); 1439 } 1440 putback_movable_pages(&source); 1441 } 1442 } 1443 1444 return ret; 1445 } 1446 1447 /* 1448 * remove from free_area[] and mark all as Reserved. 1449 */ 1450 static int 1451 offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages, 1452 void *data) 1453 { 1454 __offline_isolated_pages(start, start + nr_pages); 1455 return 0; 1456 } 1457 1458 static void 1459 offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) 1460 { 1461 walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL, 1462 offline_isolated_pages_cb); 1463 } 1464 1465 /* 1466 * Check all pages in range, recoreded as memory resource, are isolated. 1467 */ 1468 static int 1469 check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages, 1470 void *data) 1471 { 1472 int ret; 1473 long offlined = *(long *)data; 1474 ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true); 1475 offlined = nr_pages; 1476 if (!ret) 1477 *(long *)data += offlined; 1478 return ret; 1479 } 1480 1481 static long 1482 check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) 1483 { 1484 long offlined = 0; 1485 int ret; 1486 1487 ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined, 1488 check_pages_isolated_cb); 1489 if (ret < 0) 1490 offlined = (long)ret; 1491 return offlined; 1492 } 1493 1494 static int __init cmdline_parse_movable_node(char *p) 1495 { 1496 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 1497 movable_node_enabled = true; 1498 #else 1499 pr_warn("movable_node parameter depends on CONFIG_HAVE_MEMBLOCK_NODE_MAP to work properly\n"); 1500 #endif 1501 return 0; 1502 } 1503 early_param("movable_node", cmdline_parse_movable_node); 1504 1505 /* check which state of node_states will be changed when offline memory */ 1506 static void node_states_check_changes_offline(unsigned long nr_pages, 1507 struct zone *zone, struct memory_notify *arg) 1508 { 1509 struct pglist_data *pgdat = zone->zone_pgdat; 1510 unsigned long present_pages = 0; 1511 enum zone_type zt; 1512 1513 arg->status_change_nid = NUMA_NO_NODE; 1514 arg->status_change_nid_normal = NUMA_NO_NODE; 1515 arg->status_change_nid_high = NUMA_NO_NODE; 1516 1517 /* 1518 * Check whether node_states[N_NORMAL_MEMORY] will be changed. 1519 * If the memory to be offline is within the range 1520 * [0..ZONE_NORMAL], and it is the last present memory there, 1521 * the zones in that range will become empty after the offlining, 1522 * thus we can determine that we need to clear the node from 1523 * node_states[N_NORMAL_MEMORY]. 1524 */ 1525 for (zt = 0; zt <= ZONE_NORMAL; zt++) 1526 present_pages += pgdat->node_zones[zt].present_pages; 1527 if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages) 1528 arg->status_change_nid_normal = zone_to_nid(zone); 1529 1530 #ifdef CONFIG_HIGHMEM 1531 /* 1532 * node_states[N_HIGH_MEMORY] contains nodes which 1533 * have normal memory or high memory. 1534 * Here we add the present_pages belonging to ZONE_HIGHMEM. 1535 * If the zone is within the range of [0..ZONE_HIGHMEM), and 1536 * we determine that the zones in that range become empty, 1537 * we need to clear the node for N_HIGH_MEMORY. 1538 */ 1539 present_pages += pgdat->node_zones[ZONE_HIGHMEM].present_pages; 1540 if (zone_idx(zone) <= ZONE_HIGHMEM && nr_pages >= present_pages) 1541 arg->status_change_nid_high = zone_to_nid(zone); 1542 #endif 1543 1544 /* 1545 * We have accounted the pages from [0..ZONE_NORMAL), and 1546 * in case of CONFIG_HIGHMEM the pages from ZONE_HIGHMEM 1547 * as well. 1548 * Here we count the possible pages from ZONE_MOVABLE. 1549 * If after having accounted all the pages, we see that the nr_pages 1550 * to be offlined is over or equal to the accounted pages, 1551 * we know that the node will become empty, and so, we can clear 1552 * it for N_MEMORY as well. 1553 */ 1554 present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages; 1555 1556 if (nr_pages >= present_pages) 1557 arg->status_change_nid = zone_to_nid(zone); 1558 } 1559 1560 static void node_states_clear_node(int node, struct memory_notify *arg) 1561 { 1562 if (arg->status_change_nid_normal >= 0) 1563 node_clear_state(node, N_NORMAL_MEMORY); 1564 1565 if (arg->status_change_nid_high >= 0) 1566 node_clear_state(node, N_HIGH_MEMORY); 1567 1568 if (arg->status_change_nid >= 0) 1569 node_clear_state(node, N_MEMORY); 1570 } 1571 1572 static int __ref __offline_pages(unsigned long start_pfn, 1573 unsigned long end_pfn) 1574 { 1575 unsigned long pfn, nr_pages; 1576 long offlined_pages; 1577 int ret, node; 1578 unsigned long flags; 1579 unsigned long valid_start, valid_end; 1580 struct zone *zone; 1581 struct memory_notify arg; 1582 char *reason; 1583 1584 mem_hotplug_begin(); 1585 1586 /* This makes hotplug much easier...and readable. 1587 we assume this for now. .*/ 1588 if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start, 1589 &valid_end)) { 1590 ret = -EINVAL; 1591 reason = "multizone range"; 1592 goto failed_removal; 1593 } 1594 1595 zone = page_zone(pfn_to_page(valid_start)); 1596 node = zone_to_nid(zone); 1597 nr_pages = end_pfn - start_pfn; 1598 1599 /* set above range as isolated */ 1600 ret = start_isolate_page_range(start_pfn, end_pfn, 1601 MIGRATE_MOVABLE, 1602 SKIP_HWPOISON | REPORT_FAILURE); 1603 if (ret) { 1604 reason = "failure to isolate range"; 1605 goto failed_removal; 1606 } 1607 1608 arg.start_pfn = start_pfn; 1609 arg.nr_pages = nr_pages; 1610 node_states_check_changes_offline(nr_pages, zone, &arg); 1611 1612 ret = memory_notify(MEM_GOING_OFFLINE, &arg); 1613 ret = notifier_to_errno(ret); 1614 if (ret) { 1615 reason = "notifier failure"; 1616 goto failed_removal_isolated; 1617 } 1618 1619 do { 1620 for (pfn = start_pfn; pfn;) { 1621 if (signal_pending(current)) { 1622 ret = -EINTR; 1623 reason = "signal backoff"; 1624 goto failed_removal_isolated; 1625 } 1626 1627 cond_resched(); 1628 lru_add_drain_all(); 1629 1630 pfn = scan_movable_pages(pfn, end_pfn); 1631 if (pfn) { 1632 /* 1633 * TODO: fatal migration failures should bail 1634 * out 1635 */ 1636 do_migrate_range(pfn, end_pfn); 1637 } 1638 } 1639 1640 /* 1641 * Dissolve free hugepages in the memory block before doing 1642 * offlining actually in order to make hugetlbfs's object 1643 * counting consistent. 1644 */ 1645 ret = dissolve_free_huge_pages(start_pfn, end_pfn); 1646 if (ret) { 1647 reason = "failure to dissolve huge pages"; 1648 goto failed_removal_isolated; 1649 } 1650 /* check again */ 1651 offlined_pages = check_pages_isolated(start_pfn, end_pfn); 1652 } while (offlined_pages < 0); 1653 1654 pr_info("Offlined Pages %ld\n", offlined_pages); 1655 /* Ok, all of our target is isolated. 1656 We cannot do rollback at this point. */ 1657 offline_isolated_pages(start_pfn, end_pfn); 1658 /* reset pagetype flags and makes migrate type to be MOVABLE */ 1659 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1660 /* removal success */ 1661 adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages); 1662 zone->present_pages -= offlined_pages; 1663 1664 pgdat_resize_lock(zone->zone_pgdat, &flags); 1665 zone->zone_pgdat->node_present_pages -= offlined_pages; 1666 pgdat_resize_unlock(zone->zone_pgdat, &flags); 1667 1668 init_per_zone_wmark_min(); 1669 1670 if (!populated_zone(zone)) { 1671 zone_pcp_reset(zone); 1672 build_all_zonelists(NULL); 1673 } else 1674 zone_pcp_update(zone); 1675 1676 node_states_clear_node(node, &arg); 1677 if (arg.status_change_nid >= 0) { 1678 kswapd_stop(node); 1679 kcompactd_stop(node); 1680 } 1681 1682 vm_total_pages = nr_free_pagecache_pages(); 1683 writeback_set_ratelimit(); 1684 1685 memory_notify(MEM_OFFLINE, &arg); 1686 mem_hotplug_done(); 1687 return 0; 1688 1689 failed_removal_isolated: 1690 undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE); 1691 failed_removal: 1692 pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n", 1693 (unsigned long long) start_pfn << PAGE_SHIFT, 1694 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1, 1695 reason); 1696 memory_notify(MEM_CANCEL_OFFLINE, &arg); 1697 /* pushback to free area */ 1698 mem_hotplug_done(); 1699 return ret; 1700 } 1701 1702 int offline_pages(unsigned long start_pfn, unsigned long nr_pages) 1703 { 1704 return __offline_pages(start_pfn, start_pfn + nr_pages); 1705 } 1706 #endif /* CONFIG_MEMORY_HOTREMOVE */ 1707 1708 /** 1709 * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn) 1710 * @start_pfn: start pfn of the memory range 1711 * @end_pfn: end pfn of the memory range 1712 * @arg: argument passed to func 1713 * @func: callback for each memory section walked 1714 * 1715 * This function walks through all present mem sections in range 1716 * [start_pfn, end_pfn) and call func on each mem section. 1717 * 1718 * Returns the return value of func. 1719 */ 1720 int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn, 1721 void *arg, int (*func)(struct memory_block *, void *)) 1722 { 1723 struct memory_block *mem = NULL; 1724 struct mem_section *section; 1725 unsigned long pfn, section_nr; 1726 int ret; 1727 1728 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1729 section_nr = pfn_to_section_nr(pfn); 1730 if (!present_section_nr(section_nr)) 1731 continue; 1732 1733 section = __nr_to_section(section_nr); 1734 /* same memblock? */ 1735 if (mem) 1736 if ((section_nr >= mem->start_section_nr) && 1737 (section_nr <= mem->end_section_nr)) 1738 continue; 1739 1740 mem = find_memory_block_hinted(section, mem); 1741 if (!mem) 1742 continue; 1743 1744 ret = func(mem, arg); 1745 if (ret) { 1746 kobject_put(&mem->dev.kobj); 1747 return ret; 1748 } 1749 } 1750 1751 if (mem) 1752 kobject_put(&mem->dev.kobj); 1753 1754 return 0; 1755 } 1756 1757 #ifdef CONFIG_MEMORY_HOTREMOVE 1758 static int check_memblock_offlined_cb(struct memory_block *mem, void *arg) 1759 { 1760 int ret = !is_memblock_offlined(mem); 1761 1762 if (unlikely(ret)) { 1763 phys_addr_t beginpa, endpa; 1764 1765 beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr)); 1766 endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1; 1767 pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n", 1768 &beginpa, &endpa); 1769 } 1770 1771 return ret; 1772 } 1773 1774 static int check_cpu_on_node(pg_data_t *pgdat) 1775 { 1776 int cpu; 1777 1778 for_each_present_cpu(cpu) { 1779 if (cpu_to_node(cpu) == pgdat->node_id) 1780 /* 1781 * the cpu on this node isn't removed, and we can't 1782 * offline this node. 1783 */ 1784 return -EBUSY; 1785 } 1786 1787 return 0; 1788 } 1789 1790 /** 1791 * try_offline_node 1792 * @nid: the node ID 1793 * 1794 * Offline a node if all memory sections and cpus of the node are removed. 1795 * 1796 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1797 * and online/offline operations before this call. 1798 */ 1799 void try_offline_node(int nid) 1800 { 1801 pg_data_t *pgdat = NODE_DATA(nid); 1802 unsigned long start_pfn = pgdat->node_start_pfn; 1803 unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages; 1804 unsigned long pfn; 1805 1806 for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) { 1807 unsigned long section_nr = pfn_to_section_nr(pfn); 1808 1809 if (!present_section_nr(section_nr)) 1810 continue; 1811 1812 if (pfn_to_nid(pfn) != nid) 1813 continue; 1814 1815 /* 1816 * some memory sections of this node are not removed, and we 1817 * can't offline node now. 1818 */ 1819 return; 1820 } 1821 1822 if (check_cpu_on_node(pgdat)) 1823 return; 1824 1825 /* 1826 * all memory/cpu of this node are removed, we can offline this 1827 * node now. 1828 */ 1829 node_set_offline(nid); 1830 unregister_one_node(nid); 1831 } 1832 EXPORT_SYMBOL(try_offline_node); 1833 1834 /** 1835 * remove_memory 1836 * @nid: the node ID 1837 * @start: physical address of the region to remove 1838 * @size: size of the region to remove 1839 * 1840 * NOTE: The caller must call lock_device_hotplug() to serialize hotplug 1841 * and online/offline operations before this call, as required by 1842 * try_offline_node(). 1843 */ 1844 void __ref __remove_memory(int nid, u64 start, u64 size) 1845 { 1846 int ret; 1847 1848 BUG_ON(check_hotplug_memory_range(start, size)); 1849 1850 mem_hotplug_begin(); 1851 1852 /* 1853 * All memory blocks must be offlined before removing memory. Check 1854 * whether all memory blocks in question are offline and trigger a BUG() 1855 * if this is not the case. 1856 */ 1857 ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL, 1858 check_memblock_offlined_cb); 1859 if (ret) 1860 BUG(); 1861 1862 /* remove memmap entry */ 1863 firmware_map_remove(start, start + size, "System RAM"); 1864 memblock_free(start, size); 1865 memblock_remove(start, size); 1866 1867 arch_remove_memory(nid, start, size, NULL); 1868 1869 try_offline_node(nid); 1870 1871 mem_hotplug_done(); 1872 } 1873 1874 void remove_memory(int nid, u64 start, u64 size) 1875 { 1876 lock_device_hotplug(); 1877 __remove_memory(nid, start, size); 1878 unlock_device_hotplug(); 1879 } 1880 EXPORT_SYMBOL_GPL(remove_memory); 1881 #endif /* CONFIG_MEMORY_HOTREMOVE */ 1882