xref: /openbmc/linux/mm/memory_hotplug.c (revision e620a1e0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/mm/memory_hotplug.c
4  *
5  *  Copyright (C)
6  */
7 
8 #include <linux/stddef.h>
9 #include <linux/mm.h>
10 #include <linux/sched/signal.h>
11 #include <linux/swap.h>
12 #include <linux/interrupt.h>
13 #include <linux/pagemap.h>
14 #include <linux/compiler.h>
15 #include <linux/export.h>
16 #include <linux/pagevec.h>
17 #include <linux/writeback.h>
18 #include <linux/slab.h>
19 #include <linux/sysctl.h>
20 #include <linux/cpu.h>
21 #include <linux/memory.h>
22 #include <linux/memremap.h>
23 #include <linux/memory_hotplug.h>
24 #include <linux/highmem.h>
25 #include <linux/vmalloc.h>
26 #include <linux/ioport.h>
27 #include <linux/delay.h>
28 #include <linux/migrate.h>
29 #include <linux/page-isolation.h>
30 #include <linux/pfn.h>
31 #include <linux/suspend.h>
32 #include <linux/mm_inline.h>
33 #include <linux/firmware-map.h>
34 #include <linux/stop_machine.h>
35 #include <linux/hugetlb.h>
36 #include <linux/memblock.h>
37 #include <linux/compaction.h>
38 #include <linux/rmap.h>
39 
40 #include <asm/tlbflush.h>
41 
42 #include "internal.h"
43 #include "shuffle.h"
44 
45 /*
46  * online_page_callback contains pointer to current page onlining function.
47  * Initially it is generic_online_page(). If it is required it could be
48  * changed by calling set_online_page_callback() for callback registration
49  * and restore_online_page_callback() for generic callback restore.
50  */
51 
52 static void generic_online_page(struct page *page, unsigned int order);
53 
54 static online_page_callback_t online_page_callback = generic_online_page;
55 static DEFINE_MUTEX(online_page_callback_lock);
56 
57 DEFINE_STATIC_PERCPU_RWSEM(mem_hotplug_lock);
58 
59 void get_online_mems(void)
60 {
61 	percpu_down_read(&mem_hotplug_lock);
62 }
63 
64 void put_online_mems(void)
65 {
66 	percpu_up_read(&mem_hotplug_lock);
67 }
68 
69 bool movable_node_enabled = false;
70 
71 #ifndef CONFIG_MEMORY_HOTPLUG_DEFAULT_ONLINE
72 bool memhp_auto_online;
73 #else
74 bool memhp_auto_online = true;
75 #endif
76 EXPORT_SYMBOL_GPL(memhp_auto_online);
77 
78 static int __init setup_memhp_default_state(char *str)
79 {
80 	if (!strcmp(str, "online"))
81 		memhp_auto_online = true;
82 	else if (!strcmp(str, "offline"))
83 		memhp_auto_online = false;
84 
85 	return 1;
86 }
87 __setup("memhp_default_state=", setup_memhp_default_state);
88 
89 void mem_hotplug_begin(void)
90 {
91 	cpus_read_lock();
92 	percpu_down_write(&mem_hotplug_lock);
93 }
94 
95 void mem_hotplug_done(void)
96 {
97 	percpu_up_write(&mem_hotplug_lock);
98 	cpus_read_unlock();
99 }
100 
101 u64 max_mem_size = U64_MAX;
102 
103 /* add this memory to iomem resource */
104 static struct resource *register_memory_resource(u64 start, u64 size)
105 {
106 	struct resource *res;
107 	unsigned long flags =  IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
108 	char *resource_name = "System RAM";
109 
110 	if (start + size > max_mem_size)
111 		return ERR_PTR(-E2BIG);
112 
113 	/*
114 	 * Request ownership of the new memory range.  This might be
115 	 * a child of an existing resource that was present but
116 	 * not marked as busy.
117 	 */
118 	res = __request_region(&iomem_resource, start, size,
119 			       resource_name, flags);
120 
121 	if (!res) {
122 		pr_debug("Unable to reserve System RAM region: %016llx->%016llx\n",
123 				start, start + size);
124 		return ERR_PTR(-EEXIST);
125 	}
126 	return res;
127 }
128 
129 static void release_memory_resource(struct resource *res)
130 {
131 	if (!res)
132 		return;
133 	release_resource(res);
134 	kfree(res);
135 }
136 
137 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
138 void get_page_bootmem(unsigned long info,  struct page *page,
139 		      unsigned long type)
140 {
141 	page->freelist = (void *)type;
142 	SetPagePrivate(page);
143 	set_page_private(page, info);
144 	page_ref_inc(page);
145 }
146 
147 void put_page_bootmem(struct page *page)
148 {
149 	unsigned long type;
150 
151 	type = (unsigned long) page->freelist;
152 	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
153 	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
154 
155 	if (page_ref_dec_return(page) == 1) {
156 		page->freelist = NULL;
157 		ClearPagePrivate(page);
158 		set_page_private(page, 0);
159 		INIT_LIST_HEAD(&page->lru);
160 		free_reserved_page(page);
161 	}
162 }
163 
164 #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
165 #ifndef CONFIG_SPARSEMEM_VMEMMAP
166 static void register_page_bootmem_info_section(unsigned long start_pfn)
167 {
168 	unsigned long mapsize, section_nr, i;
169 	struct mem_section *ms;
170 	struct page *page, *memmap;
171 	struct mem_section_usage *usage;
172 
173 	section_nr = pfn_to_section_nr(start_pfn);
174 	ms = __nr_to_section(section_nr);
175 
176 	/* Get section's memmap address */
177 	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
178 
179 	/*
180 	 * Get page for the memmap's phys address
181 	 * XXX: need more consideration for sparse_vmemmap...
182 	 */
183 	page = virt_to_page(memmap);
184 	mapsize = sizeof(struct page) * PAGES_PER_SECTION;
185 	mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
186 
187 	/* remember memmap's page */
188 	for (i = 0; i < mapsize; i++, page++)
189 		get_page_bootmem(section_nr, page, SECTION_INFO);
190 
191 	usage = ms->usage;
192 	page = virt_to_page(usage);
193 
194 	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
195 
196 	for (i = 0; i < mapsize; i++, page++)
197 		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
198 
199 }
200 #else /* CONFIG_SPARSEMEM_VMEMMAP */
201 static void register_page_bootmem_info_section(unsigned long start_pfn)
202 {
203 	unsigned long mapsize, section_nr, i;
204 	struct mem_section *ms;
205 	struct page *page, *memmap;
206 	struct mem_section_usage *usage;
207 
208 	section_nr = pfn_to_section_nr(start_pfn);
209 	ms = __nr_to_section(section_nr);
210 
211 	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
212 
213 	register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
214 
215 	usage = ms->usage;
216 	page = virt_to_page(usage);
217 
218 	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
219 
220 	for (i = 0; i < mapsize; i++, page++)
221 		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
222 }
223 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
224 
225 void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
226 {
227 	unsigned long i, pfn, end_pfn, nr_pages;
228 	int node = pgdat->node_id;
229 	struct page *page;
230 
231 	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
232 	page = virt_to_page(pgdat);
233 
234 	for (i = 0; i < nr_pages; i++, page++)
235 		get_page_bootmem(node, page, NODE_INFO);
236 
237 	pfn = pgdat->node_start_pfn;
238 	end_pfn = pgdat_end_pfn(pgdat);
239 
240 	/* register section info */
241 	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
242 		/*
243 		 * Some platforms can assign the same pfn to multiple nodes - on
244 		 * node0 as well as nodeN.  To avoid registering a pfn against
245 		 * multiple nodes we check that this pfn does not already
246 		 * reside in some other nodes.
247 		 */
248 		if (pfn_valid(pfn) && (early_pfn_to_nid(pfn) == node))
249 			register_page_bootmem_info_section(pfn);
250 	}
251 }
252 #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
253 
254 static int check_pfn_span(unsigned long pfn, unsigned long nr_pages,
255 		const char *reason)
256 {
257 	/*
258 	 * Disallow all operations smaller than a sub-section and only
259 	 * allow operations smaller than a section for
260 	 * SPARSEMEM_VMEMMAP. Note that check_hotplug_memory_range()
261 	 * enforces a larger memory_block_size_bytes() granularity for
262 	 * memory that will be marked online, so this check should only
263 	 * fire for direct arch_{add,remove}_memory() users outside of
264 	 * add_memory_resource().
265 	 */
266 	unsigned long min_align;
267 
268 	if (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))
269 		min_align = PAGES_PER_SUBSECTION;
270 	else
271 		min_align = PAGES_PER_SECTION;
272 	if (!IS_ALIGNED(pfn, min_align)
273 			|| !IS_ALIGNED(nr_pages, min_align)) {
274 		WARN(1, "Misaligned __%s_pages start: %#lx end: #%lx\n",
275 				reason, pfn, pfn + nr_pages - 1);
276 		return -EINVAL;
277 	}
278 	return 0;
279 }
280 
281 /*
282  * Reasonably generic function for adding memory.  It is
283  * expected that archs that support memory hotplug will
284  * call this function after deciding the zone to which to
285  * add the new pages.
286  */
287 int __ref __add_pages(int nid, unsigned long pfn, unsigned long nr_pages,
288 		struct mhp_restrictions *restrictions)
289 {
290 	int err;
291 	unsigned long nr, start_sec, end_sec;
292 	struct vmem_altmap *altmap = restrictions->altmap;
293 
294 	if (altmap) {
295 		/*
296 		 * Validate altmap is within bounds of the total request
297 		 */
298 		if (altmap->base_pfn != pfn
299 				|| vmem_altmap_offset(altmap) > nr_pages) {
300 			pr_warn_once("memory add fail, invalid altmap\n");
301 			return -EINVAL;
302 		}
303 		altmap->alloc = 0;
304 	}
305 
306 	err = check_pfn_span(pfn, nr_pages, "add");
307 	if (err)
308 		return err;
309 
310 	start_sec = pfn_to_section_nr(pfn);
311 	end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
312 	for (nr = start_sec; nr <= end_sec; nr++) {
313 		unsigned long pfns;
314 
315 		pfns = min(nr_pages, PAGES_PER_SECTION
316 				- (pfn & ~PAGE_SECTION_MASK));
317 		err = sparse_add_section(nid, pfn, pfns, altmap);
318 		if (err)
319 			break;
320 		pfn += pfns;
321 		nr_pages -= pfns;
322 		cond_resched();
323 	}
324 	vmemmap_populate_print_last();
325 	return err;
326 }
327 
328 /* find the smallest valid pfn in the range [start_pfn, end_pfn) */
329 static unsigned long find_smallest_section_pfn(int nid, struct zone *zone,
330 				     unsigned long start_pfn,
331 				     unsigned long end_pfn)
332 {
333 	for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SUBSECTION) {
334 		if (unlikely(!pfn_valid(start_pfn)))
335 			continue;
336 
337 		if (unlikely(pfn_to_nid(start_pfn) != nid))
338 			continue;
339 
340 		if (zone && zone != page_zone(pfn_to_page(start_pfn)))
341 			continue;
342 
343 		return start_pfn;
344 	}
345 
346 	return 0;
347 }
348 
349 /* find the biggest valid pfn in the range [start_pfn, end_pfn). */
350 static unsigned long find_biggest_section_pfn(int nid, struct zone *zone,
351 				    unsigned long start_pfn,
352 				    unsigned long end_pfn)
353 {
354 	unsigned long pfn;
355 
356 	/* pfn is the end pfn of a memory section. */
357 	pfn = end_pfn - 1;
358 	for (; pfn >= start_pfn; pfn -= PAGES_PER_SUBSECTION) {
359 		if (unlikely(!pfn_valid(pfn)))
360 			continue;
361 
362 		if (unlikely(pfn_to_nid(pfn) != nid))
363 			continue;
364 
365 		if (zone && zone != page_zone(pfn_to_page(pfn)))
366 			continue;
367 
368 		return pfn;
369 	}
370 
371 	return 0;
372 }
373 
374 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
375 			     unsigned long end_pfn)
376 {
377 	unsigned long zone_start_pfn = zone->zone_start_pfn;
378 	unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */
379 	unsigned long zone_end_pfn = z;
380 	unsigned long pfn;
381 	int nid = zone_to_nid(zone);
382 
383 	zone_span_writelock(zone);
384 	if (zone_start_pfn == start_pfn) {
385 		/*
386 		 * If the section is smallest section in the zone, it need
387 		 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
388 		 * In this case, we find second smallest valid mem_section
389 		 * for shrinking zone.
390 		 */
391 		pfn = find_smallest_section_pfn(nid, zone, end_pfn,
392 						zone_end_pfn);
393 		if (pfn) {
394 			zone->zone_start_pfn = pfn;
395 			zone->spanned_pages = zone_end_pfn - pfn;
396 		}
397 	} else if (zone_end_pfn == end_pfn) {
398 		/*
399 		 * If the section is biggest section in the zone, it need
400 		 * shrink zone->spanned_pages.
401 		 * In this case, we find second biggest valid mem_section for
402 		 * shrinking zone.
403 		 */
404 		pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
405 					       start_pfn);
406 		if (pfn)
407 			zone->spanned_pages = pfn - zone_start_pfn + 1;
408 	}
409 
410 	/*
411 	 * The section is not biggest or smallest mem_section in the zone, it
412 	 * only creates a hole in the zone. So in this case, we need not
413 	 * change the zone. But perhaps, the zone has only hole data. Thus
414 	 * it check the zone has only hole or not.
415 	 */
416 	pfn = zone_start_pfn;
417 	for (; pfn < zone_end_pfn; pfn += PAGES_PER_SUBSECTION) {
418 		if (unlikely(!pfn_valid(pfn)))
419 			continue;
420 
421 		if (page_zone(pfn_to_page(pfn)) != zone)
422 			continue;
423 
424 		/* Skip range to be removed */
425 		if (pfn >= start_pfn && pfn < end_pfn)
426 			continue;
427 
428 		/* If we find valid section, we have nothing to do */
429 		zone_span_writeunlock(zone);
430 		return;
431 	}
432 
433 	/* The zone has no valid section */
434 	zone->zone_start_pfn = 0;
435 	zone->spanned_pages = 0;
436 	zone_span_writeunlock(zone);
437 }
438 
439 static void shrink_pgdat_span(struct pglist_data *pgdat,
440 			      unsigned long start_pfn, unsigned long end_pfn)
441 {
442 	unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
443 	unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */
444 	unsigned long pgdat_end_pfn = p;
445 	unsigned long pfn;
446 	int nid = pgdat->node_id;
447 
448 	if (pgdat_start_pfn == start_pfn) {
449 		/*
450 		 * If the section is smallest section in the pgdat, it need
451 		 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages.
452 		 * In this case, we find second smallest valid mem_section
453 		 * for shrinking zone.
454 		 */
455 		pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
456 						pgdat_end_pfn);
457 		if (pfn) {
458 			pgdat->node_start_pfn = pfn;
459 			pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
460 		}
461 	} else if (pgdat_end_pfn == end_pfn) {
462 		/*
463 		 * If the section is biggest section in the pgdat, it need
464 		 * shrink pgdat->node_spanned_pages.
465 		 * In this case, we find second biggest valid mem_section for
466 		 * shrinking zone.
467 		 */
468 		pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
469 					       start_pfn);
470 		if (pfn)
471 			pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
472 	}
473 
474 	/*
475 	 * If the section is not biggest or smallest mem_section in the pgdat,
476 	 * it only creates a hole in the pgdat. So in this case, we need not
477 	 * change the pgdat.
478 	 * But perhaps, the pgdat has only hole data. Thus it check the pgdat
479 	 * has only hole or not.
480 	 */
481 	pfn = pgdat_start_pfn;
482 	for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SUBSECTION) {
483 		if (unlikely(!pfn_valid(pfn)))
484 			continue;
485 
486 		if (pfn_to_nid(pfn) != nid)
487 			continue;
488 
489 		/* Skip range to be removed */
490 		if (pfn >= start_pfn && pfn < end_pfn)
491 			continue;
492 
493 		/* If we find valid section, we have nothing to do */
494 		return;
495 	}
496 
497 	/* The pgdat has no valid section */
498 	pgdat->node_start_pfn = 0;
499 	pgdat->node_spanned_pages = 0;
500 }
501 
502 static void __remove_zone(struct zone *zone, unsigned long start_pfn,
503 		unsigned long nr_pages)
504 {
505 	struct pglist_data *pgdat = zone->zone_pgdat;
506 	unsigned long flags;
507 
508 	pgdat_resize_lock(zone->zone_pgdat, &flags);
509 	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
510 	shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
511 	pgdat_resize_unlock(zone->zone_pgdat, &flags);
512 }
513 
514 static void __remove_section(struct zone *zone, unsigned long pfn,
515 		unsigned long nr_pages, unsigned long map_offset,
516 		struct vmem_altmap *altmap)
517 {
518 	struct mem_section *ms = __nr_to_section(pfn_to_section_nr(pfn));
519 
520 	if (WARN_ON_ONCE(!valid_section(ms)))
521 		return;
522 
523 	__remove_zone(zone, pfn, nr_pages);
524 	sparse_remove_section(ms, pfn, nr_pages, map_offset, altmap);
525 }
526 
527 /**
528  * __remove_pages() - remove sections of pages from a zone
529  * @zone: zone from which pages need to be removed
530  * @pfn: starting pageframe (must be aligned to start of a section)
531  * @nr_pages: number of pages to remove (must be multiple of section size)
532  * @altmap: alternative device page map or %NULL if default memmap is used
533  *
534  * Generic helper function to remove section mappings and sysfs entries
535  * for the section of the memory we are removing. Caller needs to make
536  * sure that pages are marked reserved and zones are adjust properly by
537  * calling offline_pages().
538  */
539 void __remove_pages(struct zone *zone, unsigned long pfn,
540 		    unsigned long nr_pages, struct vmem_altmap *altmap)
541 {
542 	unsigned long map_offset = 0;
543 	unsigned long nr, start_sec, end_sec;
544 
545 	map_offset = vmem_altmap_offset(altmap);
546 
547 	clear_zone_contiguous(zone);
548 
549 	if (check_pfn_span(pfn, nr_pages, "remove"))
550 		return;
551 
552 	start_sec = pfn_to_section_nr(pfn);
553 	end_sec = pfn_to_section_nr(pfn + nr_pages - 1);
554 	for (nr = start_sec; nr <= end_sec; nr++) {
555 		unsigned long pfns;
556 
557 		cond_resched();
558 		pfns = min(nr_pages, PAGES_PER_SECTION
559 				- (pfn & ~PAGE_SECTION_MASK));
560 		__remove_section(zone, pfn, pfns, map_offset, altmap);
561 		pfn += pfns;
562 		nr_pages -= pfns;
563 		map_offset = 0;
564 	}
565 
566 	set_zone_contiguous(zone);
567 }
568 
569 int set_online_page_callback(online_page_callback_t callback)
570 {
571 	int rc = -EINVAL;
572 
573 	get_online_mems();
574 	mutex_lock(&online_page_callback_lock);
575 
576 	if (online_page_callback == generic_online_page) {
577 		online_page_callback = callback;
578 		rc = 0;
579 	}
580 
581 	mutex_unlock(&online_page_callback_lock);
582 	put_online_mems();
583 
584 	return rc;
585 }
586 EXPORT_SYMBOL_GPL(set_online_page_callback);
587 
588 int restore_online_page_callback(online_page_callback_t callback)
589 {
590 	int rc = -EINVAL;
591 
592 	get_online_mems();
593 	mutex_lock(&online_page_callback_lock);
594 
595 	if (online_page_callback == callback) {
596 		online_page_callback = generic_online_page;
597 		rc = 0;
598 	}
599 
600 	mutex_unlock(&online_page_callback_lock);
601 	put_online_mems();
602 
603 	return rc;
604 }
605 EXPORT_SYMBOL_GPL(restore_online_page_callback);
606 
607 void __online_page_set_limits(struct page *page)
608 {
609 }
610 EXPORT_SYMBOL_GPL(__online_page_set_limits);
611 
612 void __online_page_increment_counters(struct page *page)
613 {
614 	adjust_managed_page_count(page, 1);
615 }
616 EXPORT_SYMBOL_GPL(__online_page_increment_counters);
617 
618 void __online_page_free(struct page *page)
619 {
620 	__free_reserved_page(page);
621 }
622 EXPORT_SYMBOL_GPL(__online_page_free);
623 
624 static void generic_online_page(struct page *page, unsigned int order)
625 {
626 	kernel_map_pages(page, 1 << order, 1);
627 	__free_pages_core(page, order);
628 	totalram_pages_add(1UL << order);
629 #ifdef CONFIG_HIGHMEM
630 	if (PageHighMem(page))
631 		totalhigh_pages_add(1UL << order);
632 #endif
633 }
634 
635 static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
636 			void *arg)
637 {
638 	const unsigned long end_pfn = start_pfn + nr_pages;
639 	unsigned long pfn;
640 	int order;
641 
642 	/*
643 	 * Online the pages. The callback might decide to keep some pages
644 	 * PG_reserved (to add them to the buddy later), but we still account
645 	 * them as being online/belonging to this zone ("present").
646 	 */
647 	for (pfn = start_pfn; pfn < end_pfn; pfn += 1ul << order) {
648 		order = min(MAX_ORDER - 1, get_order(PFN_PHYS(end_pfn - pfn)));
649 		/* __free_pages_core() wants pfns to be aligned to the order */
650 		if (WARN_ON_ONCE(!IS_ALIGNED(pfn, 1ul << order)))
651 			order = 0;
652 		(*online_page_callback)(pfn_to_page(pfn), order);
653 	}
654 
655 	/* mark all involved sections as online */
656 	online_mem_sections(start_pfn, end_pfn);
657 
658 	*(unsigned long *)arg += nr_pages;
659 	return 0;
660 }
661 
662 /* check which state of node_states will be changed when online memory */
663 static void node_states_check_changes_online(unsigned long nr_pages,
664 	struct zone *zone, struct memory_notify *arg)
665 {
666 	int nid = zone_to_nid(zone);
667 
668 	arg->status_change_nid = NUMA_NO_NODE;
669 	arg->status_change_nid_normal = NUMA_NO_NODE;
670 	arg->status_change_nid_high = NUMA_NO_NODE;
671 
672 	if (!node_state(nid, N_MEMORY))
673 		arg->status_change_nid = nid;
674 	if (zone_idx(zone) <= ZONE_NORMAL && !node_state(nid, N_NORMAL_MEMORY))
675 		arg->status_change_nid_normal = nid;
676 #ifdef CONFIG_HIGHMEM
677 	if (zone_idx(zone) <= ZONE_HIGHMEM && !node_state(nid, N_HIGH_MEMORY))
678 		arg->status_change_nid_high = nid;
679 #endif
680 }
681 
682 static void node_states_set_node(int node, struct memory_notify *arg)
683 {
684 	if (arg->status_change_nid_normal >= 0)
685 		node_set_state(node, N_NORMAL_MEMORY);
686 
687 	if (arg->status_change_nid_high >= 0)
688 		node_set_state(node, N_HIGH_MEMORY);
689 
690 	if (arg->status_change_nid >= 0)
691 		node_set_state(node, N_MEMORY);
692 }
693 
694 static void __meminit resize_zone_range(struct zone *zone, unsigned long start_pfn,
695 		unsigned long nr_pages)
696 {
697 	unsigned long old_end_pfn = zone_end_pfn(zone);
698 
699 	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
700 		zone->zone_start_pfn = start_pfn;
701 
702 	zone->spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - zone->zone_start_pfn;
703 }
704 
705 static void __meminit resize_pgdat_range(struct pglist_data *pgdat, unsigned long start_pfn,
706                                      unsigned long nr_pages)
707 {
708 	unsigned long old_end_pfn = pgdat_end_pfn(pgdat);
709 
710 	if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
711 		pgdat->node_start_pfn = start_pfn;
712 
713 	pgdat->node_spanned_pages = max(start_pfn + nr_pages, old_end_pfn) - pgdat->node_start_pfn;
714 
715 }
716 /*
717  * Associate the pfn range with the given zone, initializing the memmaps
718  * and resizing the pgdat/zone data to span the added pages. After this
719  * call, all affected pages are PG_reserved.
720  */
721 void __ref move_pfn_range_to_zone(struct zone *zone, unsigned long start_pfn,
722 		unsigned long nr_pages, struct vmem_altmap *altmap)
723 {
724 	struct pglist_data *pgdat = zone->zone_pgdat;
725 	int nid = pgdat->node_id;
726 	unsigned long flags;
727 
728 	clear_zone_contiguous(zone);
729 
730 	/* TODO Huh pgdat is irqsave while zone is not. It used to be like that before */
731 	pgdat_resize_lock(pgdat, &flags);
732 	zone_span_writelock(zone);
733 	if (zone_is_empty(zone))
734 		init_currently_empty_zone(zone, start_pfn, nr_pages);
735 	resize_zone_range(zone, start_pfn, nr_pages);
736 	zone_span_writeunlock(zone);
737 	resize_pgdat_range(pgdat, start_pfn, nr_pages);
738 	pgdat_resize_unlock(pgdat, &flags);
739 
740 	/*
741 	 * TODO now we have a visible range of pages which are not associated
742 	 * with their zone properly. Not nice but set_pfnblock_flags_mask
743 	 * expects the zone spans the pfn range. All the pages in the range
744 	 * are reserved so nobody should be touching them so we should be safe
745 	 */
746 	memmap_init_zone(nr_pages, nid, zone_idx(zone), start_pfn,
747 			MEMMAP_HOTPLUG, altmap);
748 
749 	set_zone_contiguous(zone);
750 }
751 
752 /*
753  * Returns a default kernel memory zone for the given pfn range.
754  * If no kernel zone covers this pfn range it will automatically go
755  * to the ZONE_NORMAL.
756  */
757 static struct zone *default_kernel_zone_for_pfn(int nid, unsigned long start_pfn,
758 		unsigned long nr_pages)
759 {
760 	struct pglist_data *pgdat = NODE_DATA(nid);
761 	int zid;
762 
763 	for (zid = 0; zid <= ZONE_NORMAL; zid++) {
764 		struct zone *zone = &pgdat->node_zones[zid];
765 
766 		if (zone_intersects(zone, start_pfn, nr_pages))
767 			return zone;
768 	}
769 
770 	return &pgdat->node_zones[ZONE_NORMAL];
771 }
772 
773 static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn,
774 		unsigned long nr_pages)
775 {
776 	struct zone *kernel_zone = default_kernel_zone_for_pfn(nid, start_pfn,
777 			nr_pages);
778 	struct zone *movable_zone = &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
779 	bool in_kernel = zone_intersects(kernel_zone, start_pfn, nr_pages);
780 	bool in_movable = zone_intersects(movable_zone, start_pfn, nr_pages);
781 
782 	/*
783 	 * We inherit the existing zone in a simple case where zones do not
784 	 * overlap in the given range
785 	 */
786 	if (in_kernel ^ in_movable)
787 		return (in_kernel) ? kernel_zone : movable_zone;
788 
789 	/*
790 	 * If the range doesn't belong to any zone or two zones overlap in the
791 	 * given range then we use movable zone only if movable_node is
792 	 * enabled because we always online to a kernel zone by default.
793 	 */
794 	return movable_node_enabled ? movable_zone : kernel_zone;
795 }
796 
797 struct zone * zone_for_pfn_range(int online_type, int nid, unsigned start_pfn,
798 		unsigned long nr_pages)
799 {
800 	if (online_type == MMOP_ONLINE_KERNEL)
801 		return default_kernel_zone_for_pfn(nid, start_pfn, nr_pages);
802 
803 	if (online_type == MMOP_ONLINE_MOVABLE)
804 		return &NODE_DATA(nid)->node_zones[ZONE_MOVABLE];
805 
806 	return default_zone_for_pfn(nid, start_pfn, nr_pages);
807 }
808 
809 int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
810 {
811 	unsigned long flags;
812 	unsigned long onlined_pages = 0;
813 	struct zone *zone;
814 	int need_zonelists_rebuild = 0;
815 	int nid;
816 	int ret;
817 	struct memory_notify arg;
818 	struct memory_block *mem;
819 
820 	mem_hotplug_begin();
821 
822 	/*
823 	 * We can't use pfn_to_nid() because nid might be stored in struct page
824 	 * which is not yet initialized. Instead, we find nid from memory block.
825 	 */
826 	mem = find_memory_block(__pfn_to_section(pfn));
827 	nid = mem->nid;
828 	put_device(&mem->dev);
829 
830 	/* associate pfn range with the zone */
831 	zone = zone_for_pfn_range(online_type, nid, pfn, nr_pages);
832 	move_pfn_range_to_zone(zone, pfn, nr_pages, NULL);
833 
834 	arg.start_pfn = pfn;
835 	arg.nr_pages = nr_pages;
836 	node_states_check_changes_online(nr_pages, zone, &arg);
837 
838 	ret = memory_notify(MEM_GOING_ONLINE, &arg);
839 	ret = notifier_to_errno(ret);
840 	if (ret)
841 		goto failed_addition;
842 
843 	/*
844 	 * If this zone is not populated, then it is not in zonelist.
845 	 * This means the page allocator ignores this zone.
846 	 * So, zonelist must be updated after online.
847 	 */
848 	if (!populated_zone(zone)) {
849 		need_zonelists_rebuild = 1;
850 		setup_zone_pageset(zone);
851 	}
852 
853 	ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
854 		online_pages_range);
855 	if (ret) {
856 		/* not a single memory resource was applicable */
857 		if (need_zonelists_rebuild)
858 			zone_pcp_reset(zone);
859 		goto failed_addition;
860 	}
861 
862 	zone->present_pages += onlined_pages;
863 
864 	pgdat_resize_lock(zone->zone_pgdat, &flags);
865 	zone->zone_pgdat->node_present_pages += onlined_pages;
866 	pgdat_resize_unlock(zone->zone_pgdat, &flags);
867 
868 	shuffle_zone(zone);
869 
870 	node_states_set_node(nid, &arg);
871 	if (need_zonelists_rebuild)
872 		build_all_zonelists(NULL);
873 	else
874 		zone_pcp_update(zone);
875 
876 	init_per_zone_wmark_min();
877 
878 	kswapd_run(nid);
879 	kcompactd_run(nid);
880 
881 	vm_total_pages = nr_free_pagecache_pages();
882 
883 	writeback_set_ratelimit();
884 
885 	memory_notify(MEM_ONLINE, &arg);
886 	mem_hotplug_done();
887 	return 0;
888 
889 failed_addition:
890 	pr_debug("online_pages [mem %#010llx-%#010llx] failed\n",
891 		 (unsigned long long) pfn << PAGE_SHIFT,
892 		 (((unsigned long long) pfn + nr_pages) << PAGE_SHIFT) - 1);
893 	memory_notify(MEM_CANCEL_ONLINE, &arg);
894 	mem_hotplug_done();
895 	return ret;
896 }
897 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
898 
899 static void reset_node_present_pages(pg_data_t *pgdat)
900 {
901 	struct zone *z;
902 
903 	for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
904 		z->present_pages = 0;
905 
906 	pgdat->node_present_pages = 0;
907 }
908 
909 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
910 static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
911 {
912 	struct pglist_data *pgdat;
913 	unsigned long start_pfn = PFN_DOWN(start);
914 
915 	pgdat = NODE_DATA(nid);
916 	if (!pgdat) {
917 		pgdat = arch_alloc_nodedata(nid);
918 		if (!pgdat)
919 			return NULL;
920 
921 		pgdat->per_cpu_nodestats =
922 			alloc_percpu(struct per_cpu_nodestat);
923 		arch_refresh_nodedata(nid, pgdat);
924 	} else {
925 		int cpu;
926 		/*
927 		 * Reset the nr_zones, order and classzone_idx before reuse.
928 		 * Note that kswapd will init kswapd_classzone_idx properly
929 		 * when it starts in the near future.
930 		 */
931 		pgdat->nr_zones = 0;
932 		pgdat->kswapd_order = 0;
933 		pgdat->kswapd_classzone_idx = 0;
934 		for_each_online_cpu(cpu) {
935 			struct per_cpu_nodestat *p;
936 
937 			p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
938 			memset(p, 0, sizeof(*p));
939 		}
940 	}
941 
942 	/* we can use NODE_DATA(nid) from here */
943 
944 	pgdat->node_id = nid;
945 	pgdat->node_start_pfn = start_pfn;
946 
947 	/* init node's zones as empty zones, we don't have any present pages.*/
948 	free_area_init_core_hotplug(nid);
949 
950 	/*
951 	 * The node we allocated has no zone fallback lists. For avoiding
952 	 * to access not-initialized zonelist, build here.
953 	 */
954 	build_all_zonelists(pgdat);
955 
956 	/*
957 	 * When memory is hot-added, all the memory is in offline state. So
958 	 * clear all zones' present_pages because they will be updated in
959 	 * online_pages() and offline_pages().
960 	 */
961 	reset_node_managed_pages(pgdat);
962 	reset_node_present_pages(pgdat);
963 
964 	return pgdat;
965 }
966 
967 static void rollback_node_hotadd(int nid)
968 {
969 	pg_data_t *pgdat = NODE_DATA(nid);
970 
971 	arch_refresh_nodedata(nid, NULL);
972 	free_percpu(pgdat->per_cpu_nodestats);
973 	arch_free_nodedata(pgdat);
974 }
975 
976 
977 /**
978  * try_online_node - online a node if offlined
979  * @nid: the node ID
980  * @start: start addr of the node
981  * @set_node_online: Whether we want to online the node
982  * called by cpu_up() to online a node without onlined memory.
983  *
984  * Returns:
985  * 1 -> a new node has been allocated
986  * 0 -> the node is already online
987  * -ENOMEM -> the node could not be allocated
988  */
989 static int __try_online_node(int nid, u64 start, bool set_node_online)
990 {
991 	pg_data_t *pgdat;
992 	int ret = 1;
993 
994 	if (node_online(nid))
995 		return 0;
996 
997 	pgdat = hotadd_new_pgdat(nid, start);
998 	if (!pgdat) {
999 		pr_err("Cannot online node %d due to NULL pgdat\n", nid);
1000 		ret = -ENOMEM;
1001 		goto out;
1002 	}
1003 
1004 	if (set_node_online) {
1005 		node_set_online(nid);
1006 		ret = register_one_node(nid);
1007 		BUG_ON(ret);
1008 	}
1009 out:
1010 	return ret;
1011 }
1012 
1013 /*
1014  * Users of this function always want to online/register the node
1015  */
1016 int try_online_node(int nid)
1017 {
1018 	int ret;
1019 
1020 	mem_hotplug_begin();
1021 	ret =  __try_online_node(nid, 0, true);
1022 	mem_hotplug_done();
1023 	return ret;
1024 }
1025 
1026 static int check_hotplug_memory_range(u64 start, u64 size)
1027 {
1028 	/* memory range must be block size aligned */
1029 	if (!size || !IS_ALIGNED(start, memory_block_size_bytes()) ||
1030 	    !IS_ALIGNED(size, memory_block_size_bytes())) {
1031 		pr_err("Block size [%#lx] unaligned hotplug range: start %#llx, size %#llx",
1032 		       memory_block_size_bytes(), start, size);
1033 		return -EINVAL;
1034 	}
1035 
1036 	return 0;
1037 }
1038 
1039 static int online_memory_block(struct memory_block *mem, void *arg)
1040 {
1041 	return device_online(&mem->dev);
1042 }
1043 
1044 /*
1045  * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1046  * and online/offline operations (triggered e.g. by sysfs).
1047  *
1048  * we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG
1049  */
1050 int __ref add_memory_resource(int nid, struct resource *res)
1051 {
1052 	struct mhp_restrictions restrictions = {};
1053 	u64 start, size;
1054 	bool new_node = false;
1055 	int ret;
1056 
1057 	start = res->start;
1058 	size = resource_size(res);
1059 
1060 	ret = check_hotplug_memory_range(start, size);
1061 	if (ret)
1062 		return ret;
1063 
1064 	mem_hotplug_begin();
1065 
1066 	/*
1067 	 * Add new range to memblock so that when hotadd_new_pgdat() is called
1068 	 * to allocate new pgdat, get_pfn_range_for_nid() will be able to find
1069 	 * this new range and calculate total pages correctly.  The range will
1070 	 * be removed at hot-remove time.
1071 	 */
1072 	memblock_add_node(start, size, nid);
1073 
1074 	ret = __try_online_node(nid, start, false);
1075 	if (ret < 0)
1076 		goto error;
1077 	new_node = ret;
1078 
1079 	/* call arch's memory hotadd */
1080 	ret = arch_add_memory(nid, start, size, &restrictions);
1081 	if (ret < 0)
1082 		goto error;
1083 
1084 	/* create memory block devices after memory was added */
1085 	ret = create_memory_block_devices(start, size);
1086 	if (ret) {
1087 		arch_remove_memory(nid, start, size, NULL);
1088 		goto error;
1089 	}
1090 
1091 	if (new_node) {
1092 		/* If sysfs file of new node can't be created, cpu on the node
1093 		 * can't be hot-added. There is no rollback way now.
1094 		 * So, check by BUG_ON() to catch it reluctantly..
1095 		 * We online node here. We can't roll back from here.
1096 		 */
1097 		node_set_online(nid);
1098 		ret = __register_one_node(nid);
1099 		BUG_ON(ret);
1100 	}
1101 
1102 	/* link memory sections under this node.*/
1103 	ret = link_mem_sections(nid, PFN_DOWN(start), PFN_UP(start + size - 1));
1104 	BUG_ON(ret);
1105 
1106 	/* create new memmap entry */
1107 	firmware_map_add_hotplug(start, start + size, "System RAM");
1108 
1109 	/* device_online() will take the lock when calling online_pages() */
1110 	mem_hotplug_done();
1111 
1112 	/* online pages if requested */
1113 	if (memhp_auto_online)
1114 		walk_memory_blocks(start, size, NULL, online_memory_block);
1115 
1116 	return ret;
1117 error:
1118 	/* rollback pgdat allocation and others */
1119 	if (new_node)
1120 		rollback_node_hotadd(nid);
1121 	memblock_remove(start, size);
1122 	mem_hotplug_done();
1123 	return ret;
1124 }
1125 
1126 /* requires device_hotplug_lock, see add_memory_resource() */
1127 int __ref __add_memory(int nid, u64 start, u64 size)
1128 {
1129 	struct resource *res;
1130 	int ret;
1131 
1132 	res = register_memory_resource(start, size);
1133 	if (IS_ERR(res))
1134 		return PTR_ERR(res);
1135 
1136 	ret = add_memory_resource(nid, res);
1137 	if (ret < 0)
1138 		release_memory_resource(res);
1139 	return ret;
1140 }
1141 
1142 int add_memory(int nid, u64 start, u64 size)
1143 {
1144 	int rc;
1145 
1146 	lock_device_hotplug();
1147 	rc = __add_memory(nid, start, size);
1148 	unlock_device_hotplug();
1149 
1150 	return rc;
1151 }
1152 EXPORT_SYMBOL_GPL(add_memory);
1153 
1154 #ifdef CONFIG_MEMORY_HOTREMOVE
1155 /*
1156  * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
1157  * set and the size of the free page is given by page_order(). Using this,
1158  * the function determines if the pageblock contains only free pages.
1159  * Due to buddy contraints, a free page at least the size of a pageblock will
1160  * be located at the start of the pageblock
1161  */
1162 static inline int pageblock_free(struct page *page)
1163 {
1164 	return PageBuddy(page) && page_order(page) >= pageblock_order;
1165 }
1166 
1167 /* Return the pfn of the start of the next active pageblock after a given pfn */
1168 static unsigned long next_active_pageblock(unsigned long pfn)
1169 {
1170 	struct page *page = pfn_to_page(pfn);
1171 
1172 	/* Ensure the starting page is pageblock-aligned */
1173 	BUG_ON(pfn & (pageblock_nr_pages - 1));
1174 
1175 	/* If the entire pageblock is free, move to the end of free page */
1176 	if (pageblock_free(page)) {
1177 		int order;
1178 		/* be careful. we don't have locks, page_order can be changed.*/
1179 		order = page_order(page);
1180 		if ((order < MAX_ORDER) && (order >= pageblock_order))
1181 			return pfn + (1 << order);
1182 	}
1183 
1184 	return pfn + pageblock_nr_pages;
1185 }
1186 
1187 static bool is_pageblock_removable_nolock(unsigned long pfn)
1188 {
1189 	struct page *page = pfn_to_page(pfn);
1190 	struct zone *zone;
1191 
1192 	/*
1193 	 * We have to be careful here because we are iterating over memory
1194 	 * sections which are not zone aware so we might end up outside of
1195 	 * the zone but still within the section.
1196 	 * We have to take care about the node as well. If the node is offline
1197 	 * its NODE_DATA will be NULL - see page_zone.
1198 	 */
1199 	if (!node_online(page_to_nid(page)))
1200 		return false;
1201 
1202 	zone = page_zone(page);
1203 	pfn = page_to_pfn(page);
1204 	if (!zone_spans_pfn(zone, pfn))
1205 		return false;
1206 
1207 	return !has_unmovable_pages(zone, page, 0, MIGRATE_MOVABLE, SKIP_HWPOISON);
1208 }
1209 
1210 /* Checks if this range of memory is likely to be hot-removable. */
1211 bool is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
1212 {
1213 	unsigned long end_pfn, pfn;
1214 
1215 	end_pfn = min(start_pfn + nr_pages,
1216 			zone_end_pfn(page_zone(pfn_to_page(start_pfn))));
1217 
1218 	/* Check the starting page of each pageblock within the range */
1219 	for (pfn = start_pfn; pfn < end_pfn; pfn = next_active_pageblock(pfn)) {
1220 		if (!is_pageblock_removable_nolock(pfn))
1221 			return false;
1222 		cond_resched();
1223 	}
1224 
1225 	/* All pageblocks in the memory block are likely to be hot-removable */
1226 	return true;
1227 }
1228 
1229 /*
1230  * Confirm all pages in a range [start, end) belong to the same zone.
1231  * When true, return its valid [start, end).
1232  */
1233 int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn,
1234 			 unsigned long *valid_start, unsigned long *valid_end)
1235 {
1236 	unsigned long pfn, sec_end_pfn;
1237 	unsigned long start, end;
1238 	struct zone *zone = NULL;
1239 	struct page *page;
1240 	int i;
1241 	for (pfn = start_pfn, sec_end_pfn = SECTION_ALIGN_UP(start_pfn + 1);
1242 	     pfn < end_pfn;
1243 	     pfn = sec_end_pfn, sec_end_pfn += PAGES_PER_SECTION) {
1244 		/* Make sure the memory section is present first */
1245 		if (!present_section_nr(pfn_to_section_nr(pfn)))
1246 			continue;
1247 		for (; pfn < sec_end_pfn && pfn < end_pfn;
1248 		     pfn += MAX_ORDER_NR_PAGES) {
1249 			i = 0;
1250 			/* This is just a CONFIG_HOLES_IN_ZONE check.*/
1251 			while ((i < MAX_ORDER_NR_PAGES) &&
1252 				!pfn_valid_within(pfn + i))
1253 				i++;
1254 			if (i == MAX_ORDER_NR_PAGES || pfn + i >= end_pfn)
1255 				continue;
1256 			/* Check if we got outside of the zone */
1257 			if (zone && !zone_spans_pfn(zone, pfn + i))
1258 				return 0;
1259 			page = pfn_to_page(pfn + i);
1260 			if (zone && page_zone(page) != zone)
1261 				return 0;
1262 			if (!zone)
1263 				start = pfn + i;
1264 			zone = page_zone(page);
1265 			end = pfn + MAX_ORDER_NR_PAGES;
1266 		}
1267 	}
1268 
1269 	if (zone) {
1270 		*valid_start = start;
1271 		*valid_end = min(end, end_pfn);
1272 		return 1;
1273 	} else {
1274 		return 0;
1275 	}
1276 }
1277 
1278 /*
1279  * Scan pfn range [start,end) to find movable/migratable pages (LRU pages,
1280  * non-lru movable pages and hugepages). We scan pfn because it's much
1281  * easier than scanning over linked list. This function returns the pfn
1282  * of the first found movable page if it's found, otherwise 0.
1283  */
1284 static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
1285 {
1286 	unsigned long pfn;
1287 
1288 	for (pfn = start; pfn < end; pfn++) {
1289 		struct page *page, *head;
1290 		unsigned long skip;
1291 
1292 		if (!pfn_valid(pfn))
1293 			continue;
1294 		page = pfn_to_page(pfn);
1295 		if (PageLRU(page))
1296 			return pfn;
1297 		if (__PageMovable(page))
1298 			return pfn;
1299 
1300 		if (!PageHuge(page))
1301 			continue;
1302 		head = compound_head(page);
1303 		if (page_huge_active(head))
1304 			return pfn;
1305 		skip = compound_nr(head) - (page - head);
1306 		pfn += skip - 1;
1307 	}
1308 	return 0;
1309 }
1310 
1311 static struct page *new_node_page(struct page *page, unsigned long private)
1312 {
1313 	int nid = page_to_nid(page);
1314 	nodemask_t nmask = node_states[N_MEMORY];
1315 
1316 	/*
1317 	 * try to allocate from a different node but reuse this node if there
1318 	 * are no other online nodes to be used (e.g. we are offlining a part
1319 	 * of the only existing node)
1320 	 */
1321 	node_clear(nid, nmask);
1322 	if (nodes_empty(nmask))
1323 		node_set(nid, nmask);
1324 
1325 	return new_page_nodemask(page, nid, &nmask);
1326 }
1327 
1328 static int
1329 do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
1330 {
1331 	unsigned long pfn;
1332 	struct page *page;
1333 	int ret = 0;
1334 	LIST_HEAD(source);
1335 
1336 	for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1337 		if (!pfn_valid(pfn))
1338 			continue;
1339 		page = pfn_to_page(pfn);
1340 
1341 		if (PageHuge(page)) {
1342 			struct page *head = compound_head(page);
1343 			pfn = page_to_pfn(head) + compound_nr(head) - 1;
1344 			isolate_huge_page(head, &source);
1345 			continue;
1346 		} else if (PageTransHuge(page))
1347 			pfn = page_to_pfn(compound_head(page))
1348 				+ hpage_nr_pages(page) - 1;
1349 
1350 		/*
1351 		 * HWPoison pages have elevated reference counts so the migration would
1352 		 * fail on them. It also doesn't make any sense to migrate them in the
1353 		 * first place. Still try to unmap such a page in case it is still mapped
1354 		 * (e.g. current hwpoison implementation doesn't unmap KSM pages but keep
1355 		 * the unmap as the catch all safety net).
1356 		 */
1357 		if (PageHWPoison(page)) {
1358 			if (WARN_ON(PageLRU(page)))
1359 				isolate_lru_page(page);
1360 			if (page_mapped(page))
1361 				try_to_unmap(page, TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS);
1362 			continue;
1363 		}
1364 
1365 		if (!get_page_unless_zero(page))
1366 			continue;
1367 		/*
1368 		 * We can skip free pages. And we can deal with pages on
1369 		 * LRU and non-lru movable pages.
1370 		 */
1371 		if (PageLRU(page))
1372 			ret = isolate_lru_page(page);
1373 		else
1374 			ret = isolate_movable_page(page, ISOLATE_UNEVICTABLE);
1375 		if (!ret) { /* Success */
1376 			list_add_tail(&page->lru, &source);
1377 			if (!__PageMovable(page))
1378 				inc_node_page_state(page, NR_ISOLATED_ANON +
1379 						    page_is_file_cache(page));
1380 
1381 		} else {
1382 			pr_warn("failed to isolate pfn %lx\n", pfn);
1383 			dump_page(page, "isolation failed");
1384 		}
1385 		put_page(page);
1386 	}
1387 	if (!list_empty(&source)) {
1388 		/* Allocate a new page from the nearest neighbor node */
1389 		ret = migrate_pages(&source, new_node_page, NULL, 0,
1390 					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
1391 		if (ret) {
1392 			list_for_each_entry(page, &source, lru) {
1393 				pr_warn("migrating pfn %lx failed ret:%d ",
1394 				       page_to_pfn(page), ret);
1395 				dump_page(page, "migration failure");
1396 			}
1397 			putback_movable_pages(&source);
1398 		}
1399 	}
1400 
1401 	return ret;
1402 }
1403 
1404 /*
1405  * remove from free_area[] and mark all as Reserved.
1406  */
1407 static int
1408 offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
1409 			void *data)
1410 {
1411 	unsigned long *offlined_pages = (unsigned long *)data;
1412 
1413 	*offlined_pages += __offline_isolated_pages(start, start + nr_pages);
1414 	return 0;
1415 }
1416 
1417 /*
1418  * Check all pages in range, recoreded as memory resource, are isolated.
1419  */
1420 static int
1421 check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
1422 			void *data)
1423 {
1424 	return test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
1425 }
1426 
1427 static int __init cmdline_parse_movable_node(char *p)
1428 {
1429 #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
1430 	movable_node_enabled = true;
1431 #else
1432 	pr_warn("movable_node parameter depends on CONFIG_HAVE_MEMBLOCK_NODE_MAP to work properly\n");
1433 #endif
1434 	return 0;
1435 }
1436 early_param("movable_node", cmdline_parse_movable_node);
1437 
1438 /* check which state of node_states will be changed when offline memory */
1439 static void node_states_check_changes_offline(unsigned long nr_pages,
1440 		struct zone *zone, struct memory_notify *arg)
1441 {
1442 	struct pglist_data *pgdat = zone->zone_pgdat;
1443 	unsigned long present_pages = 0;
1444 	enum zone_type zt;
1445 
1446 	arg->status_change_nid = NUMA_NO_NODE;
1447 	arg->status_change_nid_normal = NUMA_NO_NODE;
1448 	arg->status_change_nid_high = NUMA_NO_NODE;
1449 
1450 	/*
1451 	 * Check whether node_states[N_NORMAL_MEMORY] will be changed.
1452 	 * If the memory to be offline is within the range
1453 	 * [0..ZONE_NORMAL], and it is the last present memory there,
1454 	 * the zones in that range will become empty after the offlining,
1455 	 * thus we can determine that we need to clear the node from
1456 	 * node_states[N_NORMAL_MEMORY].
1457 	 */
1458 	for (zt = 0; zt <= ZONE_NORMAL; zt++)
1459 		present_pages += pgdat->node_zones[zt].present_pages;
1460 	if (zone_idx(zone) <= ZONE_NORMAL && nr_pages >= present_pages)
1461 		arg->status_change_nid_normal = zone_to_nid(zone);
1462 
1463 #ifdef CONFIG_HIGHMEM
1464 	/*
1465 	 * node_states[N_HIGH_MEMORY] contains nodes which
1466 	 * have normal memory or high memory.
1467 	 * Here we add the present_pages belonging to ZONE_HIGHMEM.
1468 	 * If the zone is within the range of [0..ZONE_HIGHMEM), and
1469 	 * we determine that the zones in that range become empty,
1470 	 * we need to clear the node for N_HIGH_MEMORY.
1471 	 */
1472 	present_pages += pgdat->node_zones[ZONE_HIGHMEM].present_pages;
1473 	if (zone_idx(zone) <= ZONE_HIGHMEM && nr_pages >= present_pages)
1474 		arg->status_change_nid_high = zone_to_nid(zone);
1475 #endif
1476 
1477 	/*
1478 	 * We have accounted the pages from [0..ZONE_NORMAL), and
1479 	 * in case of CONFIG_HIGHMEM the pages from ZONE_HIGHMEM
1480 	 * as well.
1481 	 * Here we count the possible pages from ZONE_MOVABLE.
1482 	 * If after having accounted all the pages, we see that the nr_pages
1483 	 * to be offlined is over or equal to the accounted pages,
1484 	 * we know that the node will become empty, and so, we can clear
1485 	 * it for N_MEMORY as well.
1486 	 */
1487 	present_pages += pgdat->node_zones[ZONE_MOVABLE].present_pages;
1488 
1489 	if (nr_pages >= present_pages)
1490 		arg->status_change_nid = zone_to_nid(zone);
1491 }
1492 
1493 static void node_states_clear_node(int node, struct memory_notify *arg)
1494 {
1495 	if (arg->status_change_nid_normal >= 0)
1496 		node_clear_state(node, N_NORMAL_MEMORY);
1497 
1498 	if (arg->status_change_nid_high >= 0)
1499 		node_clear_state(node, N_HIGH_MEMORY);
1500 
1501 	if (arg->status_change_nid >= 0)
1502 		node_clear_state(node, N_MEMORY);
1503 }
1504 
1505 static int __ref __offline_pages(unsigned long start_pfn,
1506 		  unsigned long end_pfn)
1507 {
1508 	unsigned long pfn, nr_pages;
1509 	unsigned long offlined_pages = 0;
1510 	int ret, node, nr_isolate_pageblock;
1511 	unsigned long flags;
1512 	unsigned long valid_start, valid_end;
1513 	struct zone *zone;
1514 	struct memory_notify arg;
1515 	char *reason;
1516 
1517 	mem_hotplug_begin();
1518 
1519 	/* This makes hotplug much easier...and readable.
1520 	   we assume this for now. .*/
1521 	if (!test_pages_in_a_zone(start_pfn, end_pfn, &valid_start,
1522 				  &valid_end)) {
1523 		ret = -EINVAL;
1524 		reason = "multizone range";
1525 		goto failed_removal;
1526 	}
1527 
1528 	zone = page_zone(pfn_to_page(valid_start));
1529 	node = zone_to_nid(zone);
1530 	nr_pages = end_pfn - start_pfn;
1531 
1532 	/* set above range as isolated */
1533 	ret = start_isolate_page_range(start_pfn, end_pfn,
1534 				       MIGRATE_MOVABLE,
1535 				       SKIP_HWPOISON | REPORT_FAILURE);
1536 	if (ret < 0) {
1537 		reason = "failure to isolate range";
1538 		goto failed_removal;
1539 	}
1540 	nr_isolate_pageblock = ret;
1541 
1542 	arg.start_pfn = start_pfn;
1543 	arg.nr_pages = nr_pages;
1544 	node_states_check_changes_offline(nr_pages, zone, &arg);
1545 
1546 	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
1547 	ret = notifier_to_errno(ret);
1548 	if (ret) {
1549 		reason = "notifier failure";
1550 		goto failed_removal_isolated;
1551 	}
1552 
1553 	do {
1554 		for (pfn = start_pfn; pfn;) {
1555 			if (signal_pending(current)) {
1556 				ret = -EINTR;
1557 				reason = "signal backoff";
1558 				goto failed_removal_isolated;
1559 			}
1560 
1561 			cond_resched();
1562 			lru_add_drain_all();
1563 
1564 			pfn = scan_movable_pages(pfn, end_pfn);
1565 			if (pfn) {
1566 				/*
1567 				 * TODO: fatal migration failures should bail
1568 				 * out
1569 				 */
1570 				do_migrate_range(pfn, end_pfn);
1571 			}
1572 		}
1573 
1574 		/*
1575 		 * Dissolve free hugepages in the memory block before doing
1576 		 * offlining actually in order to make hugetlbfs's object
1577 		 * counting consistent.
1578 		 */
1579 		ret = dissolve_free_huge_pages(start_pfn, end_pfn);
1580 		if (ret) {
1581 			reason = "failure to dissolve huge pages";
1582 			goto failed_removal_isolated;
1583 		}
1584 		/* check again */
1585 		ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn,
1586 					    NULL, check_pages_isolated_cb);
1587 	} while (ret);
1588 
1589 	/* Ok, all of our target is isolated.
1590 	   We cannot do rollback at this point. */
1591 	walk_system_ram_range(start_pfn, end_pfn - start_pfn,
1592 			      &offlined_pages, offline_isolated_pages_cb);
1593 	pr_info("Offlined Pages %ld\n", offlined_pages);
1594 	/*
1595 	 * Onlining will reset pagetype flags and makes migrate type
1596 	 * MOVABLE, so just need to decrease the number of isolated
1597 	 * pageblocks zone counter here.
1598 	 */
1599 	spin_lock_irqsave(&zone->lock, flags);
1600 	zone->nr_isolate_pageblock -= nr_isolate_pageblock;
1601 	spin_unlock_irqrestore(&zone->lock, flags);
1602 
1603 	/* removal success */
1604 	adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
1605 	zone->present_pages -= offlined_pages;
1606 
1607 	pgdat_resize_lock(zone->zone_pgdat, &flags);
1608 	zone->zone_pgdat->node_present_pages -= offlined_pages;
1609 	pgdat_resize_unlock(zone->zone_pgdat, &flags);
1610 
1611 	init_per_zone_wmark_min();
1612 
1613 	if (!populated_zone(zone)) {
1614 		zone_pcp_reset(zone);
1615 		build_all_zonelists(NULL);
1616 	} else
1617 		zone_pcp_update(zone);
1618 
1619 	node_states_clear_node(node, &arg);
1620 	if (arg.status_change_nid >= 0) {
1621 		kswapd_stop(node);
1622 		kcompactd_stop(node);
1623 	}
1624 
1625 	vm_total_pages = nr_free_pagecache_pages();
1626 	writeback_set_ratelimit();
1627 
1628 	memory_notify(MEM_OFFLINE, &arg);
1629 	mem_hotplug_done();
1630 	return 0;
1631 
1632 failed_removal_isolated:
1633 	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1634 	memory_notify(MEM_CANCEL_OFFLINE, &arg);
1635 failed_removal:
1636 	pr_debug("memory offlining [mem %#010llx-%#010llx] failed due to %s\n",
1637 		 (unsigned long long) start_pfn << PAGE_SHIFT,
1638 		 ((unsigned long long) end_pfn << PAGE_SHIFT) - 1,
1639 		 reason);
1640 	/* pushback to free area */
1641 	mem_hotplug_done();
1642 	return ret;
1643 }
1644 
1645 int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
1646 {
1647 	return __offline_pages(start_pfn, start_pfn + nr_pages);
1648 }
1649 
1650 static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
1651 {
1652 	int ret = !is_memblock_offlined(mem);
1653 
1654 	if (unlikely(ret)) {
1655 		phys_addr_t beginpa, endpa;
1656 
1657 		beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
1658 		endpa = beginpa + memory_block_size_bytes() - 1;
1659 		pr_warn("removing memory fails, because memory [%pa-%pa] is onlined\n",
1660 			&beginpa, &endpa);
1661 
1662 		return -EBUSY;
1663 	}
1664 	return 0;
1665 }
1666 
1667 static int check_cpu_on_node(pg_data_t *pgdat)
1668 {
1669 	int cpu;
1670 
1671 	for_each_present_cpu(cpu) {
1672 		if (cpu_to_node(cpu) == pgdat->node_id)
1673 			/*
1674 			 * the cpu on this node isn't removed, and we can't
1675 			 * offline this node.
1676 			 */
1677 			return -EBUSY;
1678 	}
1679 
1680 	return 0;
1681 }
1682 
1683 /**
1684  * try_offline_node
1685  * @nid: the node ID
1686  *
1687  * Offline a node if all memory sections and cpus of the node are removed.
1688  *
1689  * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1690  * and online/offline operations before this call.
1691  */
1692 void try_offline_node(int nid)
1693 {
1694 	pg_data_t *pgdat = NODE_DATA(nid);
1695 	unsigned long start_pfn = pgdat->node_start_pfn;
1696 	unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
1697 	unsigned long pfn;
1698 
1699 	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
1700 		unsigned long section_nr = pfn_to_section_nr(pfn);
1701 
1702 		if (!present_section_nr(section_nr))
1703 			continue;
1704 
1705 		if (pfn_to_nid(pfn) != nid)
1706 			continue;
1707 
1708 		/*
1709 		 * some memory sections of this node are not removed, and we
1710 		 * can't offline node now.
1711 		 */
1712 		return;
1713 	}
1714 
1715 	if (check_cpu_on_node(pgdat))
1716 		return;
1717 
1718 	/*
1719 	 * all memory/cpu of this node are removed, we can offline this
1720 	 * node now.
1721 	 */
1722 	node_set_offline(nid);
1723 	unregister_one_node(nid);
1724 }
1725 EXPORT_SYMBOL(try_offline_node);
1726 
1727 static void __release_memory_resource(resource_size_t start,
1728 				      resource_size_t size)
1729 {
1730 	int ret;
1731 
1732 	/*
1733 	 * When removing memory in the same granularity as it was added,
1734 	 * this function never fails. It might only fail if resources
1735 	 * have to be adjusted or split. We'll ignore the error, as
1736 	 * removing of memory cannot fail.
1737 	 */
1738 	ret = release_mem_region_adjustable(&iomem_resource, start, size);
1739 	if (ret) {
1740 		resource_size_t endres = start + size - 1;
1741 
1742 		pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
1743 			&start, &endres, ret);
1744 	}
1745 }
1746 
1747 static int __ref try_remove_memory(int nid, u64 start, u64 size)
1748 {
1749 	int rc = 0;
1750 
1751 	BUG_ON(check_hotplug_memory_range(start, size));
1752 
1753 	mem_hotplug_begin();
1754 
1755 	/*
1756 	 * All memory blocks must be offlined before removing memory.  Check
1757 	 * whether all memory blocks in question are offline and return error
1758 	 * if this is not the case.
1759 	 */
1760 	rc = walk_memory_blocks(start, size, NULL, check_memblock_offlined_cb);
1761 	if (rc)
1762 		goto done;
1763 
1764 	/* remove memmap entry */
1765 	firmware_map_remove(start, start + size, "System RAM");
1766 	memblock_free(start, size);
1767 	memblock_remove(start, size);
1768 
1769 	/* remove memory block devices before removing memory */
1770 	remove_memory_block_devices(start, size);
1771 
1772 	arch_remove_memory(nid, start, size, NULL);
1773 	__release_memory_resource(start, size);
1774 
1775 	try_offline_node(nid);
1776 
1777 done:
1778 	mem_hotplug_done();
1779 	return rc;
1780 }
1781 
1782 /**
1783  * remove_memory
1784  * @nid: the node ID
1785  * @start: physical address of the region to remove
1786  * @size: size of the region to remove
1787  *
1788  * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1789  * and online/offline operations before this call, as required by
1790  * try_offline_node().
1791  */
1792 void __remove_memory(int nid, u64 start, u64 size)
1793 {
1794 
1795 	/*
1796 	 * trigger BUG() if some memory is not offlined prior to calling this
1797 	 * function
1798 	 */
1799 	if (try_remove_memory(nid, start, size))
1800 		BUG();
1801 }
1802 
1803 /*
1804  * Remove memory if every memory block is offline, otherwise return -EBUSY is
1805  * some memory is not offline
1806  */
1807 int remove_memory(int nid, u64 start, u64 size)
1808 {
1809 	int rc;
1810 
1811 	lock_device_hotplug();
1812 	rc  = try_remove_memory(nid, start, size);
1813 	unlock_device_hotplug();
1814 
1815 	return rc;
1816 }
1817 EXPORT_SYMBOL_GPL(remove_memory);
1818 #endif /* CONFIG_MEMORY_HOTREMOVE */
1819