xref: /openbmc/linux/mm/memory_hotplug.c (revision 206a81c1)
1 /*
2  *  linux/mm/memory_hotplug.c
3  *
4  *  Copyright (C)
5  */
6 
7 #include <linux/stddef.h>
8 #include <linux/mm.h>
9 #include <linux/swap.h>
10 #include <linux/interrupt.h>
11 #include <linux/pagemap.h>
12 #include <linux/compiler.h>
13 #include <linux/export.h>
14 #include <linux/pagevec.h>
15 #include <linux/writeback.h>
16 #include <linux/slab.h>
17 #include <linux/sysctl.h>
18 #include <linux/cpu.h>
19 #include <linux/memory.h>
20 #include <linux/memory_hotplug.h>
21 #include <linux/highmem.h>
22 #include <linux/vmalloc.h>
23 #include <linux/ioport.h>
24 #include <linux/delay.h>
25 #include <linux/migrate.h>
26 #include <linux/page-isolation.h>
27 #include <linux/pfn.h>
28 #include <linux/suspend.h>
29 #include <linux/mm_inline.h>
30 #include <linux/firmware-map.h>
31 #include <linux/stop_machine.h>
32 #include <linux/hugetlb.h>
33 #include <linux/memblock.h>
34 
35 #include <asm/tlbflush.h>
36 
37 #include "internal.h"
38 
39 /*
40  * online_page_callback contains pointer to current page onlining function.
41  * Initially it is generic_online_page(). If it is required it could be
42  * changed by calling set_online_page_callback() for callback registration
43  * and restore_online_page_callback() for generic callback restore.
44  */
45 
46 static void generic_online_page(struct page *page);
47 
48 static online_page_callback_t online_page_callback = generic_online_page;
49 static DEFINE_MUTEX(online_page_callback_lock);
50 
51 /* The same as the cpu_hotplug lock, but for memory hotplug. */
52 static struct {
53 	struct task_struct *active_writer;
54 	struct mutex lock; /* Synchronizes accesses to refcount, */
55 	/*
56 	 * Also blocks the new readers during
57 	 * an ongoing mem hotplug operation.
58 	 */
59 	int refcount;
60 
61 #ifdef CONFIG_DEBUG_LOCK_ALLOC
62 	struct lockdep_map dep_map;
63 #endif
64 } mem_hotplug = {
65 	.active_writer = NULL,
66 	.lock = __MUTEX_INITIALIZER(mem_hotplug.lock),
67 	.refcount = 0,
68 #ifdef CONFIG_DEBUG_LOCK_ALLOC
69 	.dep_map = {.name = "mem_hotplug.lock" },
70 #endif
71 };
72 
73 /* Lockdep annotations for get/put_online_mems() and mem_hotplug_begin/end() */
74 #define memhp_lock_acquire_read() lock_map_acquire_read(&mem_hotplug.dep_map)
75 #define memhp_lock_acquire()      lock_map_acquire(&mem_hotplug.dep_map)
76 #define memhp_lock_release()      lock_map_release(&mem_hotplug.dep_map)
77 
78 void get_online_mems(void)
79 {
80 	might_sleep();
81 	if (mem_hotplug.active_writer == current)
82 		return;
83 	memhp_lock_acquire_read();
84 	mutex_lock(&mem_hotplug.lock);
85 	mem_hotplug.refcount++;
86 	mutex_unlock(&mem_hotplug.lock);
87 
88 }
89 
90 void put_online_mems(void)
91 {
92 	if (mem_hotplug.active_writer == current)
93 		return;
94 	mutex_lock(&mem_hotplug.lock);
95 
96 	if (WARN_ON(!mem_hotplug.refcount))
97 		mem_hotplug.refcount++; /* try to fix things up */
98 
99 	if (!--mem_hotplug.refcount && unlikely(mem_hotplug.active_writer))
100 		wake_up_process(mem_hotplug.active_writer);
101 	mutex_unlock(&mem_hotplug.lock);
102 	memhp_lock_release();
103 
104 }
105 
106 static void mem_hotplug_begin(void)
107 {
108 	mem_hotplug.active_writer = current;
109 
110 	memhp_lock_acquire();
111 	for (;;) {
112 		mutex_lock(&mem_hotplug.lock);
113 		if (likely(!mem_hotplug.refcount))
114 			break;
115 		__set_current_state(TASK_UNINTERRUPTIBLE);
116 		mutex_unlock(&mem_hotplug.lock);
117 		schedule();
118 	}
119 }
120 
121 static void mem_hotplug_done(void)
122 {
123 	mem_hotplug.active_writer = NULL;
124 	mutex_unlock(&mem_hotplug.lock);
125 	memhp_lock_release();
126 }
127 
128 /* add this memory to iomem resource */
129 static struct resource *register_memory_resource(u64 start, u64 size)
130 {
131 	struct resource *res;
132 	res = kzalloc(sizeof(struct resource), GFP_KERNEL);
133 	BUG_ON(!res);
134 
135 	res->name = "System RAM";
136 	res->start = start;
137 	res->end = start + size - 1;
138 	res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
139 	if (request_resource(&iomem_resource, res) < 0) {
140 		pr_debug("System RAM resource %pR cannot be added\n", res);
141 		kfree(res);
142 		res = NULL;
143 	}
144 	return res;
145 }
146 
147 static void release_memory_resource(struct resource *res)
148 {
149 	if (!res)
150 		return;
151 	release_resource(res);
152 	kfree(res);
153 	return;
154 }
155 
156 #ifdef CONFIG_MEMORY_HOTPLUG_SPARSE
157 void get_page_bootmem(unsigned long info,  struct page *page,
158 		      unsigned long type)
159 {
160 	page->lru.next = (struct list_head *) type;
161 	SetPagePrivate(page);
162 	set_page_private(page, info);
163 	atomic_inc(&page->_count);
164 }
165 
166 void put_page_bootmem(struct page *page)
167 {
168 	unsigned long type;
169 
170 	type = (unsigned long) page->lru.next;
171 	BUG_ON(type < MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE ||
172 	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
173 
174 	if (atomic_dec_return(&page->_count) == 1) {
175 		ClearPagePrivate(page);
176 		set_page_private(page, 0);
177 		INIT_LIST_HEAD(&page->lru);
178 		free_reserved_page(page);
179 	}
180 }
181 
182 #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
183 #ifndef CONFIG_SPARSEMEM_VMEMMAP
184 static void register_page_bootmem_info_section(unsigned long start_pfn)
185 {
186 	unsigned long *usemap, mapsize, section_nr, i;
187 	struct mem_section *ms;
188 	struct page *page, *memmap;
189 
190 	section_nr = pfn_to_section_nr(start_pfn);
191 	ms = __nr_to_section(section_nr);
192 
193 	/* Get section's memmap address */
194 	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
195 
196 	/*
197 	 * Get page for the memmap's phys address
198 	 * XXX: need more consideration for sparse_vmemmap...
199 	 */
200 	page = virt_to_page(memmap);
201 	mapsize = sizeof(struct page) * PAGES_PER_SECTION;
202 	mapsize = PAGE_ALIGN(mapsize) >> PAGE_SHIFT;
203 
204 	/* remember memmap's page */
205 	for (i = 0; i < mapsize; i++, page++)
206 		get_page_bootmem(section_nr, page, SECTION_INFO);
207 
208 	usemap = __nr_to_section(section_nr)->pageblock_flags;
209 	page = virt_to_page(usemap);
210 
211 	mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
212 
213 	for (i = 0; i < mapsize; i++, page++)
214 		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
215 
216 }
217 #else /* CONFIG_SPARSEMEM_VMEMMAP */
218 static void register_page_bootmem_info_section(unsigned long start_pfn)
219 {
220 	unsigned long *usemap, mapsize, section_nr, i;
221 	struct mem_section *ms;
222 	struct page *page, *memmap;
223 
224 	if (!pfn_valid(start_pfn))
225 		return;
226 
227 	section_nr = pfn_to_section_nr(start_pfn);
228 	ms = __nr_to_section(section_nr);
229 
230 	memmap = sparse_decode_mem_map(ms->section_mem_map, section_nr);
231 
232 	register_page_bootmem_memmap(section_nr, memmap, PAGES_PER_SECTION);
233 
234 	usemap = __nr_to_section(section_nr)->pageblock_flags;
235 	page = virt_to_page(usemap);
236 
237 	mapsize = PAGE_ALIGN(usemap_size()) >> PAGE_SHIFT;
238 
239 	for (i = 0; i < mapsize; i++, page++)
240 		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
241 }
242 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
243 
244 void register_page_bootmem_info_node(struct pglist_data *pgdat)
245 {
246 	unsigned long i, pfn, end_pfn, nr_pages;
247 	int node = pgdat->node_id;
248 	struct page *page;
249 	struct zone *zone;
250 
251 	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
252 	page = virt_to_page(pgdat);
253 
254 	for (i = 0; i < nr_pages; i++, page++)
255 		get_page_bootmem(node, page, NODE_INFO);
256 
257 	zone = &pgdat->node_zones[0];
258 	for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) {
259 		if (zone_is_initialized(zone)) {
260 			nr_pages = zone->wait_table_hash_nr_entries
261 				* sizeof(wait_queue_head_t);
262 			nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT;
263 			page = virt_to_page(zone->wait_table);
264 
265 			for (i = 0; i < nr_pages; i++, page++)
266 				get_page_bootmem(node, page, NODE_INFO);
267 		}
268 	}
269 
270 	pfn = pgdat->node_start_pfn;
271 	end_pfn = pgdat_end_pfn(pgdat);
272 
273 	/* register section info */
274 	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
275 		/*
276 		 * Some platforms can assign the same pfn to multiple nodes - on
277 		 * node0 as well as nodeN.  To avoid registering a pfn against
278 		 * multiple nodes we check that this pfn does not already
279 		 * reside in some other nodes.
280 		 */
281 		if (pfn_valid(pfn) && (pfn_to_nid(pfn) == node))
282 			register_page_bootmem_info_section(pfn);
283 	}
284 }
285 #endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
286 
287 static void grow_zone_span(struct zone *zone, unsigned long start_pfn,
288 			   unsigned long end_pfn)
289 {
290 	unsigned long old_zone_end_pfn;
291 
292 	zone_span_writelock(zone);
293 
294 	old_zone_end_pfn = zone_end_pfn(zone);
295 	if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
296 		zone->zone_start_pfn = start_pfn;
297 
298 	zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
299 				zone->zone_start_pfn;
300 
301 	zone_span_writeunlock(zone);
302 }
303 
304 static void resize_zone(struct zone *zone, unsigned long start_pfn,
305 		unsigned long end_pfn)
306 {
307 	zone_span_writelock(zone);
308 
309 	if (end_pfn - start_pfn) {
310 		zone->zone_start_pfn = start_pfn;
311 		zone->spanned_pages = end_pfn - start_pfn;
312 	} else {
313 		/*
314 		 * make it consist as free_area_init_core(),
315 		 * if spanned_pages = 0, then keep start_pfn = 0
316 		 */
317 		zone->zone_start_pfn = 0;
318 		zone->spanned_pages = 0;
319 	}
320 
321 	zone_span_writeunlock(zone);
322 }
323 
324 static void fix_zone_id(struct zone *zone, unsigned long start_pfn,
325 		unsigned long end_pfn)
326 {
327 	enum zone_type zid = zone_idx(zone);
328 	int nid = zone->zone_pgdat->node_id;
329 	unsigned long pfn;
330 
331 	for (pfn = start_pfn; pfn < end_pfn; pfn++)
332 		set_page_links(pfn_to_page(pfn), zid, nid, pfn);
333 }
334 
335 /* Can fail with -ENOMEM from allocating a wait table with vmalloc() or
336  * alloc_bootmem_node_nopanic()/memblock_virt_alloc_node_nopanic() */
337 static int __ref ensure_zone_is_initialized(struct zone *zone,
338 			unsigned long start_pfn, unsigned long num_pages)
339 {
340 	if (!zone_is_initialized(zone))
341 		return init_currently_empty_zone(zone, start_pfn, num_pages,
342 						 MEMMAP_HOTPLUG);
343 	return 0;
344 }
345 
346 static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,
347 		unsigned long start_pfn, unsigned long end_pfn)
348 {
349 	int ret;
350 	unsigned long flags;
351 	unsigned long z1_start_pfn;
352 
353 	ret = ensure_zone_is_initialized(z1, start_pfn, end_pfn - start_pfn);
354 	if (ret)
355 		return ret;
356 
357 	pgdat_resize_lock(z1->zone_pgdat, &flags);
358 
359 	/* can't move pfns which are higher than @z2 */
360 	if (end_pfn > zone_end_pfn(z2))
361 		goto out_fail;
362 	/* the move out part must be at the left most of @z2 */
363 	if (start_pfn > z2->zone_start_pfn)
364 		goto out_fail;
365 	/* must included/overlap */
366 	if (end_pfn <= z2->zone_start_pfn)
367 		goto out_fail;
368 
369 	/* use start_pfn for z1's start_pfn if z1 is empty */
370 	if (!zone_is_empty(z1))
371 		z1_start_pfn = z1->zone_start_pfn;
372 	else
373 		z1_start_pfn = start_pfn;
374 
375 	resize_zone(z1, z1_start_pfn, end_pfn);
376 	resize_zone(z2, end_pfn, zone_end_pfn(z2));
377 
378 	pgdat_resize_unlock(z1->zone_pgdat, &flags);
379 
380 	fix_zone_id(z1, start_pfn, end_pfn);
381 
382 	return 0;
383 out_fail:
384 	pgdat_resize_unlock(z1->zone_pgdat, &flags);
385 	return -1;
386 }
387 
388 static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,
389 		unsigned long start_pfn, unsigned long end_pfn)
390 {
391 	int ret;
392 	unsigned long flags;
393 	unsigned long z2_end_pfn;
394 
395 	ret = ensure_zone_is_initialized(z2, start_pfn, end_pfn - start_pfn);
396 	if (ret)
397 		return ret;
398 
399 	pgdat_resize_lock(z1->zone_pgdat, &flags);
400 
401 	/* can't move pfns which are lower than @z1 */
402 	if (z1->zone_start_pfn > start_pfn)
403 		goto out_fail;
404 	/* the move out part mast at the right most of @z1 */
405 	if (zone_end_pfn(z1) >  end_pfn)
406 		goto out_fail;
407 	/* must included/overlap */
408 	if (start_pfn >= zone_end_pfn(z1))
409 		goto out_fail;
410 
411 	/* use end_pfn for z2's end_pfn if z2 is empty */
412 	if (!zone_is_empty(z2))
413 		z2_end_pfn = zone_end_pfn(z2);
414 	else
415 		z2_end_pfn = end_pfn;
416 
417 	resize_zone(z1, z1->zone_start_pfn, start_pfn);
418 	resize_zone(z2, start_pfn, z2_end_pfn);
419 
420 	pgdat_resize_unlock(z1->zone_pgdat, &flags);
421 
422 	fix_zone_id(z2, start_pfn, end_pfn);
423 
424 	return 0;
425 out_fail:
426 	pgdat_resize_unlock(z1->zone_pgdat, &flags);
427 	return -1;
428 }
429 
430 static void grow_pgdat_span(struct pglist_data *pgdat, unsigned long start_pfn,
431 			    unsigned long end_pfn)
432 {
433 	unsigned long old_pgdat_end_pfn = pgdat_end_pfn(pgdat);
434 
435 	if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
436 		pgdat->node_start_pfn = start_pfn;
437 
438 	pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
439 					pgdat->node_start_pfn;
440 }
441 
442 static int __meminit __add_zone(struct zone *zone, unsigned long phys_start_pfn)
443 {
444 	struct pglist_data *pgdat = zone->zone_pgdat;
445 	int nr_pages = PAGES_PER_SECTION;
446 	int nid = pgdat->node_id;
447 	int zone_type;
448 	unsigned long flags;
449 	int ret;
450 
451 	zone_type = zone - pgdat->node_zones;
452 	ret = ensure_zone_is_initialized(zone, phys_start_pfn, nr_pages);
453 	if (ret)
454 		return ret;
455 
456 	pgdat_resize_lock(zone->zone_pgdat, &flags);
457 	grow_zone_span(zone, phys_start_pfn, phys_start_pfn + nr_pages);
458 	grow_pgdat_span(zone->zone_pgdat, phys_start_pfn,
459 			phys_start_pfn + nr_pages);
460 	pgdat_resize_unlock(zone->zone_pgdat, &flags);
461 	memmap_init_zone(nr_pages, nid, zone_type,
462 			 phys_start_pfn, MEMMAP_HOTPLUG);
463 	return 0;
464 }
465 
466 static int __meminit __add_section(int nid, struct zone *zone,
467 					unsigned long phys_start_pfn)
468 {
469 	int ret;
470 
471 	if (pfn_valid(phys_start_pfn))
472 		return -EEXIST;
473 
474 	ret = sparse_add_one_section(zone, phys_start_pfn);
475 
476 	if (ret < 0)
477 		return ret;
478 
479 	ret = __add_zone(zone, phys_start_pfn);
480 
481 	if (ret < 0)
482 		return ret;
483 
484 	return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
485 }
486 
487 /*
488  * Reasonably generic function for adding memory.  It is
489  * expected that archs that support memory hotplug will
490  * call this function after deciding the zone to which to
491  * add the new pages.
492  */
493 int __ref __add_pages(int nid, struct zone *zone, unsigned long phys_start_pfn,
494 			unsigned long nr_pages)
495 {
496 	unsigned long i;
497 	int err = 0;
498 	int start_sec, end_sec;
499 	/* during initialize mem_map, align hot-added range to section */
500 	start_sec = pfn_to_section_nr(phys_start_pfn);
501 	end_sec = pfn_to_section_nr(phys_start_pfn + nr_pages - 1);
502 
503 	for (i = start_sec; i <= end_sec; i++) {
504 		err = __add_section(nid, zone, i << PFN_SECTION_SHIFT);
505 
506 		/*
507 		 * EEXIST is finally dealt with by ioresource collision
508 		 * check. see add_memory() => register_memory_resource()
509 		 * Warning will be printed if there is collision.
510 		 */
511 		if (err && (err != -EEXIST))
512 			break;
513 		err = 0;
514 	}
515 
516 	return err;
517 }
518 EXPORT_SYMBOL_GPL(__add_pages);
519 
520 #ifdef CONFIG_MEMORY_HOTREMOVE
521 /* find the smallest valid pfn in the range [start_pfn, end_pfn) */
522 static int find_smallest_section_pfn(int nid, struct zone *zone,
523 				     unsigned long start_pfn,
524 				     unsigned long end_pfn)
525 {
526 	struct mem_section *ms;
527 
528 	for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
529 		ms = __pfn_to_section(start_pfn);
530 
531 		if (unlikely(!valid_section(ms)))
532 			continue;
533 
534 		if (unlikely(pfn_to_nid(start_pfn) != nid))
535 			continue;
536 
537 		if (zone && zone != page_zone(pfn_to_page(start_pfn)))
538 			continue;
539 
540 		return start_pfn;
541 	}
542 
543 	return 0;
544 }
545 
546 /* find the biggest valid pfn in the range [start_pfn, end_pfn). */
547 static int find_biggest_section_pfn(int nid, struct zone *zone,
548 				    unsigned long start_pfn,
549 				    unsigned long end_pfn)
550 {
551 	struct mem_section *ms;
552 	unsigned long pfn;
553 
554 	/* pfn is the end pfn of a memory section. */
555 	pfn = end_pfn - 1;
556 	for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
557 		ms = __pfn_to_section(pfn);
558 
559 		if (unlikely(!valid_section(ms)))
560 			continue;
561 
562 		if (unlikely(pfn_to_nid(pfn) != nid))
563 			continue;
564 
565 		if (zone && zone != page_zone(pfn_to_page(pfn)))
566 			continue;
567 
568 		return pfn;
569 	}
570 
571 	return 0;
572 }
573 
574 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
575 			     unsigned long end_pfn)
576 {
577 	unsigned long zone_start_pfn = zone->zone_start_pfn;
578 	unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */
579 	unsigned long zone_end_pfn = z;
580 	unsigned long pfn;
581 	struct mem_section *ms;
582 	int nid = zone_to_nid(zone);
583 
584 	zone_span_writelock(zone);
585 	if (zone_start_pfn == start_pfn) {
586 		/*
587 		 * If the section is smallest section in the zone, it need
588 		 * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
589 		 * In this case, we find second smallest valid mem_section
590 		 * for shrinking zone.
591 		 */
592 		pfn = find_smallest_section_pfn(nid, zone, end_pfn,
593 						zone_end_pfn);
594 		if (pfn) {
595 			zone->zone_start_pfn = pfn;
596 			zone->spanned_pages = zone_end_pfn - pfn;
597 		}
598 	} else if (zone_end_pfn == end_pfn) {
599 		/*
600 		 * If the section is biggest section in the zone, it need
601 		 * shrink zone->spanned_pages.
602 		 * In this case, we find second biggest valid mem_section for
603 		 * shrinking zone.
604 		 */
605 		pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
606 					       start_pfn);
607 		if (pfn)
608 			zone->spanned_pages = pfn - zone_start_pfn + 1;
609 	}
610 
611 	/*
612 	 * The section is not biggest or smallest mem_section in the zone, it
613 	 * only creates a hole in the zone. So in this case, we need not
614 	 * change the zone. But perhaps, the zone has only hole data. Thus
615 	 * it check the zone has only hole or not.
616 	 */
617 	pfn = zone_start_pfn;
618 	for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
619 		ms = __pfn_to_section(pfn);
620 
621 		if (unlikely(!valid_section(ms)))
622 			continue;
623 
624 		if (page_zone(pfn_to_page(pfn)) != zone)
625 			continue;
626 
627 		 /* If the section is current section, it continues the loop */
628 		if (start_pfn == pfn)
629 			continue;
630 
631 		/* If we find valid section, we have nothing to do */
632 		zone_span_writeunlock(zone);
633 		return;
634 	}
635 
636 	/* The zone has no valid section */
637 	zone->zone_start_pfn = 0;
638 	zone->spanned_pages = 0;
639 	zone_span_writeunlock(zone);
640 }
641 
642 static void shrink_pgdat_span(struct pglist_data *pgdat,
643 			      unsigned long start_pfn, unsigned long end_pfn)
644 {
645 	unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
646 	unsigned long p = pgdat_end_pfn(pgdat); /* pgdat_end_pfn namespace clash */
647 	unsigned long pgdat_end_pfn = p;
648 	unsigned long pfn;
649 	struct mem_section *ms;
650 	int nid = pgdat->node_id;
651 
652 	if (pgdat_start_pfn == start_pfn) {
653 		/*
654 		 * If the section is smallest section in the pgdat, it need
655 		 * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages.
656 		 * In this case, we find second smallest valid mem_section
657 		 * for shrinking zone.
658 		 */
659 		pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
660 						pgdat_end_pfn);
661 		if (pfn) {
662 			pgdat->node_start_pfn = pfn;
663 			pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
664 		}
665 	} else if (pgdat_end_pfn == end_pfn) {
666 		/*
667 		 * If the section is biggest section in the pgdat, it need
668 		 * shrink pgdat->node_spanned_pages.
669 		 * In this case, we find second biggest valid mem_section for
670 		 * shrinking zone.
671 		 */
672 		pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
673 					       start_pfn);
674 		if (pfn)
675 			pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
676 	}
677 
678 	/*
679 	 * If the section is not biggest or smallest mem_section in the pgdat,
680 	 * it only creates a hole in the pgdat. So in this case, we need not
681 	 * change the pgdat.
682 	 * But perhaps, the pgdat has only hole data. Thus it check the pgdat
683 	 * has only hole or not.
684 	 */
685 	pfn = pgdat_start_pfn;
686 	for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
687 		ms = __pfn_to_section(pfn);
688 
689 		if (unlikely(!valid_section(ms)))
690 			continue;
691 
692 		if (pfn_to_nid(pfn) != nid)
693 			continue;
694 
695 		 /* If the section is current section, it continues the loop */
696 		if (start_pfn == pfn)
697 			continue;
698 
699 		/* If we find valid section, we have nothing to do */
700 		return;
701 	}
702 
703 	/* The pgdat has no valid section */
704 	pgdat->node_start_pfn = 0;
705 	pgdat->node_spanned_pages = 0;
706 }
707 
708 static void __remove_zone(struct zone *zone, unsigned long start_pfn)
709 {
710 	struct pglist_data *pgdat = zone->zone_pgdat;
711 	int nr_pages = PAGES_PER_SECTION;
712 	int zone_type;
713 	unsigned long flags;
714 
715 	zone_type = zone - pgdat->node_zones;
716 
717 	pgdat_resize_lock(zone->zone_pgdat, &flags);
718 	shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
719 	shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
720 	pgdat_resize_unlock(zone->zone_pgdat, &flags);
721 }
722 
723 static int __remove_section(struct zone *zone, struct mem_section *ms)
724 {
725 	unsigned long start_pfn;
726 	int scn_nr;
727 	int ret = -EINVAL;
728 
729 	if (!valid_section(ms))
730 		return ret;
731 
732 	ret = unregister_memory_section(ms);
733 	if (ret)
734 		return ret;
735 
736 	scn_nr = __section_nr(ms);
737 	start_pfn = section_nr_to_pfn(scn_nr);
738 	__remove_zone(zone, start_pfn);
739 
740 	sparse_remove_one_section(zone, ms);
741 	return 0;
742 }
743 
744 /**
745  * __remove_pages() - remove sections of pages from a zone
746  * @zone: zone from which pages need to be removed
747  * @phys_start_pfn: starting pageframe (must be aligned to start of a section)
748  * @nr_pages: number of pages to remove (must be multiple of section size)
749  *
750  * Generic helper function to remove section mappings and sysfs entries
751  * for the section of the memory we are removing. Caller needs to make
752  * sure that pages are marked reserved and zones are adjust properly by
753  * calling offline_pages().
754  */
755 int __remove_pages(struct zone *zone, unsigned long phys_start_pfn,
756 		 unsigned long nr_pages)
757 {
758 	unsigned long i;
759 	int sections_to_remove;
760 	resource_size_t start, size;
761 	int ret = 0;
762 
763 	/*
764 	 * We can only remove entire sections
765 	 */
766 	BUG_ON(phys_start_pfn & ~PAGE_SECTION_MASK);
767 	BUG_ON(nr_pages % PAGES_PER_SECTION);
768 
769 	start = phys_start_pfn << PAGE_SHIFT;
770 	size = nr_pages * PAGE_SIZE;
771 	ret = release_mem_region_adjustable(&iomem_resource, start, size);
772 	if (ret) {
773 		resource_size_t endres = start + size - 1;
774 
775 		pr_warn("Unable to release resource <%pa-%pa> (%d)\n",
776 				&start, &endres, ret);
777 	}
778 
779 	sections_to_remove = nr_pages / PAGES_PER_SECTION;
780 	for (i = 0; i < sections_to_remove; i++) {
781 		unsigned long pfn = phys_start_pfn + i*PAGES_PER_SECTION;
782 		ret = __remove_section(zone, __pfn_to_section(pfn));
783 		if (ret)
784 			break;
785 	}
786 	return ret;
787 }
788 EXPORT_SYMBOL_GPL(__remove_pages);
789 #endif /* CONFIG_MEMORY_HOTREMOVE */
790 
791 int set_online_page_callback(online_page_callback_t callback)
792 {
793 	int rc = -EINVAL;
794 
795 	get_online_mems();
796 	mutex_lock(&online_page_callback_lock);
797 
798 	if (online_page_callback == generic_online_page) {
799 		online_page_callback = callback;
800 		rc = 0;
801 	}
802 
803 	mutex_unlock(&online_page_callback_lock);
804 	put_online_mems();
805 
806 	return rc;
807 }
808 EXPORT_SYMBOL_GPL(set_online_page_callback);
809 
810 int restore_online_page_callback(online_page_callback_t callback)
811 {
812 	int rc = -EINVAL;
813 
814 	get_online_mems();
815 	mutex_lock(&online_page_callback_lock);
816 
817 	if (online_page_callback == callback) {
818 		online_page_callback = generic_online_page;
819 		rc = 0;
820 	}
821 
822 	mutex_unlock(&online_page_callback_lock);
823 	put_online_mems();
824 
825 	return rc;
826 }
827 EXPORT_SYMBOL_GPL(restore_online_page_callback);
828 
829 void __online_page_set_limits(struct page *page)
830 {
831 }
832 EXPORT_SYMBOL_GPL(__online_page_set_limits);
833 
834 void __online_page_increment_counters(struct page *page)
835 {
836 	adjust_managed_page_count(page, 1);
837 }
838 EXPORT_SYMBOL_GPL(__online_page_increment_counters);
839 
840 void __online_page_free(struct page *page)
841 {
842 	__free_reserved_page(page);
843 }
844 EXPORT_SYMBOL_GPL(__online_page_free);
845 
846 static void generic_online_page(struct page *page)
847 {
848 	__online_page_set_limits(page);
849 	__online_page_increment_counters(page);
850 	__online_page_free(page);
851 }
852 
853 static int online_pages_range(unsigned long start_pfn, unsigned long nr_pages,
854 			void *arg)
855 {
856 	unsigned long i;
857 	unsigned long onlined_pages = *(unsigned long *)arg;
858 	struct page *page;
859 	if (PageReserved(pfn_to_page(start_pfn)))
860 		for (i = 0; i < nr_pages; i++) {
861 			page = pfn_to_page(start_pfn + i);
862 			(*online_page_callback)(page);
863 			onlined_pages++;
864 		}
865 	*(unsigned long *)arg = onlined_pages;
866 	return 0;
867 }
868 
869 #ifdef CONFIG_MOVABLE_NODE
870 /*
871  * When CONFIG_MOVABLE_NODE, we permit onlining of a node which doesn't have
872  * normal memory.
873  */
874 static bool can_online_high_movable(struct zone *zone)
875 {
876 	return true;
877 }
878 #else /* CONFIG_MOVABLE_NODE */
879 /* ensure every online node has NORMAL memory */
880 static bool can_online_high_movable(struct zone *zone)
881 {
882 	return node_state(zone_to_nid(zone), N_NORMAL_MEMORY);
883 }
884 #endif /* CONFIG_MOVABLE_NODE */
885 
886 /* check which state of node_states will be changed when online memory */
887 static void node_states_check_changes_online(unsigned long nr_pages,
888 	struct zone *zone, struct memory_notify *arg)
889 {
890 	int nid = zone_to_nid(zone);
891 	enum zone_type zone_last = ZONE_NORMAL;
892 
893 	/*
894 	 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
895 	 * contains nodes which have zones of 0...ZONE_NORMAL,
896 	 * set zone_last to ZONE_NORMAL.
897 	 *
898 	 * If we don't have HIGHMEM nor movable node,
899 	 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
900 	 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
901 	 */
902 	if (N_MEMORY == N_NORMAL_MEMORY)
903 		zone_last = ZONE_MOVABLE;
904 
905 	/*
906 	 * if the memory to be online is in a zone of 0...zone_last, and
907 	 * the zones of 0...zone_last don't have memory before online, we will
908 	 * need to set the node to node_states[N_NORMAL_MEMORY] after
909 	 * the memory is online.
910 	 */
911 	if (zone_idx(zone) <= zone_last && !node_state(nid, N_NORMAL_MEMORY))
912 		arg->status_change_nid_normal = nid;
913 	else
914 		arg->status_change_nid_normal = -1;
915 
916 #ifdef CONFIG_HIGHMEM
917 	/*
918 	 * If we have movable node, node_states[N_HIGH_MEMORY]
919 	 * contains nodes which have zones of 0...ZONE_HIGHMEM,
920 	 * set zone_last to ZONE_HIGHMEM.
921 	 *
922 	 * If we don't have movable node, node_states[N_NORMAL_MEMORY]
923 	 * contains nodes which have zones of 0...ZONE_MOVABLE,
924 	 * set zone_last to ZONE_MOVABLE.
925 	 */
926 	zone_last = ZONE_HIGHMEM;
927 	if (N_MEMORY == N_HIGH_MEMORY)
928 		zone_last = ZONE_MOVABLE;
929 
930 	if (zone_idx(zone) <= zone_last && !node_state(nid, N_HIGH_MEMORY))
931 		arg->status_change_nid_high = nid;
932 	else
933 		arg->status_change_nid_high = -1;
934 #else
935 	arg->status_change_nid_high = arg->status_change_nid_normal;
936 #endif
937 
938 	/*
939 	 * if the node don't have memory befor online, we will need to
940 	 * set the node to node_states[N_MEMORY] after the memory
941 	 * is online.
942 	 */
943 	if (!node_state(nid, N_MEMORY))
944 		arg->status_change_nid = nid;
945 	else
946 		arg->status_change_nid = -1;
947 }
948 
949 static void node_states_set_node(int node, struct memory_notify *arg)
950 {
951 	if (arg->status_change_nid_normal >= 0)
952 		node_set_state(node, N_NORMAL_MEMORY);
953 
954 	if (arg->status_change_nid_high >= 0)
955 		node_set_state(node, N_HIGH_MEMORY);
956 
957 	node_set_state(node, N_MEMORY);
958 }
959 
960 
961 int __ref online_pages(unsigned long pfn, unsigned long nr_pages, int online_type)
962 {
963 	unsigned long flags;
964 	unsigned long onlined_pages = 0;
965 	struct zone *zone;
966 	int need_zonelists_rebuild = 0;
967 	int nid;
968 	int ret;
969 	struct memory_notify arg;
970 
971 	mem_hotplug_begin();
972 	/*
973 	 * This doesn't need a lock to do pfn_to_page().
974 	 * The section can't be removed here because of the
975 	 * memory_block->state_mutex.
976 	 */
977 	zone = page_zone(pfn_to_page(pfn));
978 
979 	ret = -EINVAL;
980 	if ((zone_idx(zone) > ZONE_NORMAL || online_type == ONLINE_MOVABLE) &&
981 	    !can_online_high_movable(zone))
982 		goto out;
983 
984 	if (online_type == ONLINE_KERNEL && zone_idx(zone) == ZONE_MOVABLE) {
985 		if (move_pfn_range_left(zone - 1, zone, pfn, pfn + nr_pages))
986 			goto out;
987 	}
988 	if (online_type == ONLINE_MOVABLE && zone_idx(zone) == ZONE_MOVABLE - 1) {
989 		if (move_pfn_range_right(zone, zone + 1, pfn, pfn + nr_pages))
990 			goto out;
991 	}
992 
993 	/* Previous code may changed the zone of the pfn range */
994 	zone = page_zone(pfn_to_page(pfn));
995 
996 	arg.start_pfn = pfn;
997 	arg.nr_pages = nr_pages;
998 	node_states_check_changes_online(nr_pages, zone, &arg);
999 
1000 	nid = pfn_to_nid(pfn);
1001 
1002 	ret = memory_notify(MEM_GOING_ONLINE, &arg);
1003 	ret = notifier_to_errno(ret);
1004 	if (ret) {
1005 		memory_notify(MEM_CANCEL_ONLINE, &arg);
1006 		goto out;
1007 	}
1008 	/*
1009 	 * If this zone is not populated, then it is not in zonelist.
1010 	 * This means the page allocator ignores this zone.
1011 	 * So, zonelist must be updated after online.
1012 	 */
1013 	mutex_lock(&zonelists_mutex);
1014 	if (!populated_zone(zone)) {
1015 		need_zonelists_rebuild = 1;
1016 		build_all_zonelists(NULL, zone);
1017 	}
1018 
1019 	ret = walk_system_ram_range(pfn, nr_pages, &onlined_pages,
1020 		online_pages_range);
1021 	if (ret) {
1022 		if (need_zonelists_rebuild)
1023 			zone_pcp_reset(zone);
1024 		mutex_unlock(&zonelists_mutex);
1025 		printk(KERN_DEBUG "online_pages [mem %#010llx-%#010llx] failed\n",
1026 		       (unsigned long long) pfn << PAGE_SHIFT,
1027 		       (((unsigned long long) pfn + nr_pages)
1028 			    << PAGE_SHIFT) - 1);
1029 		memory_notify(MEM_CANCEL_ONLINE, &arg);
1030 		goto out;
1031 	}
1032 
1033 	zone->present_pages += onlined_pages;
1034 
1035 	pgdat_resize_lock(zone->zone_pgdat, &flags);
1036 	zone->zone_pgdat->node_present_pages += onlined_pages;
1037 	pgdat_resize_unlock(zone->zone_pgdat, &flags);
1038 
1039 	if (onlined_pages) {
1040 		node_states_set_node(zone_to_nid(zone), &arg);
1041 		if (need_zonelists_rebuild)
1042 			build_all_zonelists(NULL, NULL);
1043 		else
1044 			zone_pcp_update(zone);
1045 	}
1046 
1047 	mutex_unlock(&zonelists_mutex);
1048 
1049 	init_per_zone_wmark_min();
1050 
1051 	if (onlined_pages)
1052 		kswapd_run(zone_to_nid(zone));
1053 
1054 	vm_total_pages = nr_free_pagecache_pages();
1055 
1056 	writeback_set_ratelimit();
1057 
1058 	if (onlined_pages)
1059 		memory_notify(MEM_ONLINE, &arg);
1060 out:
1061 	mem_hotplug_done();
1062 	return ret;
1063 }
1064 #endif /* CONFIG_MEMORY_HOTPLUG_SPARSE */
1065 
1066 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
1067 static pg_data_t __ref *hotadd_new_pgdat(int nid, u64 start)
1068 {
1069 	struct pglist_data *pgdat;
1070 	unsigned long zones_size[MAX_NR_ZONES] = {0};
1071 	unsigned long zholes_size[MAX_NR_ZONES] = {0};
1072 	unsigned long start_pfn = PFN_DOWN(start);
1073 
1074 	pgdat = NODE_DATA(nid);
1075 	if (!pgdat) {
1076 		pgdat = arch_alloc_nodedata(nid);
1077 		if (!pgdat)
1078 			return NULL;
1079 
1080 		arch_refresh_nodedata(nid, pgdat);
1081 	}
1082 
1083 	/* we can use NODE_DATA(nid) from here */
1084 
1085 	/* init node's zones as empty zones, we don't have any present pages.*/
1086 	free_area_init_node(nid, zones_size, start_pfn, zholes_size);
1087 
1088 	/*
1089 	 * The node we allocated has no zone fallback lists. For avoiding
1090 	 * to access not-initialized zonelist, build here.
1091 	 */
1092 	mutex_lock(&zonelists_mutex);
1093 	build_all_zonelists(pgdat, NULL);
1094 	mutex_unlock(&zonelists_mutex);
1095 
1096 	return pgdat;
1097 }
1098 
1099 static void rollback_node_hotadd(int nid, pg_data_t *pgdat)
1100 {
1101 	arch_refresh_nodedata(nid, NULL);
1102 	arch_free_nodedata(pgdat);
1103 	return;
1104 }
1105 
1106 
1107 /**
1108  * try_online_node - online a node if offlined
1109  *
1110  * called by cpu_up() to online a node without onlined memory.
1111  */
1112 int try_online_node(int nid)
1113 {
1114 	pg_data_t	*pgdat;
1115 	int	ret;
1116 
1117 	if (node_online(nid))
1118 		return 0;
1119 
1120 	mem_hotplug_begin();
1121 	pgdat = hotadd_new_pgdat(nid, 0);
1122 	if (!pgdat) {
1123 		pr_err("Cannot online node %d due to NULL pgdat\n", nid);
1124 		ret = -ENOMEM;
1125 		goto out;
1126 	}
1127 	node_set_online(nid);
1128 	ret = register_one_node(nid);
1129 	BUG_ON(ret);
1130 
1131 	if (pgdat->node_zonelists->_zonerefs->zone == NULL) {
1132 		mutex_lock(&zonelists_mutex);
1133 		build_all_zonelists(NULL, NULL);
1134 		mutex_unlock(&zonelists_mutex);
1135 	}
1136 
1137 out:
1138 	mem_hotplug_done();
1139 	return ret;
1140 }
1141 
1142 static int check_hotplug_memory_range(u64 start, u64 size)
1143 {
1144 	u64 start_pfn = PFN_DOWN(start);
1145 	u64 nr_pages = size >> PAGE_SHIFT;
1146 
1147 	/* Memory range must be aligned with section */
1148 	if ((start_pfn & ~PAGE_SECTION_MASK) ||
1149 	    (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) {
1150 		pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n",
1151 				(unsigned long long)start,
1152 				(unsigned long long)size);
1153 		return -EINVAL;
1154 	}
1155 
1156 	return 0;
1157 }
1158 
1159 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
1160 int __ref add_memory(int nid, u64 start, u64 size)
1161 {
1162 	pg_data_t *pgdat = NULL;
1163 	bool new_pgdat;
1164 	bool new_node;
1165 	struct resource *res;
1166 	int ret;
1167 
1168 	ret = check_hotplug_memory_range(start, size);
1169 	if (ret)
1170 		return ret;
1171 
1172 	res = register_memory_resource(start, size);
1173 	ret = -EEXIST;
1174 	if (!res)
1175 		return ret;
1176 
1177 	{	/* Stupid hack to suppress address-never-null warning */
1178 		void *p = NODE_DATA(nid);
1179 		new_pgdat = !p;
1180 	}
1181 
1182 	mem_hotplug_begin();
1183 
1184 	new_node = !node_online(nid);
1185 	if (new_node) {
1186 		pgdat = hotadd_new_pgdat(nid, start);
1187 		ret = -ENOMEM;
1188 		if (!pgdat)
1189 			goto error;
1190 	}
1191 
1192 	/* call arch's memory hotadd */
1193 	ret = arch_add_memory(nid, start, size);
1194 
1195 	if (ret < 0)
1196 		goto error;
1197 
1198 	/* we online node here. we can't roll back from here. */
1199 	node_set_online(nid);
1200 
1201 	if (new_node) {
1202 		ret = register_one_node(nid);
1203 		/*
1204 		 * If sysfs file of new node can't create, cpu on the node
1205 		 * can't be hot-added. There is no rollback way now.
1206 		 * So, check by BUG_ON() to catch it reluctantly..
1207 		 */
1208 		BUG_ON(ret);
1209 	}
1210 
1211 	/* create new memmap entry */
1212 	firmware_map_add_hotplug(start, start + size, "System RAM");
1213 
1214 	goto out;
1215 
1216 error:
1217 	/* rollback pgdat allocation and others */
1218 	if (new_pgdat)
1219 		rollback_node_hotadd(nid, pgdat);
1220 	release_memory_resource(res);
1221 
1222 out:
1223 	mem_hotplug_done();
1224 	return ret;
1225 }
1226 EXPORT_SYMBOL_GPL(add_memory);
1227 
1228 #ifdef CONFIG_MEMORY_HOTREMOVE
1229 /*
1230  * A free page on the buddy free lists (not the per-cpu lists) has PageBuddy
1231  * set and the size of the free page is given by page_order(). Using this,
1232  * the function determines if the pageblock contains only free pages.
1233  * Due to buddy contraints, a free page at least the size of a pageblock will
1234  * be located at the start of the pageblock
1235  */
1236 static inline int pageblock_free(struct page *page)
1237 {
1238 	return PageBuddy(page) && page_order(page) >= pageblock_order;
1239 }
1240 
1241 /* Return the start of the next active pageblock after a given page */
1242 static struct page *next_active_pageblock(struct page *page)
1243 {
1244 	/* Ensure the starting page is pageblock-aligned */
1245 	BUG_ON(page_to_pfn(page) & (pageblock_nr_pages - 1));
1246 
1247 	/* If the entire pageblock is free, move to the end of free page */
1248 	if (pageblock_free(page)) {
1249 		int order;
1250 		/* be careful. we don't have locks, page_order can be changed.*/
1251 		order = page_order(page);
1252 		if ((order < MAX_ORDER) && (order >= pageblock_order))
1253 			return page + (1 << order);
1254 	}
1255 
1256 	return page + pageblock_nr_pages;
1257 }
1258 
1259 /* Checks if this range of memory is likely to be hot-removable. */
1260 int is_mem_section_removable(unsigned long start_pfn, unsigned long nr_pages)
1261 {
1262 	struct page *page = pfn_to_page(start_pfn);
1263 	struct page *end_page = page + nr_pages;
1264 
1265 	/* Check the starting page of each pageblock within the range */
1266 	for (; page < end_page; page = next_active_pageblock(page)) {
1267 		if (!is_pageblock_removable_nolock(page))
1268 			return 0;
1269 		cond_resched();
1270 	}
1271 
1272 	/* All pageblocks in the memory block are likely to be hot-removable */
1273 	return 1;
1274 }
1275 
1276 /*
1277  * Confirm all pages in a range [start, end) is belongs to the same zone.
1278  */
1279 static int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn)
1280 {
1281 	unsigned long pfn;
1282 	struct zone *zone = NULL;
1283 	struct page *page;
1284 	int i;
1285 	for (pfn = start_pfn;
1286 	     pfn < end_pfn;
1287 	     pfn += MAX_ORDER_NR_PAGES) {
1288 		i = 0;
1289 		/* This is just a CONFIG_HOLES_IN_ZONE check.*/
1290 		while ((i < MAX_ORDER_NR_PAGES) && !pfn_valid_within(pfn + i))
1291 			i++;
1292 		if (i == MAX_ORDER_NR_PAGES)
1293 			continue;
1294 		page = pfn_to_page(pfn + i);
1295 		if (zone && page_zone(page) != zone)
1296 			return 0;
1297 		zone = page_zone(page);
1298 	}
1299 	return 1;
1300 }
1301 
1302 /*
1303  * Scan pfn range [start,end) to find movable/migratable pages (LRU pages
1304  * and hugepages). We scan pfn because it's much easier than scanning over
1305  * linked list. This function returns the pfn of the first found movable
1306  * page if it's found, otherwise 0.
1307  */
1308 static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
1309 {
1310 	unsigned long pfn;
1311 	struct page *page;
1312 	for (pfn = start; pfn < end; pfn++) {
1313 		if (pfn_valid(pfn)) {
1314 			page = pfn_to_page(pfn);
1315 			if (PageLRU(page))
1316 				return pfn;
1317 			if (PageHuge(page)) {
1318 				if (is_hugepage_active(page))
1319 					return pfn;
1320 				else
1321 					pfn = round_up(pfn + 1,
1322 						1 << compound_order(page)) - 1;
1323 			}
1324 		}
1325 	}
1326 	return 0;
1327 }
1328 
1329 #define NR_OFFLINE_AT_ONCE_PAGES	(256)
1330 static int
1331 do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
1332 {
1333 	unsigned long pfn;
1334 	struct page *page;
1335 	int move_pages = NR_OFFLINE_AT_ONCE_PAGES;
1336 	int not_managed = 0;
1337 	int ret = 0;
1338 	LIST_HEAD(source);
1339 
1340 	for (pfn = start_pfn; pfn < end_pfn && move_pages > 0; pfn++) {
1341 		if (!pfn_valid(pfn))
1342 			continue;
1343 		page = pfn_to_page(pfn);
1344 
1345 		if (PageHuge(page)) {
1346 			struct page *head = compound_head(page);
1347 			pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1;
1348 			if (compound_order(head) > PFN_SECTION_SHIFT) {
1349 				ret = -EBUSY;
1350 				break;
1351 			}
1352 			if (isolate_huge_page(page, &source))
1353 				move_pages -= 1 << compound_order(head);
1354 			continue;
1355 		}
1356 
1357 		if (!get_page_unless_zero(page))
1358 			continue;
1359 		/*
1360 		 * We can skip free pages. And we can only deal with pages on
1361 		 * LRU.
1362 		 */
1363 		ret = isolate_lru_page(page);
1364 		if (!ret) { /* Success */
1365 			put_page(page);
1366 			list_add_tail(&page->lru, &source);
1367 			move_pages--;
1368 			inc_zone_page_state(page, NR_ISOLATED_ANON +
1369 					    page_is_file_cache(page));
1370 
1371 		} else {
1372 #ifdef CONFIG_DEBUG_VM
1373 			printk(KERN_ALERT "removing pfn %lx from LRU failed\n",
1374 			       pfn);
1375 			dump_page(page, "failed to remove from LRU");
1376 #endif
1377 			put_page(page);
1378 			/* Because we don't have big zone->lock. we should
1379 			   check this again here. */
1380 			if (page_count(page)) {
1381 				not_managed++;
1382 				ret = -EBUSY;
1383 				break;
1384 			}
1385 		}
1386 	}
1387 	if (!list_empty(&source)) {
1388 		if (not_managed) {
1389 			putback_movable_pages(&source);
1390 			goto out;
1391 		}
1392 
1393 		/*
1394 		 * alloc_migrate_target should be improooooved!!
1395 		 * migrate_pages returns # of failed pages.
1396 		 */
1397 		ret = migrate_pages(&source, alloc_migrate_target, NULL, 0,
1398 					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
1399 		if (ret)
1400 			putback_movable_pages(&source);
1401 	}
1402 out:
1403 	return ret;
1404 }
1405 
1406 /*
1407  * remove from free_area[] and mark all as Reserved.
1408  */
1409 static int
1410 offline_isolated_pages_cb(unsigned long start, unsigned long nr_pages,
1411 			void *data)
1412 {
1413 	__offline_isolated_pages(start, start + nr_pages);
1414 	return 0;
1415 }
1416 
1417 static void
1418 offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
1419 {
1420 	walk_system_ram_range(start_pfn, end_pfn - start_pfn, NULL,
1421 				offline_isolated_pages_cb);
1422 }
1423 
1424 /*
1425  * Check all pages in range, recoreded as memory resource, are isolated.
1426  */
1427 static int
1428 check_pages_isolated_cb(unsigned long start_pfn, unsigned long nr_pages,
1429 			void *data)
1430 {
1431 	int ret;
1432 	long offlined = *(long *)data;
1433 	ret = test_pages_isolated(start_pfn, start_pfn + nr_pages, true);
1434 	offlined = nr_pages;
1435 	if (!ret)
1436 		*(long *)data += offlined;
1437 	return ret;
1438 }
1439 
1440 static long
1441 check_pages_isolated(unsigned long start_pfn, unsigned long end_pfn)
1442 {
1443 	long offlined = 0;
1444 	int ret;
1445 
1446 	ret = walk_system_ram_range(start_pfn, end_pfn - start_pfn, &offlined,
1447 			check_pages_isolated_cb);
1448 	if (ret < 0)
1449 		offlined = (long)ret;
1450 	return offlined;
1451 }
1452 
1453 #ifdef CONFIG_MOVABLE_NODE
1454 /*
1455  * When CONFIG_MOVABLE_NODE, we permit offlining of a node which doesn't have
1456  * normal memory.
1457  */
1458 static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
1459 {
1460 	return true;
1461 }
1462 #else /* CONFIG_MOVABLE_NODE */
1463 /* ensure the node has NORMAL memory if it is still online */
1464 static bool can_offline_normal(struct zone *zone, unsigned long nr_pages)
1465 {
1466 	struct pglist_data *pgdat = zone->zone_pgdat;
1467 	unsigned long present_pages = 0;
1468 	enum zone_type zt;
1469 
1470 	for (zt = 0; zt <= ZONE_NORMAL; zt++)
1471 		present_pages += pgdat->node_zones[zt].present_pages;
1472 
1473 	if (present_pages > nr_pages)
1474 		return true;
1475 
1476 	present_pages = 0;
1477 	for (; zt <= ZONE_MOVABLE; zt++)
1478 		present_pages += pgdat->node_zones[zt].present_pages;
1479 
1480 	/*
1481 	 * we can't offline the last normal memory until all
1482 	 * higher memory is offlined.
1483 	 */
1484 	return present_pages == 0;
1485 }
1486 #endif /* CONFIG_MOVABLE_NODE */
1487 
1488 static int __init cmdline_parse_movable_node(char *p)
1489 {
1490 #ifdef CONFIG_MOVABLE_NODE
1491 	/*
1492 	 * Memory used by the kernel cannot be hot-removed because Linux
1493 	 * cannot migrate the kernel pages. When memory hotplug is
1494 	 * enabled, we should prevent memblock from allocating memory
1495 	 * for the kernel.
1496 	 *
1497 	 * ACPI SRAT records all hotpluggable memory ranges. But before
1498 	 * SRAT is parsed, we don't know about it.
1499 	 *
1500 	 * The kernel image is loaded into memory at very early time. We
1501 	 * cannot prevent this anyway. So on NUMA system, we set any
1502 	 * node the kernel resides in as un-hotpluggable.
1503 	 *
1504 	 * Since on modern servers, one node could have double-digit
1505 	 * gigabytes memory, we can assume the memory around the kernel
1506 	 * image is also un-hotpluggable. So before SRAT is parsed, just
1507 	 * allocate memory near the kernel image to try the best to keep
1508 	 * the kernel away from hotpluggable memory.
1509 	 */
1510 	memblock_set_bottom_up(true);
1511 	movable_node_enabled = true;
1512 #else
1513 	pr_warn("movable_node option not supported\n");
1514 #endif
1515 	return 0;
1516 }
1517 early_param("movable_node", cmdline_parse_movable_node);
1518 
1519 /* check which state of node_states will be changed when offline memory */
1520 static void node_states_check_changes_offline(unsigned long nr_pages,
1521 		struct zone *zone, struct memory_notify *arg)
1522 {
1523 	struct pglist_data *pgdat = zone->zone_pgdat;
1524 	unsigned long present_pages = 0;
1525 	enum zone_type zt, zone_last = ZONE_NORMAL;
1526 
1527 	/*
1528 	 * If we have HIGHMEM or movable node, node_states[N_NORMAL_MEMORY]
1529 	 * contains nodes which have zones of 0...ZONE_NORMAL,
1530 	 * set zone_last to ZONE_NORMAL.
1531 	 *
1532 	 * If we don't have HIGHMEM nor movable node,
1533 	 * node_states[N_NORMAL_MEMORY] contains nodes which have zones of
1534 	 * 0...ZONE_MOVABLE, set zone_last to ZONE_MOVABLE.
1535 	 */
1536 	if (N_MEMORY == N_NORMAL_MEMORY)
1537 		zone_last = ZONE_MOVABLE;
1538 
1539 	/*
1540 	 * check whether node_states[N_NORMAL_MEMORY] will be changed.
1541 	 * If the memory to be offline is in a zone of 0...zone_last,
1542 	 * and it is the last present memory, 0...zone_last will
1543 	 * become empty after offline , thus we can determind we will
1544 	 * need to clear the node from node_states[N_NORMAL_MEMORY].
1545 	 */
1546 	for (zt = 0; zt <= zone_last; zt++)
1547 		present_pages += pgdat->node_zones[zt].present_pages;
1548 	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
1549 		arg->status_change_nid_normal = zone_to_nid(zone);
1550 	else
1551 		arg->status_change_nid_normal = -1;
1552 
1553 #ifdef CONFIG_HIGHMEM
1554 	/*
1555 	 * If we have movable node, node_states[N_HIGH_MEMORY]
1556 	 * contains nodes which have zones of 0...ZONE_HIGHMEM,
1557 	 * set zone_last to ZONE_HIGHMEM.
1558 	 *
1559 	 * If we don't have movable node, node_states[N_NORMAL_MEMORY]
1560 	 * contains nodes which have zones of 0...ZONE_MOVABLE,
1561 	 * set zone_last to ZONE_MOVABLE.
1562 	 */
1563 	zone_last = ZONE_HIGHMEM;
1564 	if (N_MEMORY == N_HIGH_MEMORY)
1565 		zone_last = ZONE_MOVABLE;
1566 
1567 	for (; zt <= zone_last; zt++)
1568 		present_pages += pgdat->node_zones[zt].present_pages;
1569 	if (zone_idx(zone) <= zone_last && nr_pages >= present_pages)
1570 		arg->status_change_nid_high = zone_to_nid(zone);
1571 	else
1572 		arg->status_change_nid_high = -1;
1573 #else
1574 	arg->status_change_nid_high = arg->status_change_nid_normal;
1575 #endif
1576 
1577 	/*
1578 	 * node_states[N_HIGH_MEMORY] contains nodes which have 0...ZONE_MOVABLE
1579 	 */
1580 	zone_last = ZONE_MOVABLE;
1581 
1582 	/*
1583 	 * check whether node_states[N_HIGH_MEMORY] will be changed
1584 	 * If we try to offline the last present @nr_pages from the node,
1585 	 * we can determind we will need to clear the node from
1586 	 * node_states[N_HIGH_MEMORY].
1587 	 */
1588 	for (; zt <= zone_last; zt++)
1589 		present_pages += pgdat->node_zones[zt].present_pages;
1590 	if (nr_pages >= present_pages)
1591 		arg->status_change_nid = zone_to_nid(zone);
1592 	else
1593 		arg->status_change_nid = -1;
1594 }
1595 
1596 static void node_states_clear_node(int node, struct memory_notify *arg)
1597 {
1598 	if (arg->status_change_nid_normal >= 0)
1599 		node_clear_state(node, N_NORMAL_MEMORY);
1600 
1601 	if ((N_MEMORY != N_NORMAL_MEMORY) &&
1602 	    (arg->status_change_nid_high >= 0))
1603 		node_clear_state(node, N_HIGH_MEMORY);
1604 
1605 	if ((N_MEMORY != N_HIGH_MEMORY) &&
1606 	    (arg->status_change_nid >= 0))
1607 		node_clear_state(node, N_MEMORY);
1608 }
1609 
1610 static int __ref __offline_pages(unsigned long start_pfn,
1611 		  unsigned long end_pfn, unsigned long timeout)
1612 {
1613 	unsigned long pfn, nr_pages, expire;
1614 	long offlined_pages;
1615 	int ret, drain, retry_max, node;
1616 	unsigned long flags;
1617 	struct zone *zone;
1618 	struct memory_notify arg;
1619 
1620 	/* at least, alignment against pageblock is necessary */
1621 	if (!IS_ALIGNED(start_pfn, pageblock_nr_pages))
1622 		return -EINVAL;
1623 	if (!IS_ALIGNED(end_pfn, pageblock_nr_pages))
1624 		return -EINVAL;
1625 	/* This makes hotplug much easier...and readable.
1626 	   we assume this for now. .*/
1627 	if (!test_pages_in_a_zone(start_pfn, end_pfn))
1628 		return -EINVAL;
1629 
1630 	mem_hotplug_begin();
1631 
1632 	zone = page_zone(pfn_to_page(start_pfn));
1633 	node = zone_to_nid(zone);
1634 	nr_pages = end_pfn - start_pfn;
1635 
1636 	ret = -EINVAL;
1637 	if (zone_idx(zone) <= ZONE_NORMAL && !can_offline_normal(zone, nr_pages))
1638 		goto out;
1639 
1640 	/* set above range as isolated */
1641 	ret = start_isolate_page_range(start_pfn, end_pfn,
1642 				       MIGRATE_MOVABLE, true);
1643 	if (ret)
1644 		goto out;
1645 
1646 	arg.start_pfn = start_pfn;
1647 	arg.nr_pages = nr_pages;
1648 	node_states_check_changes_offline(nr_pages, zone, &arg);
1649 
1650 	ret = memory_notify(MEM_GOING_OFFLINE, &arg);
1651 	ret = notifier_to_errno(ret);
1652 	if (ret)
1653 		goto failed_removal;
1654 
1655 	pfn = start_pfn;
1656 	expire = jiffies + timeout;
1657 	drain = 0;
1658 	retry_max = 5;
1659 repeat:
1660 	/* start memory hot removal */
1661 	ret = -EAGAIN;
1662 	if (time_after(jiffies, expire))
1663 		goto failed_removal;
1664 	ret = -EINTR;
1665 	if (signal_pending(current))
1666 		goto failed_removal;
1667 	ret = 0;
1668 	if (drain) {
1669 		lru_add_drain_all();
1670 		cond_resched();
1671 		drain_all_pages();
1672 	}
1673 
1674 	pfn = scan_movable_pages(start_pfn, end_pfn);
1675 	if (pfn) { /* We have movable pages */
1676 		ret = do_migrate_range(pfn, end_pfn);
1677 		if (!ret) {
1678 			drain = 1;
1679 			goto repeat;
1680 		} else {
1681 			if (ret < 0)
1682 				if (--retry_max == 0)
1683 					goto failed_removal;
1684 			yield();
1685 			drain = 1;
1686 			goto repeat;
1687 		}
1688 	}
1689 	/* drain all zone's lru pagevec, this is asynchronous... */
1690 	lru_add_drain_all();
1691 	yield();
1692 	/* drain pcp pages, this is synchronous. */
1693 	drain_all_pages();
1694 	/*
1695 	 * dissolve free hugepages in the memory block before doing offlining
1696 	 * actually in order to make hugetlbfs's object counting consistent.
1697 	 */
1698 	dissolve_free_huge_pages(start_pfn, end_pfn);
1699 	/* check again */
1700 	offlined_pages = check_pages_isolated(start_pfn, end_pfn);
1701 	if (offlined_pages < 0) {
1702 		ret = -EBUSY;
1703 		goto failed_removal;
1704 	}
1705 	printk(KERN_INFO "Offlined Pages %ld\n", offlined_pages);
1706 	/* Ok, all of our target is isolated.
1707 	   We cannot do rollback at this point. */
1708 	offline_isolated_pages(start_pfn, end_pfn);
1709 	/* reset pagetype flags and makes migrate type to be MOVABLE */
1710 	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1711 	/* removal success */
1712 	adjust_managed_page_count(pfn_to_page(start_pfn), -offlined_pages);
1713 	zone->present_pages -= offlined_pages;
1714 
1715 	pgdat_resize_lock(zone->zone_pgdat, &flags);
1716 	zone->zone_pgdat->node_present_pages -= offlined_pages;
1717 	pgdat_resize_unlock(zone->zone_pgdat, &flags);
1718 
1719 	init_per_zone_wmark_min();
1720 
1721 	if (!populated_zone(zone)) {
1722 		zone_pcp_reset(zone);
1723 		mutex_lock(&zonelists_mutex);
1724 		build_all_zonelists(NULL, NULL);
1725 		mutex_unlock(&zonelists_mutex);
1726 	} else
1727 		zone_pcp_update(zone);
1728 
1729 	node_states_clear_node(node, &arg);
1730 	if (arg.status_change_nid >= 0)
1731 		kswapd_stop(node);
1732 
1733 	vm_total_pages = nr_free_pagecache_pages();
1734 	writeback_set_ratelimit();
1735 
1736 	memory_notify(MEM_OFFLINE, &arg);
1737 	mem_hotplug_done();
1738 	return 0;
1739 
1740 failed_removal:
1741 	printk(KERN_INFO "memory offlining [mem %#010llx-%#010llx] failed\n",
1742 	       (unsigned long long) start_pfn << PAGE_SHIFT,
1743 	       ((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
1744 	memory_notify(MEM_CANCEL_OFFLINE, &arg);
1745 	/* pushback to free area */
1746 	undo_isolate_page_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1747 
1748 out:
1749 	mem_hotplug_done();
1750 	return ret;
1751 }
1752 
1753 int offline_pages(unsigned long start_pfn, unsigned long nr_pages)
1754 {
1755 	return __offline_pages(start_pfn, start_pfn + nr_pages, 120 * HZ);
1756 }
1757 #endif /* CONFIG_MEMORY_HOTREMOVE */
1758 
1759 /**
1760  * walk_memory_range - walks through all mem sections in [start_pfn, end_pfn)
1761  * @start_pfn: start pfn of the memory range
1762  * @end_pfn: end pfn of the memory range
1763  * @arg: argument passed to func
1764  * @func: callback for each memory section walked
1765  *
1766  * This function walks through all present mem sections in range
1767  * [start_pfn, end_pfn) and call func on each mem section.
1768  *
1769  * Returns the return value of func.
1770  */
1771 int walk_memory_range(unsigned long start_pfn, unsigned long end_pfn,
1772 		void *arg, int (*func)(struct memory_block *, void *))
1773 {
1774 	struct memory_block *mem = NULL;
1775 	struct mem_section *section;
1776 	unsigned long pfn, section_nr;
1777 	int ret;
1778 
1779 	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
1780 		section_nr = pfn_to_section_nr(pfn);
1781 		if (!present_section_nr(section_nr))
1782 			continue;
1783 
1784 		section = __nr_to_section(section_nr);
1785 		/* same memblock? */
1786 		if (mem)
1787 			if ((section_nr >= mem->start_section_nr) &&
1788 			    (section_nr <= mem->end_section_nr))
1789 				continue;
1790 
1791 		mem = find_memory_block_hinted(section, mem);
1792 		if (!mem)
1793 			continue;
1794 
1795 		ret = func(mem, arg);
1796 		if (ret) {
1797 			kobject_put(&mem->dev.kobj);
1798 			return ret;
1799 		}
1800 	}
1801 
1802 	if (mem)
1803 		kobject_put(&mem->dev.kobj);
1804 
1805 	return 0;
1806 }
1807 
1808 #ifdef CONFIG_MEMORY_HOTREMOVE
1809 static int check_memblock_offlined_cb(struct memory_block *mem, void *arg)
1810 {
1811 	int ret = !is_memblock_offlined(mem);
1812 
1813 	if (unlikely(ret)) {
1814 		phys_addr_t beginpa, endpa;
1815 
1816 		beginpa = PFN_PHYS(section_nr_to_pfn(mem->start_section_nr));
1817 		endpa = PFN_PHYS(section_nr_to_pfn(mem->end_section_nr + 1))-1;
1818 		pr_warn("removing memory fails, because memory "
1819 			"[%pa-%pa] is onlined\n",
1820 			&beginpa, &endpa);
1821 	}
1822 
1823 	return ret;
1824 }
1825 
1826 static int check_cpu_on_node(pg_data_t *pgdat)
1827 {
1828 	int cpu;
1829 
1830 	for_each_present_cpu(cpu) {
1831 		if (cpu_to_node(cpu) == pgdat->node_id)
1832 			/*
1833 			 * the cpu on this node isn't removed, and we can't
1834 			 * offline this node.
1835 			 */
1836 			return -EBUSY;
1837 	}
1838 
1839 	return 0;
1840 }
1841 
1842 static void unmap_cpu_on_node(pg_data_t *pgdat)
1843 {
1844 #ifdef CONFIG_ACPI_NUMA
1845 	int cpu;
1846 
1847 	for_each_possible_cpu(cpu)
1848 		if (cpu_to_node(cpu) == pgdat->node_id)
1849 			numa_clear_node(cpu);
1850 #endif
1851 }
1852 
1853 static int check_and_unmap_cpu_on_node(pg_data_t *pgdat)
1854 {
1855 	int ret;
1856 
1857 	ret = check_cpu_on_node(pgdat);
1858 	if (ret)
1859 		return ret;
1860 
1861 	/*
1862 	 * the node will be offlined when we come here, so we can clear
1863 	 * the cpu_to_node() now.
1864 	 */
1865 
1866 	unmap_cpu_on_node(pgdat);
1867 	return 0;
1868 }
1869 
1870 /**
1871  * try_offline_node
1872  *
1873  * Offline a node if all memory sections and cpus of the node are removed.
1874  *
1875  * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1876  * and online/offline operations before this call.
1877  */
1878 void try_offline_node(int nid)
1879 {
1880 	pg_data_t *pgdat = NODE_DATA(nid);
1881 	unsigned long start_pfn = pgdat->node_start_pfn;
1882 	unsigned long end_pfn = start_pfn + pgdat->node_spanned_pages;
1883 	unsigned long pfn;
1884 	struct page *pgdat_page = virt_to_page(pgdat);
1885 	int i;
1886 
1887 	for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
1888 		unsigned long section_nr = pfn_to_section_nr(pfn);
1889 
1890 		if (!present_section_nr(section_nr))
1891 			continue;
1892 
1893 		if (pfn_to_nid(pfn) != nid)
1894 			continue;
1895 
1896 		/*
1897 		 * some memory sections of this node are not removed, and we
1898 		 * can't offline node now.
1899 		 */
1900 		return;
1901 	}
1902 
1903 	if (check_and_unmap_cpu_on_node(pgdat))
1904 		return;
1905 
1906 	/*
1907 	 * all memory/cpu of this node are removed, we can offline this
1908 	 * node now.
1909 	 */
1910 	node_set_offline(nid);
1911 	unregister_one_node(nid);
1912 
1913 	if (!PageSlab(pgdat_page) && !PageCompound(pgdat_page))
1914 		/* node data is allocated from boot memory */
1915 		return;
1916 
1917 	/* free waittable in each zone */
1918 	for (i = 0; i < MAX_NR_ZONES; i++) {
1919 		struct zone *zone = pgdat->node_zones + i;
1920 
1921 		/*
1922 		 * wait_table may be allocated from boot memory,
1923 		 * here only free if it's allocated by vmalloc.
1924 		 */
1925 		if (is_vmalloc_addr(zone->wait_table))
1926 			vfree(zone->wait_table);
1927 	}
1928 
1929 	/*
1930 	 * Since there is no way to guarentee the address of pgdat/zone is not
1931 	 * on stack of any kernel threads or used by other kernel objects
1932 	 * without reference counting or other symchronizing method, do not
1933 	 * reset node_data and free pgdat here. Just reset it to 0 and reuse
1934 	 * the memory when the node is online again.
1935 	 */
1936 	memset(pgdat, 0, sizeof(*pgdat));
1937 }
1938 EXPORT_SYMBOL(try_offline_node);
1939 
1940 /**
1941  * remove_memory
1942  *
1943  * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
1944  * and online/offline operations before this call, as required by
1945  * try_offline_node().
1946  */
1947 void __ref remove_memory(int nid, u64 start, u64 size)
1948 {
1949 	int ret;
1950 
1951 	BUG_ON(check_hotplug_memory_range(start, size));
1952 
1953 	mem_hotplug_begin();
1954 
1955 	/*
1956 	 * All memory blocks must be offlined before removing memory.  Check
1957 	 * whether all memory blocks in question are offline and trigger a BUG()
1958 	 * if this is not the case.
1959 	 */
1960 	ret = walk_memory_range(PFN_DOWN(start), PFN_UP(start + size - 1), NULL,
1961 				check_memblock_offlined_cb);
1962 	if (ret)
1963 		BUG();
1964 
1965 	/* remove memmap entry */
1966 	firmware_map_remove(start, start + size, "System RAM");
1967 
1968 	arch_remove_memory(start, size);
1969 
1970 	try_offline_node(nid);
1971 
1972 	mem_hotplug_done();
1973 }
1974 EXPORT_SYMBOL_GPL(remove_memory);
1975 #endif /* CONFIG_MEMORY_HOTREMOVE */
1976