xref: /openbmc/linux/drivers/xen/balloon.c (revision 1c2dd16a)
1 /******************************************************************************
2  * Xen balloon driver - enables returning/claiming memory to/from Xen.
3  *
4  * Copyright (c) 2003, B Dragovic
5  * Copyright (c) 2003-2004, M Williamson, K Fraser
6  * Copyright (c) 2005 Dan M. Smith, IBM Corporation
7  * Copyright (c) 2010 Daniel Kiper
8  *
9  * Memory hotplug support was written by Daniel Kiper. Work on
10  * it was sponsored by Google under Google Summer of Code 2010
11  * program. Jeremy Fitzhardinge from Citrix was the mentor for
12  * this project.
13  *
14  * This program is free software; you can redistribute it and/or
15  * modify it under the terms of the GNU General Public License version 2
16  * as published by the Free Software Foundation; or, when distributed
17  * separately from the Linux kernel or incorporated into other
18  * software packages, subject to the following license:
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining a copy
21  * of this source file (the "Software"), to deal in the Software without
22  * restriction, including without limitation the rights to use, copy, modify,
23  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
24  * and to permit persons to whom the Software is furnished to do so, subject to
25  * the following conditions:
26  *
27  * The above copyright notice and this permission notice shall be included in
28  * all copies or substantial portions of the Software.
29  *
30  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
31  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
32  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
33  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
34  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
35  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
36  * IN THE SOFTWARE.
37  */
38 
39 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
40 
41 #include <linux/cpu.h>
42 #include <linux/kernel.h>
43 #include <linux/sched.h>
44 #include <linux/cred.h>
45 #include <linux/errno.h>
46 #include <linux/mm.h>
47 #include <linux/bootmem.h>
48 #include <linux/pagemap.h>
49 #include <linux/highmem.h>
50 #include <linux/mutex.h>
51 #include <linux/list.h>
52 #include <linux/gfp.h>
53 #include <linux/notifier.h>
54 #include <linux/memory.h>
55 #include <linux/memory_hotplug.h>
56 #include <linux/percpu-defs.h>
57 #include <linux/slab.h>
58 #include <linux/sysctl.h>
59 
60 #include <asm/page.h>
61 #include <asm/pgalloc.h>
62 #include <asm/pgtable.h>
63 #include <asm/tlb.h>
64 
65 #include <asm/xen/hypervisor.h>
66 #include <asm/xen/hypercall.h>
67 
68 #include <xen/xen.h>
69 #include <xen/interface/xen.h>
70 #include <xen/interface/memory.h>
71 #include <xen/balloon.h>
72 #include <xen/features.h>
73 #include <xen/page.h>
74 
75 static int xen_hotplug_unpopulated;
76 
77 #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
78 
79 static int zero;
80 static int one = 1;
81 
82 static struct ctl_table balloon_table[] = {
83 	{
84 		.procname	= "hotplug_unpopulated",
85 		.data		= &xen_hotplug_unpopulated,
86 		.maxlen		= sizeof(int),
87 		.mode		= 0644,
88 		.proc_handler	= proc_dointvec_minmax,
89 		.extra1         = &zero,
90 		.extra2         = &one,
91 	},
92 	{ }
93 };
94 
95 static struct ctl_table balloon_root[] = {
96 	{
97 		.procname	= "balloon",
98 		.mode		= 0555,
99 		.child		= balloon_table,
100 	},
101 	{ }
102 };
103 
104 static struct ctl_table xen_root[] = {
105 	{
106 		.procname	= "xen",
107 		.mode		= 0555,
108 		.child		= balloon_root,
109 	},
110 	{ }
111 };
112 
113 #endif
114 
115 /*
116  * Use one extent per PAGE_SIZE to avoid to break down the page into
117  * multiple frame.
118  */
119 #define EXTENT_ORDER (fls(XEN_PFN_PER_PAGE) - 1)
120 
121 /*
122  * balloon_process() state:
123  *
124  * BP_DONE: done or nothing to do,
125  * BP_WAIT: wait to be rescheduled,
126  * BP_EAGAIN: error, go to sleep,
127  * BP_ECANCELED: error, balloon operation canceled.
128  */
129 
130 enum bp_state {
131 	BP_DONE,
132 	BP_WAIT,
133 	BP_EAGAIN,
134 	BP_ECANCELED
135 };
136 
137 
138 static DEFINE_MUTEX(balloon_mutex);
139 
140 struct balloon_stats balloon_stats;
141 EXPORT_SYMBOL_GPL(balloon_stats);
142 
143 /* We increase/decrease in batches which fit in a page */
144 static xen_pfn_t frame_list[PAGE_SIZE / sizeof(xen_pfn_t)];
145 
146 
147 /* List of ballooned pages, threaded through the mem_map array. */
148 static LIST_HEAD(ballooned_pages);
149 static DECLARE_WAIT_QUEUE_HEAD(balloon_wq);
150 
151 /* Main work function, always executed in process context. */
152 static void balloon_process(struct work_struct *work);
153 static DECLARE_DELAYED_WORK(balloon_worker, balloon_process);
154 
155 /* When ballooning out (allocating memory to return to Xen) we don't really
156    want the kernel to try too hard since that can trigger the oom killer. */
157 #define GFP_BALLOON \
158 	(GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
159 
160 static void scrub_page(struct page *page)
161 {
162 #ifdef CONFIG_XEN_SCRUB_PAGES
163 	clear_highpage(page);
164 #endif
165 }
166 
167 /* balloon_append: add the given page to the balloon. */
168 static void __balloon_append(struct page *page)
169 {
170 	/* Lowmem is re-populated first, so highmem pages go at list tail. */
171 	if (PageHighMem(page)) {
172 		list_add_tail(&page->lru, &ballooned_pages);
173 		balloon_stats.balloon_high++;
174 	} else {
175 		list_add(&page->lru, &ballooned_pages);
176 		balloon_stats.balloon_low++;
177 	}
178 	wake_up(&balloon_wq);
179 }
180 
181 static void balloon_append(struct page *page)
182 {
183 	__balloon_append(page);
184 }
185 
186 /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
187 static struct page *balloon_retrieve(bool require_lowmem)
188 {
189 	struct page *page;
190 
191 	if (list_empty(&ballooned_pages))
192 		return NULL;
193 
194 	page = list_entry(ballooned_pages.next, struct page, lru);
195 	if (require_lowmem && PageHighMem(page))
196 		return NULL;
197 	list_del(&page->lru);
198 
199 	if (PageHighMem(page))
200 		balloon_stats.balloon_high--;
201 	else
202 		balloon_stats.balloon_low--;
203 
204 	return page;
205 }
206 
207 static struct page *balloon_next_page(struct page *page)
208 {
209 	struct list_head *next = page->lru.next;
210 	if (next == &ballooned_pages)
211 		return NULL;
212 	return list_entry(next, struct page, lru);
213 }
214 
215 static enum bp_state update_schedule(enum bp_state state)
216 {
217 	if (state == BP_WAIT)
218 		return BP_WAIT;
219 
220 	if (state == BP_ECANCELED)
221 		return BP_ECANCELED;
222 
223 	if (state == BP_DONE) {
224 		balloon_stats.schedule_delay = 1;
225 		balloon_stats.retry_count = 1;
226 		return BP_DONE;
227 	}
228 
229 	++balloon_stats.retry_count;
230 
231 	if (balloon_stats.max_retry_count != RETRY_UNLIMITED &&
232 			balloon_stats.retry_count > balloon_stats.max_retry_count) {
233 		balloon_stats.schedule_delay = 1;
234 		balloon_stats.retry_count = 1;
235 		return BP_ECANCELED;
236 	}
237 
238 	balloon_stats.schedule_delay <<= 1;
239 
240 	if (balloon_stats.schedule_delay > balloon_stats.max_schedule_delay)
241 		balloon_stats.schedule_delay = balloon_stats.max_schedule_delay;
242 
243 	return BP_EAGAIN;
244 }
245 
246 #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
247 static void release_memory_resource(struct resource *resource)
248 {
249 	if (!resource)
250 		return;
251 
252 	/*
253 	 * No need to reset region to identity mapped since we now
254 	 * know that no I/O can be in this region
255 	 */
256 	release_resource(resource);
257 	kfree(resource);
258 }
259 
260 static struct resource *additional_memory_resource(phys_addr_t size)
261 {
262 	struct resource *res;
263 	int ret;
264 
265 	res = kzalloc(sizeof(*res), GFP_KERNEL);
266 	if (!res)
267 		return NULL;
268 
269 	res->name = "System RAM";
270 	res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
271 
272 	ret = allocate_resource(&iomem_resource, res,
273 				size, 0, -1,
274 				PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
275 	if (ret < 0) {
276 		pr_err("Cannot allocate new System RAM resource\n");
277 		kfree(res);
278 		return NULL;
279 	}
280 
281 #ifdef CONFIG_SPARSEMEM
282 	{
283 		unsigned long limit = 1UL << (MAX_PHYSMEM_BITS - PAGE_SHIFT);
284 		unsigned long pfn = res->start >> PAGE_SHIFT;
285 
286 		if (pfn > limit) {
287 			pr_err("New System RAM resource outside addressable RAM (%lu > %lu)\n",
288 			       pfn, limit);
289 			release_memory_resource(res);
290 			return NULL;
291 		}
292 	}
293 #endif
294 
295 	return res;
296 }
297 
298 static enum bp_state reserve_additional_memory(void)
299 {
300 	long credit;
301 	struct resource *resource;
302 	int nid, rc;
303 	unsigned long balloon_hotplug;
304 
305 	credit = balloon_stats.target_pages + balloon_stats.target_unpopulated
306 		- balloon_stats.total_pages;
307 
308 	/*
309 	 * Already hotplugged enough pages?  Wait for them to be
310 	 * onlined.
311 	 */
312 	if (credit <= 0)
313 		return BP_WAIT;
314 
315 	balloon_hotplug = round_up(credit, PAGES_PER_SECTION);
316 
317 	resource = additional_memory_resource(balloon_hotplug * PAGE_SIZE);
318 	if (!resource)
319 		goto err;
320 
321 	nid = memory_add_physaddr_to_nid(resource->start);
322 
323 #ifdef CONFIG_XEN_HAVE_PVMMU
324 	/*
325 	 * We don't support PV MMU when Linux and Xen is using
326 	 * different page granularity.
327 	 */
328 	BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
329 
330         /*
331          * add_memory() will build page tables for the new memory so
332          * the p2m must contain invalid entries so the correct
333          * non-present PTEs will be written.
334          *
335          * If a failure occurs, the original (identity) p2m entries
336          * are not restored since this region is now known not to
337          * conflict with any devices.
338          */
339 	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
340 		unsigned long pfn, i;
341 
342 		pfn = PFN_DOWN(resource->start);
343 		for (i = 0; i < balloon_hotplug; i++) {
344 			if (!set_phys_to_machine(pfn + i, INVALID_P2M_ENTRY)) {
345 				pr_warn("set_phys_to_machine() failed, no memory added\n");
346 				goto err;
347 			}
348                 }
349 	}
350 #endif
351 
352 	/*
353 	 * add_memory_resource() will call online_pages() which in its turn
354 	 * will call xen_online_page() callback causing deadlock if we don't
355 	 * release balloon_mutex here. Unlocking here is safe because the
356 	 * callers drop the mutex before trying again.
357 	 */
358 	mutex_unlock(&balloon_mutex);
359 	rc = add_memory_resource(nid, resource, memhp_auto_online);
360 	mutex_lock(&balloon_mutex);
361 
362 	if (rc) {
363 		pr_warn("Cannot add additional memory (%i)\n", rc);
364 		goto err;
365 	}
366 
367 	balloon_stats.total_pages += balloon_hotplug;
368 
369 	return BP_WAIT;
370   err:
371 	release_memory_resource(resource);
372 	return BP_ECANCELED;
373 }
374 
375 static void xen_online_page(struct page *page)
376 {
377 	__online_page_set_limits(page);
378 
379 	mutex_lock(&balloon_mutex);
380 
381 	__balloon_append(page);
382 
383 	mutex_unlock(&balloon_mutex);
384 }
385 
386 static int xen_memory_notifier(struct notifier_block *nb, unsigned long val, void *v)
387 {
388 	if (val == MEM_ONLINE)
389 		schedule_delayed_work(&balloon_worker, 0);
390 
391 	return NOTIFY_OK;
392 }
393 
394 static struct notifier_block xen_memory_nb = {
395 	.notifier_call = xen_memory_notifier,
396 	.priority = 0
397 };
398 #else
399 static enum bp_state reserve_additional_memory(void)
400 {
401 	balloon_stats.target_pages = balloon_stats.current_pages;
402 	return BP_ECANCELED;
403 }
404 #endif /* CONFIG_XEN_BALLOON_MEMORY_HOTPLUG */
405 
406 static long current_credit(void)
407 {
408 	return balloon_stats.target_pages - balloon_stats.current_pages;
409 }
410 
411 static bool balloon_is_inflated(void)
412 {
413 	return balloon_stats.balloon_low || balloon_stats.balloon_high;
414 }
415 
416 static enum bp_state increase_reservation(unsigned long nr_pages)
417 {
418 	int rc;
419 	unsigned long i;
420 	struct page   *page;
421 	struct xen_memory_reservation reservation = {
422 		.address_bits = 0,
423 		.extent_order = EXTENT_ORDER,
424 		.domid        = DOMID_SELF
425 	};
426 
427 	if (nr_pages > ARRAY_SIZE(frame_list))
428 		nr_pages = ARRAY_SIZE(frame_list);
429 
430 	page = list_first_entry_or_null(&ballooned_pages, struct page, lru);
431 	for (i = 0; i < nr_pages; i++) {
432 		if (!page) {
433 			nr_pages = i;
434 			break;
435 		}
436 
437 		/* XENMEM_populate_physmap requires a PFN based on Xen
438 		 * granularity.
439 		 */
440 		frame_list[i] = page_to_xen_pfn(page);
441 		page = balloon_next_page(page);
442 	}
443 
444 	set_xen_guest_handle(reservation.extent_start, frame_list);
445 	reservation.nr_extents = nr_pages;
446 	rc = HYPERVISOR_memory_op(XENMEM_populate_physmap, &reservation);
447 	if (rc <= 0)
448 		return BP_EAGAIN;
449 
450 	for (i = 0; i < rc; i++) {
451 		page = balloon_retrieve(false);
452 		BUG_ON(page == NULL);
453 
454 #ifdef CONFIG_XEN_HAVE_PVMMU
455 		/*
456 		 * We don't support PV MMU when Linux and Xen is using
457 		 * different page granularity.
458 		 */
459 		BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
460 
461 		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
462 			unsigned long pfn = page_to_pfn(page);
463 
464 			set_phys_to_machine(pfn, frame_list[i]);
465 
466 			/* Link back into the page tables if not highmem. */
467 			if (!PageHighMem(page)) {
468 				int ret;
469 				ret = HYPERVISOR_update_va_mapping(
470 						(unsigned long)__va(pfn << PAGE_SHIFT),
471 						mfn_pte(frame_list[i], PAGE_KERNEL),
472 						0);
473 				BUG_ON(ret);
474 			}
475 		}
476 #endif
477 
478 		/* Relinquish the page back to the allocator. */
479 		free_reserved_page(page);
480 	}
481 
482 	balloon_stats.current_pages += rc;
483 
484 	return BP_DONE;
485 }
486 
487 static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
488 {
489 	enum bp_state state = BP_DONE;
490 	unsigned long i;
491 	struct page *page, *tmp;
492 	int ret;
493 	struct xen_memory_reservation reservation = {
494 		.address_bits = 0,
495 		.extent_order = EXTENT_ORDER,
496 		.domid        = DOMID_SELF
497 	};
498 	LIST_HEAD(pages);
499 
500 	if (nr_pages > ARRAY_SIZE(frame_list))
501 		nr_pages = ARRAY_SIZE(frame_list);
502 
503 	for (i = 0; i < nr_pages; i++) {
504 		page = alloc_page(gfp);
505 		if (page == NULL) {
506 			nr_pages = i;
507 			state = BP_EAGAIN;
508 			break;
509 		}
510 		adjust_managed_page_count(page, -1);
511 		scrub_page(page);
512 		list_add(&page->lru, &pages);
513 	}
514 
515 	/*
516 	 * Ensure that ballooned highmem pages don't have kmaps.
517 	 *
518 	 * Do this before changing the p2m as kmap_flush_unused()
519 	 * reads PTEs to obtain pages (and hence needs the original
520 	 * p2m entry).
521 	 */
522 	kmap_flush_unused();
523 
524 	/*
525 	 * Setup the frame, update direct mapping, invalidate P2M,
526 	 * and add to balloon.
527 	 */
528 	i = 0;
529 	list_for_each_entry_safe(page, tmp, &pages, lru) {
530 		/* XENMEM_decrease_reservation requires a GFN */
531 		frame_list[i++] = xen_page_to_gfn(page);
532 
533 #ifdef CONFIG_XEN_HAVE_PVMMU
534 		/*
535 		 * We don't support PV MMU when Linux and Xen is using
536 		 * different page granularity.
537 		 */
538 		BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
539 
540 		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
541 			unsigned long pfn = page_to_pfn(page);
542 
543 			if (!PageHighMem(page)) {
544 				ret = HYPERVISOR_update_va_mapping(
545 						(unsigned long)__va(pfn << PAGE_SHIFT),
546 						__pte_ma(0), 0);
547 				BUG_ON(ret);
548 			}
549 			__set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
550 		}
551 #endif
552 		list_del(&page->lru);
553 
554 		balloon_append(page);
555 	}
556 
557 	flush_tlb_all();
558 
559 	set_xen_guest_handle(reservation.extent_start, frame_list);
560 	reservation.nr_extents   = nr_pages;
561 	ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation);
562 	BUG_ON(ret != nr_pages);
563 
564 	balloon_stats.current_pages -= nr_pages;
565 
566 	return state;
567 }
568 
569 /*
570  * As this is a work item it is guaranteed to run as a single instance only.
571  * We may of course race updates of the target counts (which are protected
572  * by the balloon lock), or with changes to the Xen hard limit, but we will
573  * recover from these in time.
574  */
575 static void balloon_process(struct work_struct *work)
576 {
577 	enum bp_state state = BP_DONE;
578 	long credit;
579 
580 
581 	do {
582 		mutex_lock(&balloon_mutex);
583 
584 		credit = current_credit();
585 
586 		if (credit > 0) {
587 			if (balloon_is_inflated())
588 				state = increase_reservation(credit);
589 			else
590 				state = reserve_additional_memory();
591 		}
592 
593 		if (credit < 0)
594 			state = decrease_reservation(-credit, GFP_BALLOON);
595 
596 		state = update_schedule(state);
597 
598 		mutex_unlock(&balloon_mutex);
599 
600 		cond_resched();
601 
602 	} while (credit && state == BP_DONE);
603 
604 	/* Schedule more work if there is some still to be done. */
605 	if (state == BP_EAGAIN)
606 		schedule_delayed_work(&balloon_worker, balloon_stats.schedule_delay * HZ);
607 }
608 
609 /* Resets the Xen limit, sets new target, and kicks off processing. */
610 void balloon_set_new_target(unsigned long target)
611 {
612 	/* No need for lock. Not read-modify-write updates. */
613 	balloon_stats.target_pages = target;
614 	schedule_delayed_work(&balloon_worker, 0);
615 }
616 EXPORT_SYMBOL_GPL(balloon_set_new_target);
617 
618 static int add_ballooned_pages(int nr_pages)
619 {
620 	enum bp_state st;
621 
622 	if (xen_hotplug_unpopulated) {
623 		st = reserve_additional_memory();
624 		if (st != BP_ECANCELED) {
625 			mutex_unlock(&balloon_mutex);
626 			wait_event(balloon_wq,
627 				   !list_empty(&ballooned_pages));
628 			mutex_lock(&balloon_mutex);
629 			return 0;
630 		}
631 	}
632 
633 	st = decrease_reservation(nr_pages, GFP_USER);
634 	if (st != BP_DONE)
635 		return -ENOMEM;
636 
637 	return 0;
638 }
639 
640 /**
641  * alloc_xenballooned_pages - get pages that have been ballooned out
642  * @nr_pages: Number of pages to get
643  * @pages: pages returned
644  * @return 0 on success, error otherwise
645  */
646 int alloc_xenballooned_pages(int nr_pages, struct page **pages)
647 {
648 	int pgno = 0;
649 	struct page *page;
650 	int ret;
651 
652 	mutex_lock(&balloon_mutex);
653 
654 	balloon_stats.target_unpopulated += nr_pages;
655 
656 	while (pgno < nr_pages) {
657 		page = balloon_retrieve(true);
658 		if (page) {
659 			pages[pgno++] = page;
660 #ifdef CONFIG_XEN_HAVE_PVMMU
661 			/*
662 			 * We don't support PV MMU when Linux and Xen is using
663 			 * different page granularity.
664 			 */
665 			BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
666 
667 			ret = xen_alloc_p2m_entry(page_to_pfn(page));
668 			if (ret < 0)
669 				goto out_undo;
670 #endif
671 		} else {
672 			ret = add_ballooned_pages(nr_pages - pgno);
673 			if (ret < 0)
674 				goto out_undo;
675 		}
676 	}
677 	mutex_unlock(&balloon_mutex);
678 	return 0;
679  out_undo:
680 	mutex_unlock(&balloon_mutex);
681 	free_xenballooned_pages(pgno, pages);
682 	return ret;
683 }
684 EXPORT_SYMBOL(alloc_xenballooned_pages);
685 
686 /**
687  * free_xenballooned_pages - return pages retrieved with get_ballooned_pages
688  * @nr_pages: Number of pages
689  * @pages: pages to return
690  */
691 void free_xenballooned_pages(int nr_pages, struct page **pages)
692 {
693 	int i;
694 
695 	mutex_lock(&balloon_mutex);
696 
697 	for (i = 0; i < nr_pages; i++) {
698 		if (pages[i])
699 			balloon_append(pages[i]);
700 	}
701 
702 	balloon_stats.target_unpopulated -= nr_pages;
703 
704 	/* The balloon may be too large now. Shrink it if needed. */
705 	if (current_credit())
706 		schedule_delayed_work(&balloon_worker, 0);
707 
708 	mutex_unlock(&balloon_mutex);
709 }
710 EXPORT_SYMBOL(free_xenballooned_pages);
711 
712 static void __init balloon_add_region(unsigned long start_pfn,
713 				      unsigned long pages)
714 {
715 	unsigned long pfn, extra_pfn_end;
716 	struct page *page;
717 
718 	/*
719 	 * If the amount of usable memory has been limited (e.g., with
720 	 * the 'mem' command line parameter), don't add pages beyond
721 	 * this limit.
722 	 */
723 	extra_pfn_end = min(max_pfn, start_pfn + pages);
724 
725 	for (pfn = start_pfn; pfn < extra_pfn_end; pfn++) {
726 		page = pfn_to_page(pfn);
727 		/* totalram_pages and totalhigh_pages do not
728 		   include the boot-time balloon extension, so
729 		   don't subtract from it. */
730 		__balloon_append(page);
731 	}
732 
733 	balloon_stats.total_pages += extra_pfn_end - start_pfn;
734 }
735 
736 static int __init balloon_init(void)
737 {
738 	int i;
739 
740 	if (!xen_domain())
741 		return -ENODEV;
742 
743 	pr_info("Initialising balloon driver\n");
744 
745 	balloon_stats.current_pages = xen_pv_domain()
746 		? min(xen_start_info->nr_pages - xen_released_pages, max_pfn)
747 		: get_num_physpages();
748 	balloon_stats.target_pages  = balloon_stats.current_pages;
749 	balloon_stats.balloon_low   = 0;
750 	balloon_stats.balloon_high  = 0;
751 	balloon_stats.total_pages   = balloon_stats.current_pages;
752 
753 	balloon_stats.schedule_delay = 1;
754 	balloon_stats.max_schedule_delay = 32;
755 	balloon_stats.retry_count = 1;
756 	balloon_stats.max_retry_count = RETRY_UNLIMITED;
757 
758 #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
759 	set_online_page_callback(&xen_online_page);
760 	register_memory_notifier(&xen_memory_nb);
761 	register_sysctl_table(xen_root);
762 #endif
763 
764 	/*
765 	 * Initialize the balloon with pages from the extra memory
766 	 * regions (see arch/x86/xen/setup.c).
767 	 */
768 	for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++)
769 		if (xen_extra_mem[i].n_pfns)
770 			balloon_add_region(xen_extra_mem[i].start_pfn,
771 					   xen_extra_mem[i].n_pfns);
772 
773 	return 0;
774 }
775 subsys_initcall(balloon_init);
776