xref: /openbmc/linux/mm/hugetlb.c (revision afb46f79)
1 /*
2  * Generic hugetlb support.
3  * (C) Nadia Yvette Chambers, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/module.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/compiler.h>
17 #include <linux/cpuset.h>
18 #include <linux/mutex.h>
19 #include <linux/bootmem.h>
20 #include <linux/sysfs.h>
21 #include <linux/slab.h>
22 #include <linux/rmap.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/page-isolation.h>
26 #include <linux/jhash.h>
27 
28 #include <asm/page.h>
29 #include <asm/pgtable.h>
30 #include <asm/tlb.h>
31 
32 #include <linux/io.h>
33 #include <linux/hugetlb.h>
34 #include <linux/hugetlb_cgroup.h>
35 #include <linux/node.h>
36 #include "internal.h"
37 
38 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
39 unsigned long hugepages_treat_as_movable;
40 
41 int hugetlb_max_hstate __read_mostly;
42 unsigned int default_hstate_idx;
43 struct hstate hstates[HUGE_MAX_HSTATE];
44 
45 __initdata LIST_HEAD(huge_boot_pages);
46 
47 /* for command line parsing */
48 static struct hstate * __initdata parsed_hstate;
49 static unsigned long __initdata default_hstate_max_huge_pages;
50 static unsigned long __initdata default_hstate_size;
51 
52 /*
53  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
54  * free_huge_pages, and surplus_huge_pages.
55  */
56 DEFINE_SPINLOCK(hugetlb_lock);
57 
58 /*
59  * Serializes faults on the same logical page.  This is used to
60  * prevent spurious OOMs when the hugepage pool is fully utilized.
61  */
62 static int num_fault_mutexes;
63 static struct mutex *htlb_fault_mutex_table ____cacheline_aligned_in_smp;
64 
65 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
66 {
67 	bool free = (spool->count == 0) && (spool->used_hpages == 0);
68 
69 	spin_unlock(&spool->lock);
70 
71 	/* If no pages are used, and no other handles to the subpool
72 	 * remain, free the subpool the subpool remain */
73 	if (free)
74 		kfree(spool);
75 }
76 
77 struct hugepage_subpool *hugepage_new_subpool(long nr_blocks)
78 {
79 	struct hugepage_subpool *spool;
80 
81 	spool = kmalloc(sizeof(*spool), GFP_KERNEL);
82 	if (!spool)
83 		return NULL;
84 
85 	spin_lock_init(&spool->lock);
86 	spool->count = 1;
87 	spool->max_hpages = nr_blocks;
88 	spool->used_hpages = 0;
89 
90 	return spool;
91 }
92 
93 void hugepage_put_subpool(struct hugepage_subpool *spool)
94 {
95 	spin_lock(&spool->lock);
96 	BUG_ON(!spool->count);
97 	spool->count--;
98 	unlock_or_release_subpool(spool);
99 }
100 
101 static int hugepage_subpool_get_pages(struct hugepage_subpool *spool,
102 				      long delta)
103 {
104 	int ret = 0;
105 
106 	if (!spool)
107 		return 0;
108 
109 	spin_lock(&spool->lock);
110 	if ((spool->used_hpages + delta) <= spool->max_hpages) {
111 		spool->used_hpages += delta;
112 	} else {
113 		ret = -ENOMEM;
114 	}
115 	spin_unlock(&spool->lock);
116 
117 	return ret;
118 }
119 
120 static void hugepage_subpool_put_pages(struct hugepage_subpool *spool,
121 				       long delta)
122 {
123 	if (!spool)
124 		return;
125 
126 	spin_lock(&spool->lock);
127 	spool->used_hpages -= delta;
128 	/* If hugetlbfs_put_super couldn't free spool due to
129 	* an outstanding quota reference, free it now. */
130 	unlock_or_release_subpool(spool);
131 }
132 
133 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
134 {
135 	return HUGETLBFS_SB(inode->i_sb)->spool;
136 }
137 
138 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
139 {
140 	return subpool_inode(file_inode(vma->vm_file));
141 }
142 
143 /*
144  * Region tracking -- allows tracking of reservations and instantiated pages
145  *                    across the pages in a mapping.
146  *
147  * The region data structures are embedded into a resv_map and
148  * protected by a resv_map's lock
149  */
150 struct file_region {
151 	struct list_head link;
152 	long from;
153 	long to;
154 };
155 
156 static long region_add(struct resv_map *resv, long f, long t)
157 {
158 	struct list_head *head = &resv->regions;
159 	struct file_region *rg, *nrg, *trg;
160 
161 	spin_lock(&resv->lock);
162 	/* Locate the region we are either in or before. */
163 	list_for_each_entry(rg, head, link)
164 		if (f <= rg->to)
165 			break;
166 
167 	/* Round our left edge to the current segment if it encloses us. */
168 	if (f > rg->from)
169 		f = rg->from;
170 
171 	/* Check for and consume any regions we now overlap with. */
172 	nrg = rg;
173 	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
174 		if (&rg->link == head)
175 			break;
176 		if (rg->from > t)
177 			break;
178 
179 		/* If this area reaches higher then extend our area to
180 		 * include it completely.  If this is not the first area
181 		 * which we intend to reuse, free it. */
182 		if (rg->to > t)
183 			t = rg->to;
184 		if (rg != nrg) {
185 			list_del(&rg->link);
186 			kfree(rg);
187 		}
188 	}
189 	nrg->from = f;
190 	nrg->to = t;
191 	spin_unlock(&resv->lock);
192 	return 0;
193 }
194 
195 static long region_chg(struct resv_map *resv, long f, long t)
196 {
197 	struct list_head *head = &resv->regions;
198 	struct file_region *rg, *nrg = NULL;
199 	long chg = 0;
200 
201 retry:
202 	spin_lock(&resv->lock);
203 	/* Locate the region we are before or in. */
204 	list_for_each_entry(rg, head, link)
205 		if (f <= rg->to)
206 			break;
207 
208 	/* If we are below the current region then a new region is required.
209 	 * Subtle, allocate a new region at the position but make it zero
210 	 * size such that we can guarantee to record the reservation. */
211 	if (&rg->link == head || t < rg->from) {
212 		if (!nrg) {
213 			spin_unlock(&resv->lock);
214 			nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
215 			if (!nrg)
216 				return -ENOMEM;
217 
218 			nrg->from = f;
219 			nrg->to   = f;
220 			INIT_LIST_HEAD(&nrg->link);
221 			goto retry;
222 		}
223 
224 		list_add(&nrg->link, rg->link.prev);
225 		chg = t - f;
226 		goto out_nrg;
227 	}
228 
229 	/* Round our left edge to the current segment if it encloses us. */
230 	if (f > rg->from)
231 		f = rg->from;
232 	chg = t - f;
233 
234 	/* Check for and consume any regions we now overlap with. */
235 	list_for_each_entry(rg, rg->link.prev, link) {
236 		if (&rg->link == head)
237 			break;
238 		if (rg->from > t)
239 			goto out;
240 
241 		/* We overlap with this area, if it extends further than
242 		 * us then we must extend ourselves.  Account for its
243 		 * existing reservation. */
244 		if (rg->to > t) {
245 			chg += rg->to - t;
246 			t = rg->to;
247 		}
248 		chg -= rg->to - rg->from;
249 	}
250 
251 out:
252 	spin_unlock(&resv->lock);
253 	/*  We already know we raced and no longer need the new region */
254 	kfree(nrg);
255 	return chg;
256 out_nrg:
257 	spin_unlock(&resv->lock);
258 	return chg;
259 }
260 
261 static long region_truncate(struct resv_map *resv, long end)
262 {
263 	struct list_head *head = &resv->regions;
264 	struct file_region *rg, *trg;
265 	long chg = 0;
266 
267 	spin_lock(&resv->lock);
268 	/* Locate the region we are either in or before. */
269 	list_for_each_entry(rg, head, link)
270 		if (end <= rg->to)
271 			break;
272 	if (&rg->link == head)
273 		goto out;
274 
275 	/* If we are in the middle of a region then adjust it. */
276 	if (end > rg->from) {
277 		chg = rg->to - end;
278 		rg->to = end;
279 		rg = list_entry(rg->link.next, typeof(*rg), link);
280 	}
281 
282 	/* Drop any remaining regions. */
283 	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
284 		if (&rg->link == head)
285 			break;
286 		chg += rg->to - rg->from;
287 		list_del(&rg->link);
288 		kfree(rg);
289 	}
290 
291 out:
292 	spin_unlock(&resv->lock);
293 	return chg;
294 }
295 
296 static long region_count(struct resv_map *resv, long f, long t)
297 {
298 	struct list_head *head = &resv->regions;
299 	struct file_region *rg;
300 	long chg = 0;
301 
302 	spin_lock(&resv->lock);
303 	/* Locate each segment we overlap with, and count that overlap. */
304 	list_for_each_entry(rg, head, link) {
305 		long seg_from;
306 		long seg_to;
307 
308 		if (rg->to <= f)
309 			continue;
310 		if (rg->from >= t)
311 			break;
312 
313 		seg_from = max(rg->from, f);
314 		seg_to = min(rg->to, t);
315 
316 		chg += seg_to - seg_from;
317 	}
318 	spin_unlock(&resv->lock);
319 
320 	return chg;
321 }
322 
323 /*
324  * Convert the address within this vma to the page offset within
325  * the mapping, in pagecache page units; huge pages here.
326  */
327 static pgoff_t vma_hugecache_offset(struct hstate *h,
328 			struct vm_area_struct *vma, unsigned long address)
329 {
330 	return ((address - vma->vm_start) >> huge_page_shift(h)) +
331 			(vma->vm_pgoff >> huge_page_order(h));
332 }
333 
334 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
335 				     unsigned long address)
336 {
337 	return vma_hugecache_offset(hstate_vma(vma), vma, address);
338 }
339 
340 /*
341  * Return the size of the pages allocated when backing a VMA. In the majority
342  * cases this will be same size as used by the page table entries.
343  */
344 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
345 {
346 	struct hstate *hstate;
347 
348 	if (!is_vm_hugetlb_page(vma))
349 		return PAGE_SIZE;
350 
351 	hstate = hstate_vma(vma);
352 
353 	return 1UL << huge_page_shift(hstate);
354 }
355 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
356 
357 /*
358  * Return the page size being used by the MMU to back a VMA. In the majority
359  * of cases, the page size used by the kernel matches the MMU size. On
360  * architectures where it differs, an architecture-specific version of this
361  * function is required.
362  */
363 #ifndef vma_mmu_pagesize
364 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
365 {
366 	return vma_kernel_pagesize(vma);
367 }
368 #endif
369 
370 /*
371  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
372  * bits of the reservation map pointer, which are always clear due to
373  * alignment.
374  */
375 #define HPAGE_RESV_OWNER    (1UL << 0)
376 #define HPAGE_RESV_UNMAPPED (1UL << 1)
377 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
378 
379 /*
380  * These helpers are used to track how many pages are reserved for
381  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
382  * is guaranteed to have their future faults succeed.
383  *
384  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
385  * the reserve counters are updated with the hugetlb_lock held. It is safe
386  * to reset the VMA at fork() time as it is not in use yet and there is no
387  * chance of the global counters getting corrupted as a result of the values.
388  *
389  * The private mapping reservation is represented in a subtly different
390  * manner to a shared mapping.  A shared mapping has a region map associated
391  * with the underlying file, this region map represents the backing file
392  * pages which have ever had a reservation assigned which this persists even
393  * after the page is instantiated.  A private mapping has a region map
394  * associated with the original mmap which is attached to all VMAs which
395  * reference it, this region map represents those offsets which have consumed
396  * reservation ie. where pages have been instantiated.
397  */
398 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
399 {
400 	return (unsigned long)vma->vm_private_data;
401 }
402 
403 static void set_vma_private_data(struct vm_area_struct *vma,
404 							unsigned long value)
405 {
406 	vma->vm_private_data = (void *)value;
407 }
408 
409 struct resv_map *resv_map_alloc(void)
410 {
411 	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
412 	if (!resv_map)
413 		return NULL;
414 
415 	kref_init(&resv_map->refs);
416 	spin_lock_init(&resv_map->lock);
417 	INIT_LIST_HEAD(&resv_map->regions);
418 
419 	return resv_map;
420 }
421 
422 void resv_map_release(struct kref *ref)
423 {
424 	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
425 
426 	/* Clear out any active regions before we release the map. */
427 	region_truncate(resv_map, 0);
428 	kfree(resv_map);
429 }
430 
431 static inline struct resv_map *inode_resv_map(struct inode *inode)
432 {
433 	return inode->i_mapping->private_data;
434 }
435 
436 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
437 {
438 	VM_BUG_ON(!is_vm_hugetlb_page(vma));
439 	if (vma->vm_flags & VM_MAYSHARE) {
440 		struct address_space *mapping = vma->vm_file->f_mapping;
441 		struct inode *inode = mapping->host;
442 
443 		return inode_resv_map(inode);
444 
445 	} else {
446 		return (struct resv_map *)(get_vma_private_data(vma) &
447 							~HPAGE_RESV_MASK);
448 	}
449 }
450 
451 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
452 {
453 	VM_BUG_ON(!is_vm_hugetlb_page(vma));
454 	VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
455 
456 	set_vma_private_data(vma, (get_vma_private_data(vma) &
457 				HPAGE_RESV_MASK) | (unsigned long)map);
458 }
459 
460 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
461 {
462 	VM_BUG_ON(!is_vm_hugetlb_page(vma));
463 	VM_BUG_ON(vma->vm_flags & VM_MAYSHARE);
464 
465 	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
466 }
467 
468 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
469 {
470 	VM_BUG_ON(!is_vm_hugetlb_page(vma));
471 
472 	return (get_vma_private_data(vma) & flag) != 0;
473 }
474 
475 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
476 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
477 {
478 	VM_BUG_ON(!is_vm_hugetlb_page(vma));
479 	if (!(vma->vm_flags & VM_MAYSHARE))
480 		vma->vm_private_data = (void *)0;
481 }
482 
483 /* Returns true if the VMA has associated reserve pages */
484 static int vma_has_reserves(struct vm_area_struct *vma, long chg)
485 {
486 	if (vma->vm_flags & VM_NORESERVE) {
487 		/*
488 		 * This address is already reserved by other process(chg == 0),
489 		 * so, we should decrement reserved count. Without decrementing,
490 		 * reserve count remains after releasing inode, because this
491 		 * allocated page will go into page cache and is regarded as
492 		 * coming from reserved pool in releasing step.  Currently, we
493 		 * don't have any other solution to deal with this situation
494 		 * properly, so add work-around here.
495 		 */
496 		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
497 			return 1;
498 		else
499 			return 0;
500 	}
501 
502 	/* Shared mappings always use reserves */
503 	if (vma->vm_flags & VM_MAYSHARE)
504 		return 1;
505 
506 	/*
507 	 * Only the process that called mmap() has reserves for
508 	 * private mappings.
509 	 */
510 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER))
511 		return 1;
512 
513 	return 0;
514 }
515 
516 static void enqueue_huge_page(struct hstate *h, struct page *page)
517 {
518 	int nid = page_to_nid(page);
519 	list_move(&page->lru, &h->hugepage_freelists[nid]);
520 	h->free_huge_pages++;
521 	h->free_huge_pages_node[nid]++;
522 }
523 
524 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
525 {
526 	struct page *page;
527 
528 	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
529 		if (!is_migrate_isolate_page(page))
530 			break;
531 	/*
532 	 * if 'non-isolated free hugepage' not found on the list,
533 	 * the allocation fails.
534 	 */
535 	if (&h->hugepage_freelists[nid] == &page->lru)
536 		return NULL;
537 	list_move(&page->lru, &h->hugepage_activelist);
538 	set_page_refcounted(page);
539 	h->free_huge_pages--;
540 	h->free_huge_pages_node[nid]--;
541 	return page;
542 }
543 
544 /* Movability of hugepages depends on migration support. */
545 static inline gfp_t htlb_alloc_mask(struct hstate *h)
546 {
547 	if (hugepages_treat_as_movable || hugepage_migration_support(h))
548 		return GFP_HIGHUSER_MOVABLE;
549 	else
550 		return GFP_HIGHUSER;
551 }
552 
553 static struct page *dequeue_huge_page_vma(struct hstate *h,
554 				struct vm_area_struct *vma,
555 				unsigned long address, int avoid_reserve,
556 				long chg)
557 {
558 	struct page *page = NULL;
559 	struct mempolicy *mpol;
560 	nodemask_t *nodemask;
561 	struct zonelist *zonelist;
562 	struct zone *zone;
563 	struct zoneref *z;
564 	unsigned int cpuset_mems_cookie;
565 
566 	/*
567 	 * A child process with MAP_PRIVATE mappings created by their parent
568 	 * have no page reserves. This check ensures that reservations are
569 	 * not "stolen". The child may still get SIGKILLed
570 	 */
571 	if (!vma_has_reserves(vma, chg) &&
572 			h->free_huge_pages - h->resv_huge_pages == 0)
573 		goto err;
574 
575 	/* If reserves cannot be used, ensure enough pages are in the pool */
576 	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
577 		goto err;
578 
579 retry_cpuset:
580 	cpuset_mems_cookie = read_mems_allowed_begin();
581 	zonelist = huge_zonelist(vma, address,
582 					htlb_alloc_mask(h), &mpol, &nodemask);
583 
584 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
585 						MAX_NR_ZONES - 1, nodemask) {
586 		if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) {
587 			page = dequeue_huge_page_node(h, zone_to_nid(zone));
588 			if (page) {
589 				if (avoid_reserve)
590 					break;
591 				if (!vma_has_reserves(vma, chg))
592 					break;
593 
594 				SetPagePrivate(page);
595 				h->resv_huge_pages--;
596 				break;
597 			}
598 		}
599 	}
600 
601 	mpol_cond_put(mpol);
602 	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
603 		goto retry_cpuset;
604 	return page;
605 
606 err:
607 	return NULL;
608 }
609 
610 static void update_and_free_page(struct hstate *h, struct page *page)
611 {
612 	int i;
613 
614 	VM_BUG_ON(h->order >= MAX_ORDER);
615 
616 	h->nr_huge_pages--;
617 	h->nr_huge_pages_node[page_to_nid(page)]--;
618 	for (i = 0; i < pages_per_huge_page(h); i++) {
619 		page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
620 				1 << PG_referenced | 1 << PG_dirty |
621 				1 << PG_active | 1 << PG_reserved |
622 				1 << PG_private | 1 << PG_writeback);
623 	}
624 	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
625 	set_compound_page_dtor(page, NULL);
626 	set_page_refcounted(page);
627 	arch_release_hugepage(page);
628 	__free_pages(page, huge_page_order(h));
629 }
630 
631 struct hstate *size_to_hstate(unsigned long size)
632 {
633 	struct hstate *h;
634 
635 	for_each_hstate(h) {
636 		if (huge_page_size(h) == size)
637 			return h;
638 	}
639 	return NULL;
640 }
641 
642 static void free_huge_page(struct page *page)
643 {
644 	/*
645 	 * Can't pass hstate in here because it is called from the
646 	 * compound page destructor.
647 	 */
648 	struct hstate *h = page_hstate(page);
649 	int nid = page_to_nid(page);
650 	struct hugepage_subpool *spool =
651 		(struct hugepage_subpool *)page_private(page);
652 	bool restore_reserve;
653 
654 	set_page_private(page, 0);
655 	page->mapping = NULL;
656 	BUG_ON(page_count(page));
657 	BUG_ON(page_mapcount(page));
658 	restore_reserve = PagePrivate(page);
659 	ClearPagePrivate(page);
660 
661 	spin_lock(&hugetlb_lock);
662 	hugetlb_cgroup_uncharge_page(hstate_index(h),
663 				     pages_per_huge_page(h), page);
664 	if (restore_reserve)
665 		h->resv_huge_pages++;
666 
667 	if (h->surplus_huge_pages_node[nid] && huge_page_order(h) < MAX_ORDER) {
668 		/* remove the page from active list */
669 		list_del(&page->lru);
670 		update_and_free_page(h, page);
671 		h->surplus_huge_pages--;
672 		h->surplus_huge_pages_node[nid]--;
673 	} else {
674 		arch_clear_hugepage_flags(page);
675 		enqueue_huge_page(h, page);
676 	}
677 	spin_unlock(&hugetlb_lock);
678 	hugepage_subpool_put_pages(spool, 1);
679 }
680 
681 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
682 {
683 	INIT_LIST_HEAD(&page->lru);
684 	set_compound_page_dtor(page, free_huge_page);
685 	spin_lock(&hugetlb_lock);
686 	set_hugetlb_cgroup(page, NULL);
687 	h->nr_huge_pages++;
688 	h->nr_huge_pages_node[nid]++;
689 	spin_unlock(&hugetlb_lock);
690 	put_page(page); /* free it into the hugepage allocator */
691 }
692 
693 static void __init prep_compound_gigantic_page(struct page *page,
694 					       unsigned long order)
695 {
696 	int i;
697 	int nr_pages = 1 << order;
698 	struct page *p = page + 1;
699 
700 	/* we rely on prep_new_huge_page to set the destructor */
701 	set_compound_order(page, order);
702 	__SetPageHead(page);
703 	__ClearPageReserved(page);
704 	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
705 		__SetPageTail(p);
706 		/*
707 		 * For gigantic hugepages allocated through bootmem at
708 		 * boot, it's safer to be consistent with the not-gigantic
709 		 * hugepages and clear the PG_reserved bit from all tail pages
710 		 * too.  Otherwse drivers using get_user_pages() to access tail
711 		 * pages may get the reference counting wrong if they see
712 		 * PG_reserved set on a tail page (despite the head page not
713 		 * having PG_reserved set).  Enforcing this consistency between
714 		 * head and tail pages allows drivers to optimize away a check
715 		 * on the head page when they need know if put_page() is needed
716 		 * after get_user_pages().
717 		 */
718 		__ClearPageReserved(p);
719 		set_page_count(p, 0);
720 		p->first_page = page;
721 	}
722 }
723 
724 /*
725  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
726  * transparent huge pages.  See the PageTransHuge() documentation for more
727  * details.
728  */
729 int PageHuge(struct page *page)
730 {
731 	if (!PageCompound(page))
732 		return 0;
733 
734 	page = compound_head(page);
735 	return get_compound_page_dtor(page) == free_huge_page;
736 }
737 EXPORT_SYMBOL_GPL(PageHuge);
738 
739 /*
740  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
741  * normal or transparent huge pages.
742  */
743 int PageHeadHuge(struct page *page_head)
744 {
745 	if (!PageHead(page_head))
746 		return 0;
747 
748 	return get_compound_page_dtor(page_head) == free_huge_page;
749 }
750 
751 pgoff_t __basepage_index(struct page *page)
752 {
753 	struct page *page_head = compound_head(page);
754 	pgoff_t index = page_index(page_head);
755 	unsigned long compound_idx;
756 
757 	if (!PageHuge(page_head))
758 		return page_index(page);
759 
760 	if (compound_order(page_head) >= MAX_ORDER)
761 		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
762 	else
763 		compound_idx = page - page_head;
764 
765 	return (index << compound_order(page_head)) + compound_idx;
766 }
767 
768 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
769 {
770 	struct page *page;
771 
772 	if (h->order >= MAX_ORDER)
773 		return NULL;
774 
775 	page = alloc_pages_exact_node(nid,
776 		htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
777 						__GFP_REPEAT|__GFP_NOWARN,
778 		huge_page_order(h));
779 	if (page) {
780 		if (arch_prepare_hugepage(page)) {
781 			__free_pages(page, huge_page_order(h));
782 			return NULL;
783 		}
784 		prep_new_huge_page(h, page, nid);
785 	}
786 
787 	return page;
788 }
789 
790 /*
791  * common helper functions for hstate_next_node_to_{alloc|free}.
792  * We may have allocated or freed a huge page based on a different
793  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
794  * be outside of *nodes_allowed.  Ensure that we use an allowed
795  * node for alloc or free.
796  */
797 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
798 {
799 	nid = next_node(nid, *nodes_allowed);
800 	if (nid == MAX_NUMNODES)
801 		nid = first_node(*nodes_allowed);
802 	VM_BUG_ON(nid >= MAX_NUMNODES);
803 
804 	return nid;
805 }
806 
807 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
808 {
809 	if (!node_isset(nid, *nodes_allowed))
810 		nid = next_node_allowed(nid, nodes_allowed);
811 	return nid;
812 }
813 
814 /*
815  * returns the previously saved node ["this node"] from which to
816  * allocate a persistent huge page for the pool and advance the
817  * next node from which to allocate, handling wrap at end of node
818  * mask.
819  */
820 static int hstate_next_node_to_alloc(struct hstate *h,
821 					nodemask_t *nodes_allowed)
822 {
823 	int nid;
824 
825 	VM_BUG_ON(!nodes_allowed);
826 
827 	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
828 	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
829 
830 	return nid;
831 }
832 
833 /*
834  * helper for free_pool_huge_page() - return the previously saved
835  * node ["this node"] from which to free a huge page.  Advance the
836  * next node id whether or not we find a free huge page to free so
837  * that the next attempt to free addresses the next node.
838  */
839 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
840 {
841 	int nid;
842 
843 	VM_BUG_ON(!nodes_allowed);
844 
845 	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
846 	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
847 
848 	return nid;
849 }
850 
851 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
852 	for (nr_nodes = nodes_weight(*mask);				\
853 		nr_nodes > 0 &&						\
854 		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
855 		nr_nodes--)
856 
857 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
858 	for (nr_nodes = nodes_weight(*mask);				\
859 		nr_nodes > 0 &&						\
860 		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
861 		nr_nodes--)
862 
863 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
864 {
865 	struct page *page;
866 	int nr_nodes, node;
867 	int ret = 0;
868 
869 	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
870 		page = alloc_fresh_huge_page_node(h, node);
871 		if (page) {
872 			ret = 1;
873 			break;
874 		}
875 	}
876 
877 	if (ret)
878 		count_vm_event(HTLB_BUDDY_PGALLOC);
879 	else
880 		count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
881 
882 	return ret;
883 }
884 
885 /*
886  * Free huge page from pool from next node to free.
887  * Attempt to keep persistent huge pages more or less
888  * balanced over allowed nodes.
889  * Called with hugetlb_lock locked.
890  */
891 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
892 							 bool acct_surplus)
893 {
894 	int nr_nodes, node;
895 	int ret = 0;
896 
897 	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
898 		/*
899 		 * If we're returning unused surplus pages, only examine
900 		 * nodes with surplus pages.
901 		 */
902 		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
903 		    !list_empty(&h->hugepage_freelists[node])) {
904 			struct page *page =
905 				list_entry(h->hugepage_freelists[node].next,
906 					  struct page, lru);
907 			list_del(&page->lru);
908 			h->free_huge_pages--;
909 			h->free_huge_pages_node[node]--;
910 			if (acct_surplus) {
911 				h->surplus_huge_pages--;
912 				h->surplus_huge_pages_node[node]--;
913 			}
914 			update_and_free_page(h, page);
915 			ret = 1;
916 			break;
917 		}
918 	}
919 
920 	return ret;
921 }
922 
923 /*
924  * Dissolve a given free hugepage into free buddy pages. This function does
925  * nothing for in-use (including surplus) hugepages.
926  */
927 static void dissolve_free_huge_page(struct page *page)
928 {
929 	spin_lock(&hugetlb_lock);
930 	if (PageHuge(page) && !page_count(page)) {
931 		struct hstate *h = page_hstate(page);
932 		int nid = page_to_nid(page);
933 		list_del(&page->lru);
934 		h->free_huge_pages--;
935 		h->free_huge_pages_node[nid]--;
936 		update_and_free_page(h, page);
937 	}
938 	spin_unlock(&hugetlb_lock);
939 }
940 
941 /*
942  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
943  * make specified memory blocks removable from the system.
944  * Note that start_pfn should aligned with (minimum) hugepage size.
945  */
946 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
947 {
948 	unsigned int order = 8 * sizeof(void *);
949 	unsigned long pfn;
950 	struct hstate *h;
951 
952 	/* Set scan step to minimum hugepage size */
953 	for_each_hstate(h)
954 		if (order > huge_page_order(h))
955 			order = huge_page_order(h);
956 	VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << order));
957 	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << order)
958 		dissolve_free_huge_page(pfn_to_page(pfn));
959 }
960 
961 static struct page *alloc_buddy_huge_page(struct hstate *h, int nid)
962 {
963 	struct page *page;
964 	unsigned int r_nid;
965 
966 	if (h->order >= MAX_ORDER)
967 		return NULL;
968 
969 	/*
970 	 * Assume we will successfully allocate the surplus page to
971 	 * prevent racing processes from causing the surplus to exceed
972 	 * overcommit
973 	 *
974 	 * This however introduces a different race, where a process B
975 	 * tries to grow the static hugepage pool while alloc_pages() is
976 	 * called by process A. B will only examine the per-node
977 	 * counters in determining if surplus huge pages can be
978 	 * converted to normal huge pages in adjust_pool_surplus(). A
979 	 * won't be able to increment the per-node counter, until the
980 	 * lock is dropped by B, but B doesn't drop hugetlb_lock until
981 	 * no more huge pages can be converted from surplus to normal
982 	 * state (and doesn't try to convert again). Thus, we have a
983 	 * case where a surplus huge page exists, the pool is grown, and
984 	 * the surplus huge page still exists after, even though it
985 	 * should just have been converted to a normal huge page. This
986 	 * does not leak memory, though, as the hugepage will be freed
987 	 * once it is out of use. It also does not allow the counters to
988 	 * go out of whack in adjust_pool_surplus() as we don't modify
989 	 * the node values until we've gotten the hugepage and only the
990 	 * per-node value is checked there.
991 	 */
992 	spin_lock(&hugetlb_lock);
993 	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
994 		spin_unlock(&hugetlb_lock);
995 		return NULL;
996 	} else {
997 		h->nr_huge_pages++;
998 		h->surplus_huge_pages++;
999 	}
1000 	spin_unlock(&hugetlb_lock);
1001 
1002 	if (nid == NUMA_NO_NODE)
1003 		page = alloc_pages(htlb_alloc_mask(h)|__GFP_COMP|
1004 				   __GFP_REPEAT|__GFP_NOWARN,
1005 				   huge_page_order(h));
1006 	else
1007 		page = alloc_pages_exact_node(nid,
1008 			htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1009 			__GFP_REPEAT|__GFP_NOWARN, huge_page_order(h));
1010 
1011 	if (page && arch_prepare_hugepage(page)) {
1012 		__free_pages(page, huge_page_order(h));
1013 		page = NULL;
1014 	}
1015 
1016 	spin_lock(&hugetlb_lock);
1017 	if (page) {
1018 		INIT_LIST_HEAD(&page->lru);
1019 		r_nid = page_to_nid(page);
1020 		set_compound_page_dtor(page, free_huge_page);
1021 		set_hugetlb_cgroup(page, NULL);
1022 		/*
1023 		 * We incremented the global counters already
1024 		 */
1025 		h->nr_huge_pages_node[r_nid]++;
1026 		h->surplus_huge_pages_node[r_nid]++;
1027 		__count_vm_event(HTLB_BUDDY_PGALLOC);
1028 	} else {
1029 		h->nr_huge_pages--;
1030 		h->surplus_huge_pages--;
1031 		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1032 	}
1033 	spin_unlock(&hugetlb_lock);
1034 
1035 	return page;
1036 }
1037 
1038 /*
1039  * This allocation function is useful in the context where vma is irrelevant.
1040  * E.g. soft-offlining uses this function because it only cares physical
1041  * address of error page.
1042  */
1043 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1044 {
1045 	struct page *page = NULL;
1046 
1047 	spin_lock(&hugetlb_lock);
1048 	if (h->free_huge_pages - h->resv_huge_pages > 0)
1049 		page = dequeue_huge_page_node(h, nid);
1050 	spin_unlock(&hugetlb_lock);
1051 
1052 	if (!page)
1053 		page = alloc_buddy_huge_page(h, nid);
1054 
1055 	return page;
1056 }
1057 
1058 /*
1059  * Increase the hugetlb pool such that it can accommodate a reservation
1060  * of size 'delta'.
1061  */
1062 static int gather_surplus_pages(struct hstate *h, int delta)
1063 {
1064 	struct list_head surplus_list;
1065 	struct page *page, *tmp;
1066 	int ret, i;
1067 	int needed, allocated;
1068 	bool alloc_ok = true;
1069 
1070 	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1071 	if (needed <= 0) {
1072 		h->resv_huge_pages += delta;
1073 		return 0;
1074 	}
1075 
1076 	allocated = 0;
1077 	INIT_LIST_HEAD(&surplus_list);
1078 
1079 	ret = -ENOMEM;
1080 retry:
1081 	spin_unlock(&hugetlb_lock);
1082 	for (i = 0; i < needed; i++) {
1083 		page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1084 		if (!page) {
1085 			alloc_ok = false;
1086 			break;
1087 		}
1088 		list_add(&page->lru, &surplus_list);
1089 	}
1090 	allocated += i;
1091 
1092 	/*
1093 	 * After retaking hugetlb_lock, we need to recalculate 'needed'
1094 	 * because either resv_huge_pages or free_huge_pages may have changed.
1095 	 */
1096 	spin_lock(&hugetlb_lock);
1097 	needed = (h->resv_huge_pages + delta) -
1098 			(h->free_huge_pages + allocated);
1099 	if (needed > 0) {
1100 		if (alloc_ok)
1101 			goto retry;
1102 		/*
1103 		 * We were not able to allocate enough pages to
1104 		 * satisfy the entire reservation so we free what
1105 		 * we've allocated so far.
1106 		 */
1107 		goto free;
1108 	}
1109 	/*
1110 	 * The surplus_list now contains _at_least_ the number of extra pages
1111 	 * needed to accommodate the reservation.  Add the appropriate number
1112 	 * of pages to the hugetlb pool and free the extras back to the buddy
1113 	 * allocator.  Commit the entire reservation here to prevent another
1114 	 * process from stealing the pages as they are added to the pool but
1115 	 * before they are reserved.
1116 	 */
1117 	needed += allocated;
1118 	h->resv_huge_pages += delta;
1119 	ret = 0;
1120 
1121 	/* Free the needed pages to the hugetlb pool */
1122 	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1123 		if ((--needed) < 0)
1124 			break;
1125 		/*
1126 		 * This page is now managed by the hugetlb allocator and has
1127 		 * no users -- drop the buddy allocator's reference.
1128 		 */
1129 		put_page_testzero(page);
1130 		VM_BUG_ON_PAGE(page_count(page), page);
1131 		enqueue_huge_page(h, page);
1132 	}
1133 free:
1134 	spin_unlock(&hugetlb_lock);
1135 
1136 	/* Free unnecessary surplus pages to the buddy allocator */
1137 	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1138 		put_page(page);
1139 	spin_lock(&hugetlb_lock);
1140 
1141 	return ret;
1142 }
1143 
1144 /*
1145  * When releasing a hugetlb pool reservation, any surplus pages that were
1146  * allocated to satisfy the reservation must be explicitly freed if they were
1147  * never used.
1148  * Called with hugetlb_lock held.
1149  */
1150 static void return_unused_surplus_pages(struct hstate *h,
1151 					unsigned long unused_resv_pages)
1152 {
1153 	unsigned long nr_pages;
1154 
1155 	/* Uncommit the reservation */
1156 	h->resv_huge_pages -= unused_resv_pages;
1157 
1158 	/* Cannot return gigantic pages currently */
1159 	if (h->order >= MAX_ORDER)
1160 		return;
1161 
1162 	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1163 
1164 	/*
1165 	 * We want to release as many surplus pages as possible, spread
1166 	 * evenly across all nodes with memory. Iterate across these nodes
1167 	 * until we can no longer free unreserved surplus pages. This occurs
1168 	 * when the nodes with surplus pages have no free pages.
1169 	 * free_pool_huge_page() will balance the the freed pages across the
1170 	 * on-line nodes with memory and will handle the hstate accounting.
1171 	 */
1172 	while (nr_pages--) {
1173 		if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1174 			break;
1175 		cond_resched_lock(&hugetlb_lock);
1176 	}
1177 }
1178 
1179 /*
1180  * Determine if the huge page at addr within the vma has an associated
1181  * reservation.  Where it does not we will need to logically increase
1182  * reservation and actually increase subpool usage before an allocation
1183  * can occur.  Where any new reservation would be required the
1184  * reservation change is prepared, but not committed.  Once the page
1185  * has been allocated from the subpool and instantiated the change should
1186  * be committed via vma_commit_reservation.  No action is required on
1187  * failure.
1188  */
1189 static long vma_needs_reservation(struct hstate *h,
1190 			struct vm_area_struct *vma, unsigned long addr)
1191 {
1192 	struct resv_map *resv;
1193 	pgoff_t idx;
1194 	long chg;
1195 
1196 	resv = vma_resv_map(vma);
1197 	if (!resv)
1198 		return 1;
1199 
1200 	idx = vma_hugecache_offset(h, vma, addr);
1201 	chg = region_chg(resv, idx, idx + 1);
1202 
1203 	if (vma->vm_flags & VM_MAYSHARE)
1204 		return chg;
1205 	else
1206 		return chg < 0 ? chg : 0;
1207 }
1208 static void vma_commit_reservation(struct hstate *h,
1209 			struct vm_area_struct *vma, unsigned long addr)
1210 {
1211 	struct resv_map *resv;
1212 	pgoff_t idx;
1213 
1214 	resv = vma_resv_map(vma);
1215 	if (!resv)
1216 		return;
1217 
1218 	idx = vma_hugecache_offset(h, vma, addr);
1219 	region_add(resv, idx, idx + 1);
1220 }
1221 
1222 static struct page *alloc_huge_page(struct vm_area_struct *vma,
1223 				    unsigned long addr, int avoid_reserve)
1224 {
1225 	struct hugepage_subpool *spool = subpool_vma(vma);
1226 	struct hstate *h = hstate_vma(vma);
1227 	struct page *page;
1228 	long chg;
1229 	int ret, idx;
1230 	struct hugetlb_cgroup *h_cg;
1231 
1232 	idx = hstate_index(h);
1233 	/*
1234 	 * Processes that did not create the mapping will have no
1235 	 * reserves and will not have accounted against subpool
1236 	 * limit. Check that the subpool limit can be made before
1237 	 * satisfying the allocation MAP_NORESERVE mappings may also
1238 	 * need pages and subpool limit allocated allocated if no reserve
1239 	 * mapping overlaps.
1240 	 */
1241 	chg = vma_needs_reservation(h, vma, addr);
1242 	if (chg < 0)
1243 		return ERR_PTR(-ENOMEM);
1244 	if (chg || avoid_reserve)
1245 		if (hugepage_subpool_get_pages(spool, 1))
1246 			return ERR_PTR(-ENOSPC);
1247 
1248 	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1249 	if (ret) {
1250 		if (chg || avoid_reserve)
1251 			hugepage_subpool_put_pages(spool, 1);
1252 		return ERR_PTR(-ENOSPC);
1253 	}
1254 	spin_lock(&hugetlb_lock);
1255 	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, chg);
1256 	if (!page) {
1257 		spin_unlock(&hugetlb_lock);
1258 		page = alloc_buddy_huge_page(h, NUMA_NO_NODE);
1259 		if (!page) {
1260 			hugetlb_cgroup_uncharge_cgroup(idx,
1261 						       pages_per_huge_page(h),
1262 						       h_cg);
1263 			if (chg || avoid_reserve)
1264 				hugepage_subpool_put_pages(spool, 1);
1265 			return ERR_PTR(-ENOSPC);
1266 		}
1267 		spin_lock(&hugetlb_lock);
1268 		list_move(&page->lru, &h->hugepage_activelist);
1269 		/* Fall through */
1270 	}
1271 	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1272 	spin_unlock(&hugetlb_lock);
1273 
1274 	set_page_private(page, (unsigned long)spool);
1275 
1276 	vma_commit_reservation(h, vma, addr);
1277 	return page;
1278 }
1279 
1280 /*
1281  * alloc_huge_page()'s wrapper which simply returns the page if allocation
1282  * succeeds, otherwise NULL. This function is called from new_vma_page(),
1283  * where no ERR_VALUE is expected to be returned.
1284  */
1285 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1286 				unsigned long addr, int avoid_reserve)
1287 {
1288 	struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1289 	if (IS_ERR(page))
1290 		page = NULL;
1291 	return page;
1292 }
1293 
1294 int __weak alloc_bootmem_huge_page(struct hstate *h)
1295 {
1296 	struct huge_bootmem_page *m;
1297 	int nr_nodes, node;
1298 
1299 	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
1300 		void *addr;
1301 
1302 		addr = memblock_virt_alloc_try_nid_nopanic(
1303 				huge_page_size(h), huge_page_size(h),
1304 				0, BOOTMEM_ALLOC_ACCESSIBLE, node);
1305 		if (addr) {
1306 			/*
1307 			 * Use the beginning of the huge page to store the
1308 			 * huge_bootmem_page struct (until gather_bootmem
1309 			 * puts them into the mem_map).
1310 			 */
1311 			m = addr;
1312 			goto found;
1313 		}
1314 	}
1315 	return 0;
1316 
1317 found:
1318 	BUG_ON((unsigned long)virt_to_phys(m) & (huge_page_size(h) - 1));
1319 	/* Put them into a private list first because mem_map is not up yet */
1320 	list_add(&m->list, &huge_boot_pages);
1321 	m->hstate = h;
1322 	return 1;
1323 }
1324 
1325 static void __init prep_compound_huge_page(struct page *page, int order)
1326 {
1327 	if (unlikely(order > (MAX_ORDER - 1)))
1328 		prep_compound_gigantic_page(page, order);
1329 	else
1330 		prep_compound_page(page, order);
1331 }
1332 
1333 /* Put bootmem huge pages into the standard lists after mem_map is up */
1334 static void __init gather_bootmem_prealloc(void)
1335 {
1336 	struct huge_bootmem_page *m;
1337 
1338 	list_for_each_entry(m, &huge_boot_pages, list) {
1339 		struct hstate *h = m->hstate;
1340 		struct page *page;
1341 
1342 #ifdef CONFIG_HIGHMEM
1343 		page = pfn_to_page(m->phys >> PAGE_SHIFT);
1344 		memblock_free_late(__pa(m),
1345 				   sizeof(struct huge_bootmem_page));
1346 #else
1347 		page = virt_to_page(m);
1348 #endif
1349 		WARN_ON(page_count(page) != 1);
1350 		prep_compound_huge_page(page, h->order);
1351 		WARN_ON(PageReserved(page));
1352 		prep_new_huge_page(h, page, page_to_nid(page));
1353 		/*
1354 		 * If we had gigantic hugepages allocated at boot time, we need
1355 		 * to restore the 'stolen' pages to totalram_pages in order to
1356 		 * fix confusing memory reports from free(1) and another
1357 		 * side-effects, like CommitLimit going negative.
1358 		 */
1359 		if (h->order > (MAX_ORDER - 1))
1360 			adjust_managed_page_count(page, 1 << h->order);
1361 	}
1362 }
1363 
1364 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
1365 {
1366 	unsigned long i;
1367 
1368 	for (i = 0; i < h->max_huge_pages; ++i) {
1369 		if (h->order >= MAX_ORDER) {
1370 			if (!alloc_bootmem_huge_page(h))
1371 				break;
1372 		} else if (!alloc_fresh_huge_page(h,
1373 					 &node_states[N_MEMORY]))
1374 			break;
1375 	}
1376 	h->max_huge_pages = i;
1377 }
1378 
1379 static void __init hugetlb_init_hstates(void)
1380 {
1381 	struct hstate *h;
1382 
1383 	for_each_hstate(h) {
1384 		/* oversize hugepages were init'ed in early boot */
1385 		if (h->order < MAX_ORDER)
1386 			hugetlb_hstate_alloc_pages(h);
1387 	}
1388 }
1389 
1390 static char * __init memfmt(char *buf, unsigned long n)
1391 {
1392 	if (n >= (1UL << 30))
1393 		sprintf(buf, "%lu GB", n >> 30);
1394 	else if (n >= (1UL << 20))
1395 		sprintf(buf, "%lu MB", n >> 20);
1396 	else
1397 		sprintf(buf, "%lu KB", n >> 10);
1398 	return buf;
1399 }
1400 
1401 static void __init report_hugepages(void)
1402 {
1403 	struct hstate *h;
1404 
1405 	for_each_hstate(h) {
1406 		char buf[32];
1407 		pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
1408 			memfmt(buf, huge_page_size(h)),
1409 			h->free_huge_pages);
1410 	}
1411 }
1412 
1413 #ifdef CONFIG_HIGHMEM
1414 static void try_to_free_low(struct hstate *h, unsigned long count,
1415 						nodemask_t *nodes_allowed)
1416 {
1417 	int i;
1418 
1419 	if (h->order >= MAX_ORDER)
1420 		return;
1421 
1422 	for_each_node_mask(i, *nodes_allowed) {
1423 		struct page *page, *next;
1424 		struct list_head *freel = &h->hugepage_freelists[i];
1425 		list_for_each_entry_safe(page, next, freel, lru) {
1426 			if (count >= h->nr_huge_pages)
1427 				return;
1428 			if (PageHighMem(page))
1429 				continue;
1430 			list_del(&page->lru);
1431 			update_and_free_page(h, page);
1432 			h->free_huge_pages--;
1433 			h->free_huge_pages_node[page_to_nid(page)]--;
1434 		}
1435 	}
1436 }
1437 #else
1438 static inline void try_to_free_low(struct hstate *h, unsigned long count,
1439 						nodemask_t *nodes_allowed)
1440 {
1441 }
1442 #endif
1443 
1444 /*
1445  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
1446  * balanced by operating on them in a round-robin fashion.
1447  * Returns 1 if an adjustment was made.
1448  */
1449 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
1450 				int delta)
1451 {
1452 	int nr_nodes, node;
1453 
1454 	VM_BUG_ON(delta != -1 && delta != 1);
1455 
1456 	if (delta < 0) {
1457 		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1458 			if (h->surplus_huge_pages_node[node])
1459 				goto found;
1460 		}
1461 	} else {
1462 		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1463 			if (h->surplus_huge_pages_node[node] <
1464 					h->nr_huge_pages_node[node])
1465 				goto found;
1466 		}
1467 	}
1468 	return 0;
1469 
1470 found:
1471 	h->surplus_huge_pages += delta;
1472 	h->surplus_huge_pages_node[node] += delta;
1473 	return 1;
1474 }
1475 
1476 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
1477 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
1478 						nodemask_t *nodes_allowed)
1479 {
1480 	unsigned long min_count, ret;
1481 
1482 	if (h->order >= MAX_ORDER)
1483 		return h->max_huge_pages;
1484 
1485 	/*
1486 	 * Increase the pool size
1487 	 * First take pages out of surplus state.  Then make up the
1488 	 * remaining difference by allocating fresh huge pages.
1489 	 *
1490 	 * We might race with alloc_buddy_huge_page() here and be unable
1491 	 * to convert a surplus huge page to a normal huge page. That is
1492 	 * not critical, though, it just means the overall size of the
1493 	 * pool might be one hugepage larger than it needs to be, but
1494 	 * within all the constraints specified by the sysctls.
1495 	 */
1496 	spin_lock(&hugetlb_lock);
1497 	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
1498 		if (!adjust_pool_surplus(h, nodes_allowed, -1))
1499 			break;
1500 	}
1501 
1502 	while (count > persistent_huge_pages(h)) {
1503 		/*
1504 		 * If this allocation races such that we no longer need the
1505 		 * page, free_huge_page will handle it by freeing the page
1506 		 * and reducing the surplus.
1507 		 */
1508 		spin_unlock(&hugetlb_lock);
1509 		ret = alloc_fresh_huge_page(h, nodes_allowed);
1510 		spin_lock(&hugetlb_lock);
1511 		if (!ret)
1512 			goto out;
1513 
1514 		/* Bail for signals. Probably ctrl-c from user */
1515 		if (signal_pending(current))
1516 			goto out;
1517 	}
1518 
1519 	/*
1520 	 * Decrease the pool size
1521 	 * First return free pages to the buddy allocator (being careful
1522 	 * to keep enough around to satisfy reservations).  Then place
1523 	 * pages into surplus state as needed so the pool will shrink
1524 	 * to the desired size as pages become free.
1525 	 *
1526 	 * By placing pages into the surplus state independent of the
1527 	 * overcommit value, we are allowing the surplus pool size to
1528 	 * exceed overcommit. There are few sane options here. Since
1529 	 * alloc_buddy_huge_page() is checking the global counter,
1530 	 * though, we'll note that we're not allowed to exceed surplus
1531 	 * and won't grow the pool anywhere else. Not until one of the
1532 	 * sysctls are changed, or the surplus pages go out of use.
1533 	 */
1534 	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
1535 	min_count = max(count, min_count);
1536 	try_to_free_low(h, min_count, nodes_allowed);
1537 	while (min_count < persistent_huge_pages(h)) {
1538 		if (!free_pool_huge_page(h, nodes_allowed, 0))
1539 			break;
1540 		cond_resched_lock(&hugetlb_lock);
1541 	}
1542 	while (count < persistent_huge_pages(h)) {
1543 		if (!adjust_pool_surplus(h, nodes_allowed, 1))
1544 			break;
1545 	}
1546 out:
1547 	ret = persistent_huge_pages(h);
1548 	spin_unlock(&hugetlb_lock);
1549 	return ret;
1550 }
1551 
1552 #define HSTATE_ATTR_RO(_name) \
1553 	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
1554 
1555 #define HSTATE_ATTR(_name) \
1556 	static struct kobj_attribute _name##_attr = \
1557 		__ATTR(_name, 0644, _name##_show, _name##_store)
1558 
1559 static struct kobject *hugepages_kobj;
1560 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
1561 
1562 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
1563 
1564 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
1565 {
1566 	int i;
1567 
1568 	for (i = 0; i < HUGE_MAX_HSTATE; i++)
1569 		if (hstate_kobjs[i] == kobj) {
1570 			if (nidp)
1571 				*nidp = NUMA_NO_NODE;
1572 			return &hstates[i];
1573 		}
1574 
1575 	return kobj_to_node_hstate(kobj, nidp);
1576 }
1577 
1578 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
1579 					struct kobj_attribute *attr, char *buf)
1580 {
1581 	struct hstate *h;
1582 	unsigned long nr_huge_pages;
1583 	int nid;
1584 
1585 	h = kobj_to_hstate(kobj, &nid);
1586 	if (nid == NUMA_NO_NODE)
1587 		nr_huge_pages = h->nr_huge_pages;
1588 	else
1589 		nr_huge_pages = h->nr_huge_pages_node[nid];
1590 
1591 	return sprintf(buf, "%lu\n", nr_huge_pages);
1592 }
1593 
1594 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
1595 			struct kobject *kobj, struct kobj_attribute *attr,
1596 			const char *buf, size_t len)
1597 {
1598 	int err;
1599 	int nid;
1600 	unsigned long count;
1601 	struct hstate *h;
1602 	NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
1603 
1604 	err = kstrtoul(buf, 10, &count);
1605 	if (err)
1606 		goto out;
1607 
1608 	h = kobj_to_hstate(kobj, &nid);
1609 	if (h->order >= MAX_ORDER) {
1610 		err = -EINVAL;
1611 		goto out;
1612 	}
1613 
1614 	if (nid == NUMA_NO_NODE) {
1615 		/*
1616 		 * global hstate attribute
1617 		 */
1618 		if (!(obey_mempolicy &&
1619 				init_nodemask_of_mempolicy(nodes_allowed))) {
1620 			NODEMASK_FREE(nodes_allowed);
1621 			nodes_allowed = &node_states[N_MEMORY];
1622 		}
1623 	} else if (nodes_allowed) {
1624 		/*
1625 		 * per node hstate attribute: adjust count to global,
1626 		 * but restrict alloc/free to the specified node.
1627 		 */
1628 		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
1629 		init_nodemask_of_node(nodes_allowed, nid);
1630 	} else
1631 		nodes_allowed = &node_states[N_MEMORY];
1632 
1633 	h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
1634 
1635 	if (nodes_allowed != &node_states[N_MEMORY])
1636 		NODEMASK_FREE(nodes_allowed);
1637 
1638 	return len;
1639 out:
1640 	NODEMASK_FREE(nodes_allowed);
1641 	return err;
1642 }
1643 
1644 static ssize_t nr_hugepages_show(struct kobject *kobj,
1645 				       struct kobj_attribute *attr, char *buf)
1646 {
1647 	return nr_hugepages_show_common(kobj, attr, buf);
1648 }
1649 
1650 static ssize_t nr_hugepages_store(struct kobject *kobj,
1651 	       struct kobj_attribute *attr, const char *buf, size_t len)
1652 {
1653 	return nr_hugepages_store_common(false, kobj, attr, buf, len);
1654 }
1655 HSTATE_ATTR(nr_hugepages);
1656 
1657 #ifdef CONFIG_NUMA
1658 
1659 /*
1660  * hstate attribute for optionally mempolicy-based constraint on persistent
1661  * huge page alloc/free.
1662  */
1663 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
1664 				       struct kobj_attribute *attr, char *buf)
1665 {
1666 	return nr_hugepages_show_common(kobj, attr, buf);
1667 }
1668 
1669 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
1670 	       struct kobj_attribute *attr, const char *buf, size_t len)
1671 {
1672 	return nr_hugepages_store_common(true, kobj, attr, buf, len);
1673 }
1674 HSTATE_ATTR(nr_hugepages_mempolicy);
1675 #endif
1676 
1677 
1678 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
1679 					struct kobj_attribute *attr, char *buf)
1680 {
1681 	struct hstate *h = kobj_to_hstate(kobj, NULL);
1682 	return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
1683 }
1684 
1685 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
1686 		struct kobj_attribute *attr, const char *buf, size_t count)
1687 {
1688 	int err;
1689 	unsigned long input;
1690 	struct hstate *h = kobj_to_hstate(kobj, NULL);
1691 
1692 	if (h->order >= MAX_ORDER)
1693 		return -EINVAL;
1694 
1695 	err = kstrtoul(buf, 10, &input);
1696 	if (err)
1697 		return err;
1698 
1699 	spin_lock(&hugetlb_lock);
1700 	h->nr_overcommit_huge_pages = input;
1701 	spin_unlock(&hugetlb_lock);
1702 
1703 	return count;
1704 }
1705 HSTATE_ATTR(nr_overcommit_hugepages);
1706 
1707 static ssize_t free_hugepages_show(struct kobject *kobj,
1708 					struct kobj_attribute *attr, char *buf)
1709 {
1710 	struct hstate *h;
1711 	unsigned long free_huge_pages;
1712 	int nid;
1713 
1714 	h = kobj_to_hstate(kobj, &nid);
1715 	if (nid == NUMA_NO_NODE)
1716 		free_huge_pages = h->free_huge_pages;
1717 	else
1718 		free_huge_pages = h->free_huge_pages_node[nid];
1719 
1720 	return sprintf(buf, "%lu\n", free_huge_pages);
1721 }
1722 HSTATE_ATTR_RO(free_hugepages);
1723 
1724 static ssize_t resv_hugepages_show(struct kobject *kobj,
1725 					struct kobj_attribute *attr, char *buf)
1726 {
1727 	struct hstate *h = kobj_to_hstate(kobj, NULL);
1728 	return sprintf(buf, "%lu\n", h->resv_huge_pages);
1729 }
1730 HSTATE_ATTR_RO(resv_hugepages);
1731 
1732 static ssize_t surplus_hugepages_show(struct kobject *kobj,
1733 					struct kobj_attribute *attr, char *buf)
1734 {
1735 	struct hstate *h;
1736 	unsigned long surplus_huge_pages;
1737 	int nid;
1738 
1739 	h = kobj_to_hstate(kobj, &nid);
1740 	if (nid == NUMA_NO_NODE)
1741 		surplus_huge_pages = h->surplus_huge_pages;
1742 	else
1743 		surplus_huge_pages = h->surplus_huge_pages_node[nid];
1744 
1745 	return sprintf(buf, "%lu\n", surplus_huge_pages);
1746 }
1747 HSTATE_ATTR_RO(surplus_hugepages);
1748 
1749 static struct attribute *hstate_attrs[] = {
1750 	&nr_hugepages_attr.attr,
1751 	&nr_overcommit_hugepages_attr.attr,
1752 	&free_hugepages_attr.attr,
1753 	&resv_hugepages_attr.attr,
1754 	&surplus_hugepages_attr.attr,
1755 #ifdef CONFIG_NUMA
1756 	&nr_hugepages_mempolicy_attr.attr,
1757 #endif
1758 	NULL,
1759 };
1760 
1761 static struct attribute_group hstate_attr_group = {
1762 	.attrs = hstate_attrs,
1763 };
1764 
1765 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
1766 				    struct kobject **hstate_kobjs,
1767 				    struct attribute_group *hstate_attr_group)
1768 {
1769 	int retval;
1770 	int hi = hstate_index(h);
1771 
1772 	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
1773 	if (!hstate_kobjs[hi])
1774 		return -ENOMEM;
1775 
1776 	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
1777 	if (retval)
1778 		kobject_put(hstate_kobjs[hi]);
1779 
1780 	return retval;
1781 }
1782 
1783 static void __init hugetlb_sysfs_init(void)
1784 {
1785 	struct hstate *h;
1786 	int err;
1787 
1788 	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
1789 	if (!hugepages_kobj)
1790 		return;
1791 
1792 	for_each_hstate(h) {
1793 		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
1794 					 hstate_kobjs, &hstate_attr_group);
1795 		if (err)
1796 			pr_err("Hugetlb: Unable to add hstate %s", h->name);
1797 	}
1798 }
1799 
1800 #ifdef CONFIG_NUMA
1801 
1802 /*
1803  * node_hstate/s - associate per node hstate attributes, via their kobjects,
1804  * with node devices in node_devices[] using a parallel array.  The array
1805  * index of a node device or _hstate == node id.
1806  * This is here to avoid any static dependency of the node device driver, in
1807  * the base kernel, on the hugetlb module.
1808  */
1809 struct node_hstate {
1810 	struct kobject		*hugepages_kobj;
1811 	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
1812 };
1813 struct node_hstate node_hstates[MAX_NUMNODES];
1814 
1815 /*
1816  * A subset of global hstate attributes for node devices
1817  */
1818 static struct attribute *per_node_hstate_attrs[] = {
1819 	&nr_hugepages_attr.attr,
1820 	&free_hugepages_attr.attr,
1821 	&surplus_hugepages_attr.attr,
1822 	NULL,
1823 };
1824 
1825 static struct attribute_group per_node_hstate_attr_group = {
1826 	.attrs = per_node_hstate_attrs,
1827 };
1828 
1829 /*
1830  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
1831  * Returns node id via non-NULL nidp.
1832  */
1833 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1834 {
1835 	int nid;
1836 
1837 	for (nid = 0; nid < nr_node_ids; nid++) {
1838 		struct node_hstate *nhs = &node_hstates[nid];
1839 		int i;
1840 		for (i = 0; i < HUGE_MAX_HSTATE; i++)
1841 			if (nhs->hstate_kobjs[i] == kobj) {
1842 				if (nidp)
1843 					*nidp = nid;
1844 				return &hstates[i];
1845 			}
1846 	}
1847 
1848 	BUG();
1849 	return NULL;
1850 }
1851 
1852 /*
1853  * Unregister hstate attributes from a single node device.
1854  * No-op if no hstate attributes attached.
1855  */
1856 static void hugetlb_unregister_node(struct node *node)
1857 {
1858 	struct hstate *h;
1859 	struct node_hstate *nhs = &node_hstates[node->dev.id];
1860 
1861 	if (!nhs->hugepages_kobj)
1862 		return;		/* no hstate attributes */
1863 
1864 	for_each_hstate(h) {
1865 		int idx = hstate_index(h);
1866 		if (nhs->hstate_kobjs[idx]) {
1867 			kobject_put(nhs->hstate_kobjs[idx]);
1868 			nhs->hstate_kobjs[idx] = NULL;
1869 		}
1870 	}
1871 
1872 	kobject_put(nhs->hugepages_kobj);
1873 	nhs->hugepages_kobj = NULL;
1874 }
1875 
1876 /*
1877  * hugetlb module exit:  unregister hstate attributes from node devices
1878  * that have them.
1879  */
1880 static void hugetlb_unregister_all_nodes(void)
1881 {
1882 	int nid;
1883 
1884 	/*
1885 	 * disable node device registrations.
1886 	 */
1887 	register_hugetlbfs_with_node(NULL, NULL);
1888 
1889 	/*
1890 	 * remove hstate attributes from any nodes that have them.
1891 	 */
1892 	for (nid = 0; nid < nr_node_ids; nid++)
1893 		hugetlb_unregister_node(node_devices[nid]);
1894 }
1895 
1896 /*
1897  * Register hstate attributes for a single node device.
1898  * No-op if attributes already registered.
1899  */
1900 static void hugetlb_register_node(struct node *node)
1901 {
1902 	struct hstate *h;
1903 	struct node_hstate *nhs = &node_hstates[node->dev.id];
1904 	int err;
1905 
1906 	if (nhs->hugepages_kobj)
1907 		return;		/* already allocated */
1908 
1909 	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
1910 							&node->dev.kobj);
1911 	if (!nhs->hugepages_kobj)
1912 		return;
1913 
1914 	for_each_hstate(h) {
1915 		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
1916 						nhs->hstate_kobjs,
1917 						&per_node_hstate_attr_group);
1918 		if (err) {
1919 			pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
1920 				h->name, node->dev.id);
1921 			hugetlb_unregister_node(node);
1922 			break;
1923 		}
1924 	}
1925 }
1926 
1927 /*
1928  * hugetlb init time:  register hstate attributes for all registered node
1929  * devices of nodes that have memory.  All on-line nodes should have
1930  * registered their associated device by this time.
1931  */
1932 static void hugetlb_register_all_nodes(void)
1933 {
1934 	int nid;
1935 
1936 	for_each_node_state(nid, N_MEMORY) {
1937 		struct node *node = node_devices[nid];
1938 		if (node->dev.id == nid)
1939 			hugetlb_register_node(node);
1940 	}
1941 
1942 	/*
1943 	 * Let the node device driver know we're here so it can
1944 	 * [un]register hstate attributes on node hotplug.
1945 	 */
1946 	register_hugetlbfs_with_node(hugetlb_register_node,
1947 				     hugetlb_unregister_node);
1948 }
1949 #else	/* !CONFIG_NUMA */
1950 
1951 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
1952 {
1953 	BUG();
1954 	if (nidp)
1955 		*nidp = -1;
1956 	return NULL;
1957 }
1958 
1959 static void hugetlb_unregister_all_nodes(void) { }
1960 
1961 static void hugetlb_register_all_nodes(void) { }
1962 
1963 #endif
1964 
1965 static void __exit hugetlb_exit(void)
1966 {
1967 	struct hstate *h;
1968 
1969 	hugetlb_unregister_all_nodes();
1970 
1971 	for_each_hstate(h) {
1972 		kobject_put(hstate_kobjs[hstate_index(h)]);
1973 	}
1974 
1975 	kobject_put(hugepages_kobj);
1976 	kfree(htlb_fault_mutex_table);
1977 }
1978 module_exit(hugetlb_exit);
1979 
1980 static int __init hugetlb_init(void)
1981 {
1982 	int i;
1983 
1984 	/* Some platform decide whether they support huge pages at boot
1985 	 * time. On these, such as powerpc, HPAGE_SHIFT is set to 0 when
1986 	 * there is no such support
1987 	 */
1988 	if (HPAGE_SHIFT == 0)
1989 		return 0;
1990 
1991 	if (!size_to_hstate(default_hstate_size)) {
1992 		default_hstate_size = HPAGE_SIZE;
1993 		if (!size_to_hstate(default_hstate_size))
1994 			hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
1995 	}
1996 	default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
1997 	if (default_hstate_max_huge_pages)
1998 		default_hstate.max_huge_pages = default_hstate_max_huge_pages;
1999 
2000 	hugetlb_init_hstates();
2001 	gather_bootmem_prealloc();
2002 	report_hugepages();
2003 
2004 	hugetlb_sysfs_init();
2005 	hugetlb_register_all_nodes();
2006 	hugetlb_cgroup_file_init();
2007 
2008 #ifdef CONFIG_SMP
2009 	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2010 #else
2011 	num_fault_mutexes = 1;
2012 #endif
2013 	htlb_fault_mutex_table =
2014 		kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2015 	BUG_ON(!htlb_fault_mutex_table);
2016 
2017 	for (i = 0; i < num_fault_mutexes; i++)
2018 		mutex_init(&htlb_fault_mutex_table[i]);
2019 	return 0;
2020 }
2021 module_init(hugetlb_init);
2022 
2023 /* Should be called on processing a hugepagesz=... option */
2024 void __init hugetlb_add_hstate(unsigned order)
2025 {
2026 	struct hstate *h;
2027 	unsigned long i;
2028 
2029 	if (size_to_hstate(PAGE_SIZE << order)) {
2030 		pr_warning("hugepagesz= specified twice, ignoring\n");
2031 		return;
2032 	}
2033 	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2034 	BUG_ON(order == 0);
2035 	h = &hstates[hugetlb_max_hstate++];
2036 	h->order = order;
2037 	h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2038 	h->nr_huge_pages = 0;
2039 	h->free_huge_pages = 0;
2040 	for (i = 0; i < MAX_NUMNODES; ++i)
2041 		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2042 	INIT_LIST_HEAD(&h->hugepage_activelist);
2043 	h->next_nid_to_alloc = first_node(node_states[N_MEMORY]);
2044 	h->next_nid_to_free = first_node(node_states[N_MEMORY]);
2045 	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2046 					huge_page_size(h)/1024);
2047 
2048 	parsed_hstate = h;
2049 }
2050 
2051 static int __init hugetlb_nrpages_setup(char *s)
2052 {
2053 	unsigned long *mhp;
2054 	static unsigned long *last_mhp;
2055 
2056 	/*
2057 	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2058 	 * so this hugepages= parameter goes to the "default hstate".
2059 	 */
2060 	if (!hugetlb_max_hstate)
2061 		mhp = &default_hstate_max_huge_pages;
2062 	else
2063 		mhp = &parsed_hstate->max_huge_pages;
2064 
2065 	if (mhp == last_mhp) {
2066 		pr_warning("hugepages= specified twice without "
2067 			   "interleaving hugepagesz=, ignoring\n");
2068 		return 1;
2069 	}
2070 
2071 	if (sscanf(s, "%lu", mhp) <= 0)
2072 		*mhp = 0;
2073 
2074 	/*
2075 	 * Global state is always initialized later in hugetlb_init.
2076 	 * But we need to allocate >= MAX_ORDER hstates here early to still
2077 	 * use the bootmem allocator.
2078 	 */
2079 	if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2080 		hugetlb_hstate_alloc_pages(parsed_hstate);
2081 
2082 	last_mhp = mhp;
2083 
2084 	return 1;
2085 }
2086 __setup("hugepages=", hugetlb_nrpages_setup);
2087 
2088 static int __init hugetlb_default_setup(char *s)
2089 {
2090 	default_hstate_size = memparse(s, &s);
2091 	return 1;
2092 }
2093 __setup("default_hugepagesz=", hugetlb_default_setup);
2094 
2095 static unsigned int cpuset_mems_nr(unsigned int *array)
2096 {
2097 	int node;
2098 	unsigned int nr = 0;
2099 
2100 	for_each_node_mask(node, cpuset_current_mems_allowed)
2101 		nr += array[node];
2102 
2103 	return nr;
2104 }
2105 
2106 #ifdef CONFIG_SYSCTL
2107 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2108 			 struct ctl_table *table, int write,
2109 			 void __user *buffer, size_t *length, loff_t *ppos)
2110 {
2111 	struct hstate *h = &default_hstate;
2112 	unsigned long tmp;
2113 	int ret;
2114 
2115 	tmp = h->max_huge_pages;
2116 
2117 	if (write && h->order >= MAX_ORDER)
2118 		return -EINVAL;
2119 
2120 	table->data = &tmp;
2121 	table->maxlen = sizeof(unsigned long);
2122 	ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2123 	if (ret)
2124 		goto out;
2125 
2126 	if (write) {
2127 		NODEMASK_ALLOC(nodemask_t, nodes_allowed,
2128 						GFP_KERNEL | __GFP_NORETRY);
2129 		if (!(obey_mempolicy &&
2130 			       init_nodemask_of_mempolicy(nodes_allowed))) {
2131 			NODEMASK_FREE(nodes_allowed);
2132 			nodes_allowed = &node_states[N_MEMORY];
2133 		}
2134 		h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
2135 
2136 		if (nodes_allowed != &node_states[N_MEMORY])
2137 			NODEMASK_FREE(nodes_allowed);
2138 	}
2139 out:
2140 	return ret;
2141 }
2142 
2143 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2144 			  void __user *buffer, size_t *length, loff_t *ppos)
2145 {
2146 
2147 	return hugetlb_sysctl_handler_common(false, table, write,
2148 							buffer, length, ppos);
2149 }
2150 
2151 #ifdef CONFIG_NUMA
2152 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2153 			  void __user *buffer, size_t *length, loff_t *ppos)
2154 {
2155 	return hugetlb_sysctl_handler_common(true, table, write,
2156 							buffer, length, ppos);
2157 }
2158 #endif /* CONFIG_NUMA */
2159 
2160 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2161 			void __user *buffer,
2162 			size_t *length, loff_t *ppos)
2163 {
2164 	struct hstate *h = &default_hstate;
2165 	unsigned long tmp;
2166 	int ret;
2167 
2168 	tmp = h->nr_overcommit_huge_pages;
2169 
2170 	if (write && h->order >= MAX_ORDER)
2171 		return -EINVAL;
2172 
2173 	table->data = &tmp;
2174 	table->maxlen = sizeof(unsigned long);
2175 	ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2176 	if (ret)
2177 		goto out;
2178 
2179 	if (write) {
2180 		spin_lock(&hugetlb_lock);
2181 		h->nr_overcommit_huge_pages = tmp;
2182 		spin_unlock(&hugetlb_lock);
2183 	}
2184 out:
2185 	return ret;
2186 }
2187 
2188 #endif /* CONFIG_SYSCTL */
2189 
2190 void hugetlb_report_meminfo(struct seq_file *m)
2191 {
2192 	struct hstate *h = &default_hstate;
2193 	seq_printf(m,
2194 			"HugePages_Total:   %5lu\n"
2195 			"HugePages_Free:    %5lu\n"
2196 			"HugePages_Rsvd:    %5lu\n"
2197 			"HugePages_Surp:    %5lu\n"
2198 			"Hugepagesize:   %8lu kB\n",
2199 			h->nr_huge_pages,
2200 			h->free_huge_pages,
2201 			h->resv_huge_pages,
2202 			h->surplus_huge_pages,
2203 			1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2204 }
2205 
2206 int hugetlb_report_node_meminfo(int nid, char *buf)
2207 {
2208 	struct hstate *h = &default_hstate;
2209 	return sprintf(buf,
2210 		"Node %d HugePages_Total: %5u\n"
2211 		"Node %d HugePages_Free:  %5u\n"
2212 		"Node %d HugePages_Surp:  %5u\n",
2213 		nid, h->nr_huge_pages_node[nid],
2214 		nid, h->free_huge_pages_node[nid],
2215 		nid, h->surplus_huge_pages_node[nid]);
2216 }
2217 
2218 void hugetlb_show_meminfo(void)
2219 {
2220 	struct hstate *h;
2221 	int nid;
2222 
2223 	for_each_node_state(nid, N_MEMORY)
2224 		for_each_hstate(h)
2225 			pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2226 				nid,
2227 				h->nr_huge_pages_node[nid],
2228 				h->free_huge_pages_node[nid],
2229 				h->surplus_huge_pages_node[nid],
2230 				1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2231 }
2232 
2233 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2234 unsigned long hugetlb_total_pages(void)
2235 {
2236 	struct hstate *h;
2237 	unsigned long nr_total_pages = 0;
2238 
2239 	for_each_hstate(h)
2240 		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2241 	return nr_total_pages;
2242 }
2243 
2244 static int hugetlb_acct_memory(struct hstate *h, long delta)
2245 {
2246 	int ret = -ENOMEM;
2247 
2248 	spin_lock(&hugetlb_lock);
2249 	/*
2250 	 * When cpuset is configured, it breaks the strict hugetlb page
2251 	 * reservation as the accounting is done on a global variable. Such
2252 	 * reservation is completely rubbish in the presence of cpuset because
2253 	 * the reservation is not checked against page availability for the
2254 	 * current cpuset. Application can still potentially OOM'ed by kernel
2255 	 * with lack of free htlb page in cpuset that the task is in.
2256 	 * Attempt to enforce strict accounting with cpuset is almost
2257 	 * impossible (or too ugly) because cpuset is too fluid that
2258 	 * task or memory node can be dynamically moved between cpusets.
2259 	 *
2260 	 * The change of semantics for shared hugetlb mapping with cpuset is
2261 	 * undesirable. However, in order to preserve some of the semantics,
2262 	 * we fall back to check against current free page availability as
2263 	 * a best attempt and hopefully to minimize the impact of changing
2264 	 * semantics that cpuset has.
2265 	 */
2266 	if (delta > 0) {
2267 		if (gather_surplus_pages(h, delta) < 0)
2268 			goto out;
2269 
2270 		if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2271 			return_unused_surplus_pages(h, delta);
2272 			goto out;
2273 		}
2274 	}
2275 
2276 	ret = 0;
2277 	if (delta < 0)
2278 		return_unused_surplus_pages(h, (unsigned long) -delta);
2279 
2280 out:
2281 	spin_unlock(&hugetlb_lock);
2282 	return ret;
2283 }
2284 
2285 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2286 {
2287 	struct resv_map *resv = vma_resv_map(vma);
2288 
2289 	/*
2290 	 * This new VMA should share its siblings reservation map if present.
2291 	 * The VMA will only ever have a valid reservation map pointer where
2292 	 * it is being copied for another still existing VMA.  As that VMA
2293 	 * has a reference to the reservation map it cannot disappear until
2294 	 * after this open call completes.  It is therefore safe to take a
2295 	 * new reference here without additional locking.
2296 	 */
2297 	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2298 		kref_get(&resv->refs);
2299 }
2300 
2301 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
2302 {
2303 	struct hstate *h = hstate_vma(vma);
2304 	struct resv_map *resv = vma_resv_map(vma);
2305 	struct hugepage_subpool *spool = subpool_vma(vma);
2306 	unsigned long reserve, start, end;
2307 
2308 	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
2309 		return;
2310 
2311 	start = vma_hugecache_offset(h, vma, vma->vm_start);
2312 	end = vma_hugecache_offset(h, vma, vma->vm_end);
2313 
2314 	reserve = (end - start) - region_count(resv, start, end);
2315 
2316 	kref_put(&resv->refs, resv_map_release);
2317 
2318 	if (reserve) {
2319 		hugetlb_acct_memory(h, -reserve);
2320 		hugepage_subpool_put_pages(spool, reserve);
2321 	}
2322 }
2323 
2324 /*
2325  * We cannot handle pagefaults against hugetlb pages at all.  They cause
2326  * handle_mm_fault() to try to instantiate regular-sized pages in the
2327  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
2328  * this far.
2329  */
2330 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2331 {
2332 	BUG();
2333 	return 0;
2334 }
2335 
2336 const struct vm_operations_struct hugetlb_vm_ops = {
2337 	.fault = hugetlb_vm_op_fault,
2338 	.open = hugetlb_vm_op_open,
2339 	.close = hugetlb_vm_op_close,
2340 };
2341 
2342 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2343 				int writable)
2344 {
2345 	pte_t entry;
2346 
2347 	if (writable) {
2348 		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2349 					 vma->vm_page_prot)));
2350 	} else {
2351 		entry = huge_pte_wrprotect(mk_huge_pte(page,
2352 					   vma->vm_page_prot));
2353 	}
2354 	entry = pte_mkyoung(entry);
2355 	entry = pte_mkhuge(entry);
2356 	entry = arch_make_huge_pte(entry, vma, page, writable);
2357 
2358 	return entry;
2359 }
2360 
2361 static void set_huge_ptep_writable(struct vm_area_struct *vma,
2362 				   unsigned long address, pte_t *ptep)
2363 {
2364 	pte_t entry;
2365 
2366 	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2367 	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2368 		update_mmu_cache(vma, address, ptep);
2369 }
2370 
2371 
2372 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2373 			    struct vm_area_struct *vma)
2374 {
2375 	pte_t *src_pte, *dst_pte, entry;
2376 	struct page *ptepage;
2377 	unsigned long addr;
2378 	int cow;
2379 	struct hstate *h = hstate_vma(vma);
2380 	unsigned long sz = huge_page_size(h);
2381 	unsigned long mmun_start;	/* For mmu_notifiers */
2382 	unsigned long mmun_end;		/* For mmu_notifiers */
2383 	int ret = 0;
2384 
2385 	cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
2386 
2387 	mmun_start = vma->vm_start;
2388 	mmun_end = vma->vm_end;
2389 	if (cow)
2390 		mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
2391 
2392 	for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
2393 		spinlock_t *src_ptl, *dst_ptl;
2394 		src_pte = huge_pte_offset(src, addr);
2395 		if (!src_pte)
2396 			continue;
2397 		dst_pte = huge_pte_alloc(dst, addr, sz);
2398 		if (!dst_pte) {
2399 			ret = -ENOMEM;
2400 			break;
2401 		}
2402 
2403 		/* If the pagetables are shared don't copy or take references */
2404 		if (dst_pte == src_pte)
2405 			continue;
2406 
2407 		dst_ptl = huge_pte_lock(h, dst, dst_pte);
2408 		src_ptl = huge_pte_lockptr(h, src, src_pte);
2409 		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
2410 		if (!huge_pte_none(huge_ptep_get(src_pte))) {
2411 			if (cow)
2412 				huge_ptep_set_wrprotect(src, addr, src_pte);
2413 			entry = huge_ptep_get(src_pte);
2414 			ptepage = pte_page(entry);
2415 			get_page(ptepage);
2416 			page_dup_rmap(ptepage);
2417 			set_huge_pte_at(dst, addr, dst_pte, entry);
2418 		}
2419 		spin_unlock(src_ptl);
2420 		spin_unlock(dst_ptl);
2421 	}
2422 
2423 	if (cow)
2424 		mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
2425 
2426 	return ret;
2427 }
2428 
2429 static int is_hugetlb_entry_migration(pte_t pte)
2430 {
2431 	swp_entry_t swp;
2432 
2433 	if (huge_pte_none(pte) || pte_present(pte))
2434 		return 0;
2435 	swp = pte_to_swp_entry(pte);
2436 	if (non_swap_entry(swp) && is_migration_entry(swp))
2437 		return 1;
2438 	else
2439 		return 0;
2440 }
2441 
2442 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
2443 {
2444 	swp_entry_t swp;
2445 
2446 	if (huge_pte_none(pte) || pte_present(pte))
2447 		return 0;
2448 	swp = pte_to_swp_entry(pte);
2449 	if (non_swap_entry(swp) && is_hwpoison_entry(swp))
2450 		return 1;
2451 	else
2452 		return 0;
2453 }
2454 
2455 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
2456 			    unsigned long start, unsigned long end,
2457 			    struct page *ref_page)
2458 {
2459 	int force_flush = 0;
2460 	struct mm_struct *mm = vma->vm_mm;
2461 	unsigned long address;
2462 	pte_t *ptep;
2463 	pte_t pte;
2464 	spinlock_t *ptl;
2465 	struct page *page;
2466 	struct hstate *h = hstate_vma(vma);
2467 	unsigned long sz = huge_page_size(h);
2468 	const unsigned long mmun_start = start;	/* For mmu_notifiers */
2469 	const unsigned long mmun_end   = end;	/* For mmu_notifiers */
2470 
2471 	WARN_ON(!is_vm_hugetlb_page(vma));
2472 	BUG_ON(start & ~huge_page_mask(h));
2473 	BUG_ON(end & ~huge_page_mask(h));
2474 
2475 	tlb_start_vma(tlb, vma);
2476 	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2477 again:
2478 	for (address = start; address < end; address += sz) {
2479 		ptep = huge_pte_offset(mm, address);
2480 		if (!ptep)
2481 			continue;
2482 
2483 		ptl = huge_pte_lock(h, mm, ptep);
2484 		if (huge_pmd_unshare(mm, &address, ptep))
2485 			goto unlock;
2486 
2487 		pte = huge_ptep_get(ptep);
2488 		if (huge_pte_none(pte))
2489 			goto unlock;
2490 
2491 		/*
2492 		 * HWPoisoned hugepage is already unmapped and dropped reference
2493 		 */
2494 		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
2495 			huge_pte_clear(mm, address, ptep);
2496 			goto unlock;
2497 		}
2498 
2499 		page = pte_page(pte);
2500 		/*
2501 		 * If a reference page is supplied, it is because a specific
2502 		 * page is being unmapped, not a range. Ensure the page we
2503 		 * are about to unmap is the actual page of interest.
2504 		 */
2505 		if (ref_page) {
2506 			if (page != ref_page)
2507 				goto unlock;
2508 
2509 			/*
2510 			 * Mark the VMA as having unmapped its page so that
2511 			 * future faults in this VMA will fail rather than
2512 			 * looking like data was lost
2513 			 */
2514 			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2515 		}
2516 
2517 		pte = huge_ptep_get_and_clear(mm, address, ptep);
2518 		tlb_remove_tlb_entry(tlb, ptep, address);
2519 		if (huge_pte_dirty(pte))
2520 			set_page_dirty(page);
2521 
2522 		page_remove_rmap(page);
2523 		force_flush = !__tlb_remove_page(tlb, page);
2524 		if (force_flush) {
2525 			spin_unlock(ptl);
2526 			break;
2527 		}
2528 		/* Bail out after unmapping reference page if supplied */
2529 		if (ref_page) {
2530 			spin_unlock(ptl);
2531 			break;
2532 		}
2533 unlock:
2534 		spin_unlock(ptl);
2535 	}
2536 	/*
2537 	 * mmu_gather ran out of room to batch pages, we break out of
2538 	 * the PTE lock to avoid doing the potential expensive TLB invalidate
2539 	 * and page-free while holding it.
2540 	 */
2541 	if (force_flush) {
2542 		force_flush = 0;
2543 		tlb_flush_mmu(tlb);
2544 		if (address < end && !ref_page)
2545 			goto again;
2546 	}
2547 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2548 	tlb_end_vma(tlb, vma);
2549 }
2550 
2551 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
2552 			  struct vm_area_struct *vma, unsigned long start,
2553 			  unsigned long end, struct page *ref_page)
2554 {
2555 	__unmap_hugepage_range(tlb, vma, start, end, ref_page);
2556 
2557 	/*
2558 	 * Clear this flag so that x86's huge_pmd_share page_table_shareable
2559 	 * test will fail on a vma being torn down, and not grab a page table
2560 	 * on its way out.  We're lucky that the flag has such an appropriate
2561 	 * name, and can in fact be safely cleared here. We could clear it
2562 	 * before the __unmap_hugepage_range above, but all that's necessary
2563 	 * is to clear it before releasing the i_mmap_mutex. This works
2564 	 * because in the context this is called, the VMA is about to be
2565 	 * destroyed and the i_mmap_mutex is held.
2566 	 */
2567 	vma->vm_flags &= ~VM_MAYSHARE;
2568 }
2569 
2570 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
2571 			  unsigned long end, struct page *ref_page)
2572 {
2573 	struct mm_struct *mm;
2574 	struct mmu_gather tlb;
2575 
2576 	mm = vma->vm_mm;
2577 
2578 	tlb_gather_mmu(&tlb, mm, start, end);
2579 	__unmap_hugepage_range(&tlb, vma, start, end, ref_page);
2580 	tlb_finish_mmu(&tlb, start, end);
2581 }
2582 
2583 /*
2584  * This is called when the original mapper is failing to COW a MAP_PRIVATE
2585  * mappping it owns the reserve page for. The intention is to unmap the page
2586  * from other VMAs and let the children be SIGKILLed if they are faulting the
2587  * same region.
2588  */
2589 static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
2590 				struct page *page, unsigned long address)
2591 {
2592 	struct hstate *h = hstate_vma(vma);
2593 	struct vm_area_struct *iter_vma;
2594 	struct address_space *mapping;
2595 	pgoff_t pgoff;
2596 
2597 	/*
2598 	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
2599 	 * from page cache lookup which is in HPAGE_SIZE units.
2600 	 */
2601 	address = address & huge_page_mask(h);
2602 	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
2603 			vma->vm_pgoff;
2604 	mapping = file_inode(vma->vm_file)->i_mapping;
2605 
2606 	/*
2607 	 * Take the mapping lock for the duration of the table walk. As
2608 	 * this mapping should be shared between all the VMAs,
2609 	 * __unmap_hugepage_range() is called as the lock is already held
2610 	 */
2611 	mutex_lock(&mapping->i_mmap_mutex);
2612 	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
2613 		/* Do not unmap the current VMA */
2614 		if (iter_vma == vma)
2615 			continue;
2616 
2617 		/*
2618 		 * Unmap the page from other VMAs without their own reserves.
2619 		 * They get marked to be SIGKILLed if they fault in these
2620 		 * areas. This is because a future no-page fault on this VMA
2621 		 * could insert a zeroed page instead of the data existing
2622 		 * from the time of fork. This would look like data corruption
2623 		 */
2624 		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
2625 			unmap_hugepage_range(iter_vma, address,
2626 					     address + huge_page_size(h), page);
2627 	}
2628 	mutex_unlock(&mapping->i_mmap_mutex);
2629 
2630 	return 1;
2631 }
2632 
2633 /*
2634  * Hugetlb_cow() should be called with page lock of the original hugepage held.
2635  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
2636  * cannot race with other handlers or page migration.
2637  * Keep the pte_same checks anyway to make transition from the mutex easier.
2638  */
2639 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
2640 			unsigned long address, pte_t *ptep, pte_t pte,
2641 			struct page *pagecache_page, spinlock_t *ptl)
2642 {
2643 	struct hstate *h = hstate_vma(vma);
2644 	struct page *old_page, *new_page;
2645 	int outside_reserve = 0;
2646 	unsigned long mmun_start;	/* For mmu_notifiers */
2647 	unsigned long mmun_end;		/* For mmu_notifiers */
2648 
2649 	old_page = pte_page(pte);
2650 
2651 retry_avoidcopy:
2652 	/* If no-one else is actually using this page, avoid the copy
2653 	 * and just make the page writable */
2654 	if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
2655 		page_move_anon_rmap(old_page, vma, address);
2656 		set_huge_ptep_writable(vma, address, ptep);
2657 		return 0;
2658 	}
2659 
2660 	/*
2661 	 * If the process that created a MAP_PRIVATE mapping is about to
2662 	 * perform a COW due to a shared page count, attempt to satisfy
2663 	 * the allocation without using the existing reserves. The pagecache
2664 	 * page is used to determine if the reserve at this address was
2665 	 * consumed or not. If reserves were used, a partial faulted mapping
2666 	 * at the time of fork() could consume its reserves on COW instead
2667 	 * of the full address range.
2668 	 */
2669 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
2670 			old_page != pagecache_page)
2671 		outside_reserve = 1;
2672 
2673 	page_cache_get(old_page);
2674 
2675 	/* Drop page table lock as buddy allocator may be called */
2676 	spin_unlock(ptl);
2677 	new_page = alloc_huge_page(vma, address, outside_reserve);
2678 
2679 	if (IS_ERR(new_page)) {
2680 		long err = PTR_ERR(new_page);
2681 		page_cache_release(old_page);
2682 
2683 		/*
2684 		 * If a process owning a MAP_PRIVATE mapping fails to COW,
2685 		 * it is due to references held by a child and an insufficient
2686 		 * huge page pool. To guarantee the original mappers
2687 		 * reliability, unmap the page from child processes. The child
2688 		 * may get SIGKILLed if it later faults.
2689 		 */
2690 		if (outside_reserve) {
2691 			BUG_ON(huge_pte_none(pte));
2692 			if (unmap_ref_private(mm, vma, old_page, address)) {
2693 				BUG_ON(huge_pte_none(pte));
2694 				spin_lock(ptl);
2695 				ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2696 				if (likely(ptep &&
2697 					   pte_same(huge_ptep_get(ptep), pte)))
2698 					goto retry_avoidcopy;
2699 				/*
2700 				 * race occurs while re-acquiring page table
2701 				 * lock, and our job is done.
2702 				 */
2703 				return 0;
2704 			}
2705 			WARN_ON_ONCE(1);
2706 		}
2707 
2708 		/* Caller expects lock to be held */
2709 		spin_lock(ptl);
2710 		if (err == -ENOMEM)
2711 			return VM_FAULT_OOM;
2712 		else
2713 			return VM_FAULT_SIGBUS;
2714 	}
2715 
2716 	/*
2717 	 * When the original hugepage is shared one, it does not have
2718 	 * anon_vma prepared.
2719 	 */
2720 	if (unlikely(anon_vma_prepare(vma))) {
2721 		page_cache_release(new_page);
2722 		page_cache_release(old_page);
2723 		/* Caller expects lock to be held */
2724 		spin_lock(ptl);
2725 		return VM_FAULT_OOM;
2726 	}
2727 
2728 	copy_user_huge_page(new_page, old_page, address, vma,
2729 			    pages_per_huge_page(h));
2730 	__SetPageUptodate(new_page);
2731 
2732 	mmun_start = address & huge_page_mask(h);
2733 	mmun_end = mmun_start + huge_page_size(h);
2734 	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
2735 	/*
2736 	 * Retake the page table lock to check for racing updates
2737 	 * before the page tables are altered
2738 	 */
2739 	spin_lock(ptl);
2740 	ptep = huge_pte_offset(mm, address & huge_page_mask(h));
2741 	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
2742 		ClearPagePrivate(new_page);
2743 
2744 		/* Break COW */
2745 		huge_ptep_clear_flush(vma, address, ptep);
2746 		set_huge_pte_at(mm, address, ptep,
2747 				make_huge_pte(vma, new_page, 1));
2748 		page_remove_rmap(old_page);
2749 		hugepage_add_new_anon_rmap(new_page, vma, address);
2750 		/* Make the old page be freed below */
2751 		new_page = old_page;
2752 	}
2753 	spin_unlock(ptl);
2754 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
2755 	page_cache_release(new_page);
2756 	page_cache_release(old_page);
2757 
2758 	/* Caller expects lock to be held */
2759 	spin_lock(ptl);
2760 	return 0;
2761 }
2762 
2763 /* Return the pagecache page at a given address within a VMA */
2764 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
2765 			struct vm_area_struct *vma, unsigned long address)
2766 {
2767 	struct address_space *mapping;
2768 	pgoff_t idx;
2769 
2770 	mapping = vma->vm_file->f_mapping;
2771 	idx = vma_hugecache_offset(h, vma, address);
2772 
2773 	return find_lock_page(mapping, idx);
2774 }
2775 
2776 /*
2777  * Return whether there is a pagecache page to back given address within VMA.
2778  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
2779  */
2780 static bool hugetlbfs_pagecache_present(struct hstate *h,
2781 			struct vm_area_struct *vma, unsigned long address)
2782 {
2783 	struct address_space *mapping;
2784 	pgoff_t idx;
2785 	struct page *page;
2786 
2787 	mapping = vma->vm_file->f_mapping;
2788 	idx = vma_hugecache_offset(h, vma, address);
2789 
2790 	page = find_get_page(mapping, idx);
2791 	if (page)
2792 		put_page(page);
2793 	return page != NULL;
2794 }
2795 
2796 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
2797 			   struct address_space *mapping, pgoff_t idx,
2798 			   unsigned long address, pte_t *ptep, unsigned int flags)
2799 {
2800 	struct hstate *h = hstate_vma(vma);
2801 	int ret = VM_FAULT_SIGBUS;
2802 	int anon_rmap = 0;
2803 	unsigned long size;
2804 	struct page *page;
2805 	pte_t new_pte;
2806 	spinlock_t *ptl;
2807 
2808 	/*
2809 	 * Currently, we are forced to kill the process in the event the
2810 	 * original mapper has unmapped pages from the child due to a failed
2811 	 * COW. Warn that such a situation has occurred as it may not be obvious
2812 	 */
2813 	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
2814 		pr_warning("PID %d killed due to inadequate hugepage pool\n",
2815 			   current->pid);
2816 		return ret;
2817 	}
2818 
2819 	/*
2820 	 * Use page lock to guard against racing truncation
2821 	 * before we get page_table_lock.
2822 	 */
2823 retry:
2824 	page = find_lock_page(mapping, idx);
2825 	if (!page) {
2826 		size = i_size_read(mapping->host) >> huge_page_shift(h);
2827 		if (idx >= size)
2828 			goto out;
2829 		page = alloc_huge_page(vma, address, 0);
2830 		if (IS_ERR(page)) {
2831 			ret = PTR_ERR(page);
2832 			if (ret == -ENOMEM)
2833 				ret = VM_FAULT_OOM;
2834 			else
2835 				ret = VM_FAULT_SIGBUS;
2836 			goto out;
2837 		}
2838 		clear_huge_page(page, address, pages_per_huge_page(h));
2839 		__SetPageUptodate(page);
2840 
2841 		if (vma->vm_flags & VM_MAYSHARE) {
2842 			int err;
2843 			struct inode *inode = mapping->host;
2844 
2845 			err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
2846 			if (err) {
2847 				put_page(page);
2848 				if (err == -EEXIST)
2849 					goto retry;
2850 				goto out;
2851 			}
2852 			ClearPagePrivate(page);
2853 
2854 			spin_lock(&inode->i_lock);
2855 			inode->i_blocks += blocks_per_huge_page(h);
2856 			spin_unlock(&inode->i_lock);
2857 		} else {
2858 			lock_page(page);
2859 			if (unlikely(anon_vma_prepare(vma))) {
2860 				ret = VM_FAULT_OOM;
2861 				goto backout_unlocked;
2862 			}
2863 			anon_rmap = 1;
2864 		}
2865 	} else {
2866 		/*
2867 		 * If memory error occurs between mmap() and fault, some process
2868 		 * don't have hwpoisoned swap entry for errored virtual address.
2869 		 * So we need to block hugepage fault by PG_hwpoison bit check.
2870 		 */
2871 		if (unlikely(PageHWPoison(page))) {
2872 			ret = VM_FAULT_HWPOISON |
2873 				VM_FAULT_SET_HINDEX(hstate_index(h));
2874 			goto backout_unlocked;
2875 		}
2876 	}
2877 
2878 	/*
2879 	 * If we are going to COW a private mapping later, we examine the
2880 	 * pending reservations for this page now. This will ensure that
2881 	 * any allocations necessary to record that reservation occur outside
2882 	 * the spinlock.
2883 	 */
2884 	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED))
2885 		if (vma_needs_reservation(h, vma, address) < 0) {
2886 			ret = VM_FAULT_OOM;
2887 			goto backout_unlocked;
2888 		}
2889 
2890 	ptl = huge_pte_lockptr(h, mm, ptep);
2891 	spin_lock(ptl);
2892 	size = i_size_read(mapping->host) >> huge_page_shift(h);
2893 	if (idx >= size)
2894 		goto backout;
2895 
2896 	ret = 0;
2897 	if (!huge_pte_none(huge_ptep_get(ptep)))
2898 		goto backout;
2899 
2900 	if (anon_rmap) {
2901 		ClearPagePrivate(page);
2902 		hugepage_add_new_anon_rmap(page, vma, address);
2903 	} else
2904 		page_dup_rmap(page);
2905 	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
2906 				&& (vma->vm_flags & VM_SHARED)));
2907 	set_huge_pte_at(mm, address, ptep, new_pte);
2908 
2909 	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
2910 		/* Optimization, do the COW without a second fault */
2911 		ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
2912 	}
2913 
2914 	spin_unlock(ptl);
2915 	unlock_page(page);
2916 out:
2917 	return ret;
2918 
2919 backout:
2920 	spin_unlock(ptl);
2921 backout_unlocked:
2922 	unlock_page(page);
2923 	put_page(page);
2924 	goto out;
2925 }
2926 
2927 #ifdef CONFIG_SMP
2928 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
2929 			    struct vm_area_struct *vma,
2930 			    struct address_space *mapping,
2931 			    pgoff_t idx, unsigned long address)
2932 {
2933 	unsigned long key[2];
2934 	u32 hash;
2935 
2936 	if (vma->vm_flags & VM_SHARED) {
2937 		key[0] = (unsigned long) mapping;
2938 		key[1] = idx;
2939 	} else {
2940 		key[0] = (unsigned long) mm;
2941 		key[1] = address >> huge_page_shift(h);
2942 	}
2943 
2944 	hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
2945 
2946 	return hash & (num_fault_mutexes - 1);
2947 }
2948 #else
2949 /*
2950  * For uniprocesor systems we always use a single mutex, so just
2951  * return 0 and avoid the hashing overhead.
2952  */
2953 static u32 fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
2954 			    struct vm_area_struct *vma,
2955 			    struct address_space *mapping,
2956 			    pgoff_t idx, unsigned long address)
2957 {
2958 	return 0;
2959 }
2960 #endif
2961 
2962 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
2963 			unsigned long address, unsigned int flags)
2964 {
2965 	pte_t *ptep, entry;
2966 	spinlock_t *ptl;
2967 	int ret;
2968 	u32 hash;
2969 	pgoff_t idx;
2970 	struct page *page = NULL;
2971 	struct page *pagecache_page = NULL;
2972 	struct hstate *h = hstate_vma(vma);
2973 	struct address_space *mapping;
2974 
2975 	address &= huge_page_mask(h);
2976 
2977 	ptep = huge_pte_offset(mm, address);
2978 	if (ptep) {
2979 		entry = huge_ptep_get(ptep);
2980 		if (unlikely(is_hugetlb_entry_migration(entry))) {
2981 			migration_entry_wait_huge(vma, mm, ptep);
2982 			return 0;
2983 		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
2984 			return VM_FAULT_HWPOISON_LARGE |
2985 				VM_FAULT_SET_HINDEX(hstate_index(h));
2986 	}
2987 
2988 	ptep = huge_pte_alloc(mm, address, huge_page_size(h));
2989 	if (!ptep)
2990 		return VM_FAULT_OOM;
2991 
2992 	mapping = vma->vm_file->f_mapping;
2993 	idx = vma_hugecache_offset(h, vma, address);
2994 
2995 	/*
2996 	 * Serialize hugepage allocation and instantiation, so that we don't
2997 	 * get spurious allocation failures if two CPUs race to instantiate
2998 	 * the same page in the page cache.
2999 	 */
3000 	hash = fault_mutex_hash(h, mm, vma, mapping, idx, address);
3001 	mutex_lock(&htlb_fault_mutex_table[hash]);
3002 
3003 	entry = huge_ptep_get(ptep);
3004 	if (huge_pte_none(entry)) {
3005 		ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3006 		goto out_mutex;
3007 	}
3008 
3009 	ret = 0;
3010 
3011 	/*
3012 	 * If we are going to COW the mapping later, we examine the pending
3013 	 * reservations for this page now. This will ensure that any
3014 	 * allocations necessary to record that reservation occur outside the
3015 	 * spinlock. For private mappings, we also lookup the pagecache
3016 	 * page now as it is used to determine if a reservation has been
3017 	 * consumed.
3018 	 */
3019 	if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3020 		if (vma_needs_reservation(h, vma, address) < 0) {
3021 			ret = VM_FAULT_OOM;
3022 			goto out_mutex;
3023 		}
3024 
3025 		if (!(vma->vm_flags & VM_MAYSHARE))
3026 			pagecache_page = hugetlbfs_pagecache_page(h,
3027 								vma, address);
3028 	}
3029 
3030 	/*
3031 	 * hugetlb_cow() requires page locks of pte_page(entry) and
3032 	 * pagecache_page, so here we need take the former one
3033 	 * when page != pagecache_page or !pagecache_page.
3034 	 * Note that locking order is always pagecache_page -> page,
3035 	 * so no worry about deadlock.
3036 	 */
3037 	page = pte_page(entry);
3038 	get_page(page);
3039 	if (page != pagecache_page)
3040 		lock_page(page);
3041 
3042 	ptl = huge_pte_lockptr(h, mm, ptep);
3043 	spin_lock(ptl);
3044 	/* Check for a racing update before calling hugetlb_cow */
3045 	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3046 		goto out_ptl;
3047 
3048 
3049 	if (flags & FAULT_FLAG_WRITE) {
3050 		if (!huge_pte_write(entry)) {
3051 			ret = hugetlb_cow(mm, vma, address, ptep, entry,
3052 					pagecache_page, ptl);
3053 			goto out_ptl;
3054 		}
3055 		entry = huge_pte_mkdirty(entry);
3056 	}
3057 	entry = pte_mkyoung(entry);
3058 	if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3059 						flags & FAULT_FLAG_WRITE))
3060 		update_mmu_cache(vma, address, ptep);
3061 
3062 out_ptl:
3063 	spin_unlock(ptl);
3064 
3065 	if (pagecache_page) {
3066 		unlock_page(pagecache_page);
3067 		put_page(pagecache_page);
3068 	}
3069 	if (page != pagecache_page)
3070 		unlock_page(page);
3071 	put_page(page);
3072 
3073 out_mutex:
3074 	mutex_unlock(&htlb_fault_mutex_table[hash]);
3075 	return ret;
3076 }
3077 
3078 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3079 			 struct page **pages, struct vm_area_struct **vmas,
3080 			 unsigned long *position, unsigned long *nr_pages,
3081 			 long i, unsigned int flags)
3082 {
3083 	unsigned long pfn_offset;
3084 	unsigned long vaddr = *position;
3085 	unsigned long remainder = *nr_pages;
3086 	struct hstate *h = hstate_vma(vma);
3087 
3088 	while (vaddr < vma->vm_end && remainder) {
3089 		pte_t *pte;
3090 		spinlock_t *ptl = NULL;
3091 		int absent;
3092 		struct page *page;
3093 
3094 		/*
3095 		 * Some archs (sparc64, sh*) have multiple pte_ts to
3096 		 * each hugepage.  We have to make sure we get the
3097 		 * first, for the page indexing below to work.
3098 		 *
3099 		 * Note that page table lock is not held when pte is null.
3100 		 */
3101 		pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3102 		if (pte)
3103 			ptl = huge_pte_lock(h, mm, pte);
3104 		absent = !pte || huge_pte_none(huge_ptep_get(pte));
3105 
3106 		/*
3107 		 * When coredumping, it suits get_dump_page if we just return
3108 		 * an error where there's an empty slot with no huge pagecache
3109 		 * to back it.  This way, we avoid allocating a hugepage, and
3110 		 * the sparse dumpfile avoids allocating disk blocks, but its
3111 		 * huge holes still show up with zeroes where they need to be.
3112 		 */
3113 		if (absent && (flags & FOLL_DUMP) &&
3114 		    !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3115 			if (pte)
3116 				spin_unlock(ptl);
3117 			remainder = 0;
3118 			break;
3119 		}
3120 
3121 		/*
3122 		 * We need call hugetlb_fault for both hugepages under migration
3123 		 * (in which case hugetlb_fault waits for the migration,) and
3124 		 * hwpoisoned hugepages (in which case we need to prevent the
3125 		 * caller from accessing to them.) In order to do this, we use
3126 		 * here is_swap_pte instead of is_hugetlb_entry_migration and
3127 		 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3128 		 * both cases, and because we can't follow correct pages
3129 		 * directly from any kind of swap entries.
3130 		 */
3131 		if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3132 		    ((flags & FOLL_WRITE) &&
3133 		      !huge_pte_write(huge_ptep_get(pte)))) {
3134 			int ret;
3135 
3136 			if (pte)
3137 				spin_unlock(ptl);
3138 			ret = hugetlb_fault(mm, vma, vaddr,
3139 				(flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3140 			if (!(ret & VM_FAULT_ERROR))
3141 				continue;
3142 
3143 			remainder = 0;
3144 			break;
3145 		}
3146 
3147 		pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3148 		page = pte_page(huge_ptep_get(pte));
3149 same_page:
3150 		if (pages) {
3151 			pages[i] = mem_map_offset(page, pfn_offset);
3152 			get_page_foll(pages[i]);
3153 		}
3154 
3155 		if (vmas)
3156 			vmas[i] = vma;
3157 
3158 		vaddr += PAGE_SIZE;
3159 		++pfn_offset;
3160 		--remainder;
3161 		++i;
3162 		if (vaddr < vma->vm_end && remainder &&
3163 				pfn_offset < pages_per_huge_page(h)) {
3164 			/*
3165 			 * We use pfn_offset to avoid touching the pageframes
3166 			 * of this compound page.
3167 			 */
3168 			goto same_page;
3169 		}
3170 		spin_unlock(ptl);
3171 	}
3172 	*nr_pages = remainder;
3173 	*position = vaddr;
3174 
3175 	return i ? i : -EFAULT;
3176 }
3177 
3178 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3179 		unsigned long address, unsigned long end, pgprot_t newprot)
3180 {
3181 	struct mm_struct *mm = vma->vm_mm;
3182 	unsigned long start = address;
3183 	pte_t *ptep;
3184 	pte_t pte;
3185 	struct hstate *h = hstate_vma(vma);
3186 	unsigned long pages = 0;
3187 
3188 	BUG_ON(address >= end);
3189 	flush_cache_range(vma, address, end);
3190 
3191 	mmu_notifier_invalidate_range_start(mm, start, end);
3192 	mutex_lock(&vma->vm_file->f_mapping->i_mmap_mutex);
3193 	for (; address < end; address += huge_page_size(h)) {
3194 		spinlock_t *ptl;
3195 		ptep = huge_pte_offset(mm, address);
3196 		if (!ptep)
3197 			continue;
3198 		ptl = huge_pte_lock(h, mm, ptep);
3199 		if (huge_pmd_unshare(mm, &address, ptep)) {
3200 			pages++;
3201 			spin_unlock(ptl);
3202 			continue;
3203 		}
3204 		if (!huge_pte_none(huge_ptep_get(ptep))) {
3205 			pte = huge_ptep_get_and_clear(mm, address, ptep);
3206 			pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3207 			pte = arch_make_huge_pte(pte, vma, NULL, 0);
3208 			set_huge_pte_at(mm, address, ptep, pte);
3209 			pages++;
3210 		}
3211 		spin_unlock(ptl);
3212 	}
3213 	/*
3214 	 * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare
3215 	 * may have cleared our pud entry and done put_page on the page table:
3216 	 * once we release i_mmap_mutex, another task can do the final put_page
3217 	 * and that page table be reused and filled with junk.
3218 	 */
3219 	flush_tlb_range(vma, start, end);
3220 	mutex_unlock(&vma->vm_file->f_mapping->i_mmap_mutex);
3221 	mmu_notifier_invalidate_range_end(mm, start, end);
3222 
3223 	return pages << h->order;
3224 }
3225 
3226 int hugetlb_reserve_pages(struct inode *inode,
3227 					long from, long to,
3228 					struct vm_area_struct *vma,
3229 					vm_flags_t vm_flags)
3230 {
3231 	long ret, chg;
3232 	struct hstate *h = hstate_inode(inode);
3233 	struct hugepage_subpool *spool = subpool_inode(inode);
3234 	struct resv_map *resv_map;
3235 
3236 	/*
3237 	 * Only apply hugepage reservation if asked. At fault time, an
3238 	 * attempt will be made for VM_NORESERVE to allocate a page
3239 	 * without using reserves
3240 	 */
3241 	if (vm_flags & VM_NORESERVE)
3242 		return 0;
3243 
3244 	/*
3245 	 * Shared mappings base their reservation on the number of pages that
3246 	 * are already allocated on behalf of the file. Private mappings need
3247 	 * to reserve the full area even if read-only as mprotect() may be
3248 	 * called to make the mapping read-write. Assume !vma is a shm mapping
3249 	 */
3250 	if (!vma || vma->vm_flags & VM_MAYSHARE) {
3251 		resv_map = inode_resv_map(inode);
3252 
3253 		chg = region_chg(resv_map, from, to);
3254 
3255 	} else {
3256 		resv_map = resv_map_alloc();
3257 		if (!resv_map)
3258 			return -ENOMEM;
3259 
3260 		chg = to - from;
3261 
3262 		set_vma_resv_map(vma, resv_map);
3263 		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
3264 	}
3265 
3266 	if (chg < 0) {
3267 		ret = chg;
3268 		goto out_err;
3269 	}
3270 
3271 	/* There must be enough pages in the subpool for the mapping */
3272 	if (hugepage_subpool_get_pages(spool, chg)) {
3273 		ret = -ENOSPC;
3274 		goto out_err;
3275 	}
3276 
3277 	/*
3278 	 * Check enough hugepages are available for the reservation.
3279 	 * Hand the pages back to the subpool if there are not
3280 	 */
3281 	ret = hugetlb_acct_memory(h, chg);
3282 	if (ret < 0) {
3283 		hugepage_subpool_put_pages(spool, chg);
3284 		goto out_err;
3285 	}
3286 
3287 	/*
3288 	 * Account for the reservations made. Shared mappings record regions
3289 	 * that have reservations as they are shared by multiple VMAs.
3290 	 * When the last VMA disappears, the region map says how much
3291 	 * the reservation was and the page cache tells how much of
3292 	 * the reservation was consumed. Private mappings are per-VMA and
3293 	 * only the consumed reservations are tracked. When the VMA
3294 	 * disappears, the original reservation is the VMA size and the
3295 	 * consumed reservations are stored in the map. Hence, nothing
3296 	 * else has to be done for private mappings here
3297 	 */
3298 	if (!vma || vma->vm_flags & VM_MAYSHARE)
3299 		region_add(resv_map, from, to);
3300 	return 0;
3301 out_err:
3302 	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3303 		kref_put(&resv_map->refs, resv_map_release);
3304 	return ret;
3305 }
3306 
3307 void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
3308 {
3309 	struct hstate *h = hstate_inode(inode);
3310 	struct resv_map *resv_map = inode_resv_map(inode);
3311 	long chg = 0;
3312 	struct hugepage_subpool *spool = subpool_inode(inode);
3313 
3314 	if (resv_map)
3315 		chg = region_truncate(resv_map, offset);
3316 	spin_lock(&inode->i_lock);
3317 	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
3318 	spin_unlock(&inode->i_lock);
3319 
3320 	hugepage_subpool_put_pages(spool, (chg - freed));
3321 	hugetlb_acct_memory(h, -(chg - freed));
3322 }
3323 
3324 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
3325 static unsigned long page_table_shareable(struct vm_area_struct *svma,
3326 				struct vm_area_struct *vma,
3327 				unsigned long addr, pgoff_t idx)
3328 {
3329 	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
3330 				svma->vm_start;
3331 	unsigned long sbase = saddr & PUD_MASK;
3332 	unsigned long s_end = sbase + PUD_SIZE;
3333 
3334 	/* Allow segments to share if only one is marked locked */
3335 	unsigned long vm_flags = vma->vm_flags & ~VM_LOCKED;
3336 	unsigned long svm_flags = svma->vm_flags & ~VM_LOCKED;
3337 
3338 	/*
3339 	 * match the virtual addresses, permission and the alignment of the
3340 	 * page table page.
3341 	 */
3342 	if (pmd_index(addr) != pmd_index(saddr) ||
3343 	    vm_flags != svm_flags ||
3344 	    sbase < svma->vm_start || svma->vm_end < s_end)
3345 		return 0;
3346 
3347 	return saddr;
3348 }
3349 
3350 static int vma_shareable(struct vm_area_struct *vma, unsigned long addr)
3351 {
3352 	unsigned long base = addr & PUD_MASK;
3353 	unsigned long end = base + PUD_SIZE;
3354 
3355 	/*
3356 	 * check on proper vm_flags and page table alignment
3357 	 */
3358 	if (vma->vm_flags & VM_MAYSHARE &&
3359 	    vma->vm_start <= base && end <= vma->vm_end)
3360 		return 1;
3361 	return 0;
3362 }
3363 
3364 /*
3365  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
3366  * and returns the corresponding pte. While this is not necessary for the
3367  * !shared pmd case because we can allocate the pmd later as well, it makes the
3368  * code much cleaner. pmd allocation is essential for the shared case because
3369  * pud has to be populated inside the same i_mmap_mutex section - otherwise
3370  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
3371  * bad pmd for sharing.
3372  */
3373 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3374 {
3375 	struct vm_area_struct *vma = find_vma(mm, addr);
3376 	struct address_space *mapping = vma->vm_file->f_mapping;
3377 	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
3378 			vma->vm_pgoff;
3379 	struct vm_area_struct *svma;
3380 	unsigned long saddr;
3381 	pte_t *spte = NULL;
3382 	pte_t *pte;
3383 	spinlock_t *ptl;
3384 
3385 	if (!vma_shareable(vma, addr))
3386 		return (pte_t *)pmd_alloc(mm, pud, addr);
3387 
3388 	mutex_lock(&mapping->i_mmap_mutex);
3389 	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
3390 		if (svma == vma)
3391 			continue;
3392 
3393 		saddr = page_table_shareable(svma, vma, addr, idx);
3394 		if (saddr) {
3395 			spte = huge_pte_offset(svma->vm_mm, saddr);
3396 			if (spte) {
3397 				get_page(virt_to_page(spte));
3398 				break;
3399 			}
3400 		}
3401 	}
3402 
3403 	if (!spte)
3404 		goto out;
3405 
3406 	ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
3407 	spin_lock(ptl);
3408 	if (pud_none(*pud))
3409 		pud_populate(mm, pud,
3410 				(pmd_t *)((unsigned long)spte & PAGE_MASK));
3411 	else
3412 		put_page(virt_to_page(spte));
3413 	spin_unlock(ptl);
3414 out:
3415 	pte = (pte_t *)pmd_alloc(mm, pud, addr);
3416 	mutex_unlock(&mapping->i_mmap_mutex);
3417 	return pte;
3418 }
3419 
3420 /*
3421  * unmap huge page backed by shared pte.
3422  *
3423  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
3424  * indicated by page_count > 1, unmap is achieved by clearing pud and
3425  * decrementing the ref count. If count == 1, the pte page is not shared.
3426  *
3427  * called with page table lock held.
3428  *
3429  * returns: 1 successfully unmapped a shared pte page
3430  *	    0 the underlying pte page is not shared, or it is the last user
3431  */
3432 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
3433 {
3434 	pgd_t *pgd = pgd_offset(mm, *addr);
3435 	pud_t *pud = pud_offset(pgd, *addr);
3436 
3437 	BUG_ON(page_count(virt_to_page(ptep)) == 0);
3438 	if (page_count(virt_to_page(ptep)) == 1)
3439 		return 0;
3440 
3441 	pud_clear(pud);
3442 	put_page(virt_to_page(ptep));
3443 	*addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
3444 	return 1;
3445 }
3446 #define want_pmd_share()	(1)
3447 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3448 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
3449 {
3450 	return NULL;
3451 }
3452 #define want_pmd_share()	(0)
3453 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
3454 
3455 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
3456 pte_t *huge_pte_alloc(struct mm_struct *mm,
3457 			unsigned long addr, unsigned long sz)
3458 {
3459 	pgd_t *pgd;
3460 	pud_t *pud;
3461 	pte_t *pte = NULL;
3462 
3463 	pgd = pgd_offset(mm, addr);
3464 	pud = pud_alloc(mm, pgd, addr);
3465 	if (pud) {
3466 		if (sz == PUD_SIZE) {
3467 			pte = (pte_t *)pud;
3468 		} else {
3469 			BUG_ON(sz != PMD_SIZE);
3470 			if (want_pmd_share() && pud_none(*pud))
3471 				pte = huge_pmd_share(mm, addr, pud);
3472 			else
3473 				pte = (pte_t *)pmd_alloc(mm, pud, addr);
3474 		}
3475 	}
3476 	BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
3477 
3478 	return pte;
3479 }
3480 
3481 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
3482 {
3483 	pgd_t *pgd;
3484 	pud_t *pud;
3485 	pmd_t *pmd = NULL;
3486 
3487 	pgd = pgd_offset(mm, addr);
3488 	if (pgd_present(*pgd)) {
3489 		pud = pud_offset(pgd, addr);
3490 		if (pud_present(*pud)) {
3491 			if (pud_huge(*pud))
3492 				return (pte_t *)pud;
3493 			pmd = pmd_offset(pud, addr);
3494 		}
3495 	}
3496 	return (pte_t *) pmd;
3497 }
3498 
3499 struct page *
3500 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
3501 		pmd_t *pmd, int write)
3502 {
3503 	struct page *page;
3504 
3505 	page = pte_page(*(pte_t *)pmd);
3506 	if (page)
3507 		page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
3508 	return page;
3509 }
3510 
3511 struct page *
3512 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3513 		pud_t *pud, int write)
3514 {
3515 	struct page *page;
3516 
3517 	page = pte_page(*(pte_t *)pud);
3518 	if (page)
3519 		page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
3520 	return page;
3521 }
3522 
3523 #else /* !CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3524 
3525 /* Can be overriden by architectures */
3526 struct page * __weak
3527 follow_huge_pud(struct mm_struct *mm, unsigned long address,
3528 	       pud_t *pud, int write)
3529 {
3530 	BUG();
3531 	return NULL;
3532 }
3533 
3534 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
3535 
3536 #ifdef CONFIG_MEMORY_FAILURE
3537 
3538 /* Should be called in hugetlb_lock */
3539 static int is_hugepage_on_freelist(struct page *hpage)
3540 {
3541 	struct page *page;
3542 	struct page *tmp;
3543 	struct hstate *h = page_hstate(hpage);
3544 	int nid = page_to_nid(hpage);
3545 
3546 	list_for_each_entry_safe(page, tmp, &h->hugepage_freelists[nid], lru)
3547 		if (page == hpage)
3548 			return 1;
3549 	return 0;
3550 }
3551 
3552 /*
3553  * This function is called from memory failure code.
3554  * Assume the caller holds page lock of the head page.
3555  */
3556 int dequeue_hwpoisoned_huge_page(struct page *hpage)
3557 {
3558 	struct hstate *h = page_hstate(hpage);
3559 	int nid = page_to_nid(hpage);
3560 	int ret = -EBUSY;
3561 
3562 	spin_lock(&hugetlb_lock);
3563 	if (is_hugepage_on_freelist(hpage)) {
3564 		/*
3565 		 * Hwpoisoned hugepage isn't linked to activelist or freelist,
3566 		 * but dangling hpage->lru can trigger list-debug warnings
3567 		 * (this happens when we call unpoison_memory() on it),
3568 		 * so let it point to itself with list_del_init().
3569 		 */
3570 		list_del_init(&hpage->lru);
3571 		set_page_refcounted(hpage);
3572 		h->free_huge_pages--;
3573 		h->free_huge_pages_node[nid]--;
3574 		ret = 0;
3575 	}
3576 	spin_unlock(&hugetlb_lock);
3577 	return ret;
3578 }
3579 #endif
3580 
3581 bool isolate_huge_page(struct page *page, struct list_head *list)
3582 {
3583 	VM_BUG_ON_PAGE(!PageHead(page), page);
3584 	if (!get_page_unless_zero(page))
3585 		return false;
3586 	spin_lock(&hugetlb_lock);
3587 	list_move_tail(&page->lru, list);
3588 	spin_unlock(&hugetlb_lock);
3589 	return true;
3590 }
3591 
3592 void putback_active_hugepage(struct page *page)
3593 {
3594 	VM_BUG_ON_PAGE(!PageHead(page), page);
3595 	spin_lock(&hugetlb_lock);
3596 	list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
3597 	spin_unlock(&hugetlb_lock);
3598 	put_page(page);
3599 }
3600 
3601 bool is_hugepage_active(struct page *page)
3602 {
3603 	VM_BUG_ON_PAGE(!PageHuge(page), page);
3604 	/*
3605 	 * This function can be called for a tail page because the caller,
3606 	 * scan_movable_pages, scans through a given pfn-range which typically
3607 	 * covers one memory block. In systems using gigantic hugepage (1GB
3608 	 * for x86_64,) a hugepage is larger than a memory block, and we don't
3609 	 * support migrating such large hugepages for now, so return false
3610 	 * when called for tail pages.
3611 	 */
3612 	if (PageTail(page))
3613 		return false;
3614 	/*
3615 	 * Refcount of a hwpoisoned hugepages is 1, but they are not active,
3616 	 * so we should return false for them.
3617 	 */
3618 	if (unlikely(PageHWPoison(page)))
3619 		return false;
3620 	return page_count(page) > 0;
3621 }
3622