xref: /openbmc/linux/mm/hugetlb.c (revision f3539c12d8196ce0a1993364d30b3a18908470d1)
1 /*
2  * Generic hugetlb support.
3  * (C) Nadia Yvette Chambers, April 2004
4  */
5 #include <linux/list.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/seq_file.h>
9 #include <linux/sysctl.h>
10 #include <linux/highmem.h>
11 #include <linux/mmu_notifier.h>
12 #include <linux/nodemask.h>
13 #include <linux/pagemap.h>
14 #include <linux/mempolicy.h>
15 #include <linux/compiler.h>
16 #include <linux/cpuset.h>
17 #include <linux/mutex.h>
18 #include <linux/bootmem.h>
19 #include <linux/sysfs.h>
20 #include <linux/slab.h>
21 #include <linux/rmap.h>
22 #include <linux/swap.h>
23 #include <linux/swapops.h>
24 #include <linux/page-isolation.h>
25 #include <linux/jhash.h>
26 
27 #include <asm/page.h>
28 #include <asm/pgtable.h>
29 #include <asm/tlb.h>
30 
31 #include <linux/io.h>
32 #include <linux/hugetlb.h>
33 #include <linux/hugetlb_cgroup.h>
34 #include <linux/node.h>
35 #include "internal.h"
36 
37 int hugepages_treat_as_movable;
38 
39 int hugetlb_max_hstate __read_mostly;
40 unsigned int default_hstate_idx;
41 struct hstate hstates[HUGE_MAX_HSTATE];
42 /*
43  * Minimum page order among possible hugepage sizes, set to a proper value
44  * at boot time.
45  */
46 static unsigned int minimum_order __read_mostly = UINT_MAX;
47 
48 __initdata LIST_HEAD(huge_boot_pages);
49 
50 /* for command line parsing */
51 static struct hstate * __initdata parsed_hstate;
52 static unsigned long __initdata default_hstate_max_huge_pages;
53 static unsigned long __initdata default_hstate_size;
54 static bool __initdata parsed_valid_hugepagesz = true;
55 
56 /*
57  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
58  * free_huge_pages, and surplus_huge_pages.
59  */
60 DEFINE_SPINLOCK(hugetlb_lock);
61 
62 /*
63  * Serializes faults on the same logical page.  This is used to
64  * prevent spurious OOMs when the hugepage pool is fully utilized.
65  */
66 static int num_fault_mutexes;
67 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
68 
69 /* Forward declaration */
70 static int hugetlb_acct_memory(struct hstate *h, long delta);
71 
72 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool)
73 {
74 	bool free = (spool->count == 0) && (spool->used_hpages == 0);
75 
76 	spin_unlock(&spool->lock);
77 
78 	/* If no pages are used, and no other handles to the subpool
79 	 * remain, give up any reservations mased on minimum size and
80 	 * free the subpool */
81 	if (free) {
82 		if (spool->min_hpages != -1)
83 			hugetlb_acct_memory(spool->hstate,
84 						-spool->min_hpages);
85 		kfree(spool);
86 	}
87 }
88 
89 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
90 						long min_hpages)
91 {
92 	struct hugepage_subpool *spool;
93 
94 	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
95 	if (!spool)
96 		return NULL;
97 
98 	spin_lock_init(&spool->lock);
99 	spool->count = 1;
100 	spool->max_hpages = max_hpages;
101 	spool->hstate = h;
102 	spool->min_hpages = min_hpages;
103 
104 	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
105 		kfree(spool);
106 		return NULL;
107 	}
108 	spool->rsv_hpages = min_hpages;
109 
110 	return spool;
111 }
112 
113 void hugepage_put_subpool(struct hugepage_subpool *spool)
114 {
115 	spin_lock(&spool->lock);
116 	BUG_ON(!spool->count);
117 	spool->count--;
118 	unlock_or_release_subpool(spool);
119 }
120 
121 /*
122  * Subpool accounting for allocating and reserving pages.
123  * Return -ENOMEM if there are not enough resources to satisfy the
124  * the request.  Otherwise, return the number of pages by which the
125  * global pools must be adjusted (upward).  The returned value may
126  * only be different than the passed value (delta) in the case where
127  * a subpool minimum size must be manitained.
128  */
129 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
130 				      long delta)
131 {
132 	long ret = delta;
133 
134 	if (!spool)
135 		return ret;
136 
137 	spin_lock(&spool->lock);
138 
139 	if (spool->max_hpages != -1) {		/* maximum size accounting */
140 		if ((spool->used_hpages + delta) <= spool->max_hpages)
141 			spool->used_hpages += delta;
142 		else {
143 			ret = -ENOMEM;
144 			goto unlock_ret;
145 		}
146 	}
147 
148 	/* minimum size accounting */
149 	if (spool->min_hpages != -1 && spool->rsv_hpages) {
150 		if (delta > spool->rsv_hpages) {
151 			/*
152 			 * Asking for more reserves than those already taken on
153 			 * behalf of subpool.  Return difference.
154 			 */
155 			ret = delta - spool->rsv_hpages;
156 			spool->rsv_hpages = 0;
157 		} else {
158 			ret = 0;	/* reserves already accounted for */
159 			spool->rsv_hpages -= delta;
160 		}
161 	}
162 
163 unlock_ret:
164 	spin_unlock(&spool->lock);
165 	return ret;
166 }
167 
168 /*
169  * Subpool accounting for freeing and unreserving pages.
170  * Return the number of global page reservations that must be dropped.
171  * The return value may only be different than the passed value (delta)
172  * in the case where a subpool minimum size must be maintained.
173  */
174 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
175 				       long delta)
176 {
177 	long ret = delta;
178 
179 	if (!spool)
180 		return delta;
181 
182 	spin_lock(&spool->lock);
183 
184 	if (spool->max_hpages != -1)		/* maximum size accounting */
185 		spool->used_hpages -= delta;
186 
187 	 /* minimum size accounting */
188 	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
189 		if (spool->rsv_hpages + delta <= spool->min_hpages)
190 			ret = 0;
191 		else
192 			ret = spool->rsv_hpages + delta - spool->min_hpages;
193 
194 		spool->rsv_hpages += delta;
195 		if (spool->rsv_hpages > spool->min_hpages)
196 			spool->rsv_hpages = spool->min_hpages;
197 	}
198 
199 	/*
200 	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
201 	 * quota reference, free it now.
202 	 */
203 	unlock_or_release_subpool(spool);
204 
205 	return ret;
206 }
207 
208 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
209 {
210 	return HUGETLBFS_SB(inode->i_sb)->spool;
211 }
212 
213 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
214 {
215 	return subpool_inode(file_inode(vma->vm_file));
216 }
217 
218 /*
219  * Region tracking -- allows tracking of reservations and instantiated pages
220  *                    across the pages in a mapping.
221  *
222  * The region data structures are embedded into a resv_map and protected
223  * by a resv_map's lock.  The set of regions within the resv_map represent
224  * reservations for huge pages, or huge pages that have already been
225  * instantiated within the map.  The from and to elements are huge page
226  * indicies into the associated mapping.  from indicates the starting index
227  * of the region.  to represents the first index past the end of  the region.
228  *
229  * For example, a file region structure with from == 0 and to == 4 represents
230  * four huge pages in a mapping.  It is important to note that the to element
231  * represents the first element past the end of the region. This is used in
232  * arithmetic as 4(to) - 0(from) = 4 huge pages in the region.
233  *
234  * Interval notation of the form [from, to) will be used to indicate that
235  * the endpoint from is inclusive and to is exclusive.
236  */
237 struct file_region {
238 	struct list_head link;
239 	long from;
240 	long to;
241 };
242 
243 /*
244  * Add the huge page range represented by [f, t) to the reserve
245  * map.  In the normal case, existing regions will be expanded
246  * to accommodate the specified range.  Sufficient regions should
247  * exist for expansion due to the previous call to region_chg
248  * with the same range.  However, it is possible that region_del
249  * could have been called after region_chg and modifed the map
250  * in such a way that no region exists to be expanded.  In this
251  * case, pull a region descriptor from the cache associated with
252  * the map and use that for the new range.
253  *
254  * Return the number of new huge pages added to the map.  This
255  * number is greater than or equal to zero.
256  */
257 static long region_add(struct resv_map *resv, long f, long t)
258 {
259 	struct list_head *head = &resv->regions;
260 	struct file_region *rg, *nrg, *trg;
261 	long add = 0;
262 
263 	spin_lock(&resv->lock);
264 	/* Locate the region we are either in or before. */
265 	list_for_each_entry(rg, head, link)
266 		if (f <= rg->to)
267 			break;
268 
269 	/*
270 	 * If no region exists which can be expanded to include the
271 	 * specified range, the list must have been modified by an
272 	 * interleving call to region_del().  Pull a region descriptor
273 	 * from the cache and use it for this range.
274 	 */
275 	if (&rg->link == head || t < rg->from) {
276 		VM_BUG_ON(resv->region_cache_count <= 0);
277 
278 		resv->region_cache_count--;
279 		nrg = list_first_entry(&resv->region_cache, struct file_region,
280 					link);
281 		list_del(&nrg->link);
282 
283 		nrg->from = f;
284 		nrg->to = t;
285 		list_add(&nrg->link, rg->link.prev);
286 
287 		add += t - f;
288 		goto out_locked;
289 	}
290 
291 	/* Round our left edge to the current segment if it encloses us. */
292 	if (f > rg->from)
293 		f = rg->from;
294 
295 	/* Check for and consume any regions we now overlap with. */
296 	nrg = rg;
297 	list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
298 		if (&rg->link == head)
299 			break;
300 		if (rg->from > t)
301 			break;
302 
303 		/* If this area reaches higher then extend our area to
304 		 * include it completely.  If this is not the first area
305 		 * which we intend to reuse, free it. */
306 		if (rg->to > t)
307 			t = rg->to;
308 		if (rg != nrg) {
309 			/* Decrement return value by the deleted range.
310 			 * Another range will span this area so that by
311 			 * end of routine add will be >= zero
312 			 */
313 			add -= (rg->to - rg->from);
314 			list_del(&rg->link);
315 			kfree(rg);
316 		}
317 	}
318 
319 	add += (nrg->from - f);		/* Added to beginning of region */
320 	nrg->from = f;
321 	add += t - nrg->to;		/* Added to end of region */
322 	nrg->to = t;
323 
324 out_locked:
325 	resv->adds_in_progress--;
326 	spin_unlock(&resv->lock);
327 	VM_BUG_ON(add < 0);
328 	return add;
329 }
330 
331 /*
332  * Examine the existing reserve map and determine how many
333  * huge pages in the specified range [f, t) are NOT currently
334  * represented.  This routine is called before a subsequent
335  * call to region_add that will actually modify the reserve
336  * map to add the specified range [f, t).  region_chg does
337  * not change the number of huge pages represented by the
338  * map.  However, if the existing regions in the map can not
339  * be expanded to represent the new range, a new file_region
340  * structure is added to the map as a placeholder.  This is
341  * so that the subsequent region_add call will have all the
342  * regions it needs and will not fail.
343  *
344  * Upon entry, region_chg will also examine the cache of region descriptors
345  * associated with the map.  If there are not enough descriptors cached, one
346  * will be allocated for the in progress add operation.
347  *
348  * Returns the number of huge pages that need to be added to the existing
349  * reservation map for the range [f, t).  This number is greater or equal to
350  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
351  * is needed and can not be allocated.
352  */
353 static long region_chg(struct resv_map *resv, long f, long t)
354 {
355 	struct list_head *head = &resv->regions;
356 	struct file_region *rg, *nrg = NULL;
357 	long chg = 0;
358 
359 retry:
360 	spin_lock(&resv->lock);
361 retry_locked:
362 	resv->adds_in_progress++;
363 
364 	/*
365 	 * Check for sufficient descriptors in the cache to accommodate
366 	 * the number of in progress add operations.
367 	 */
368 	if (resv->adds_in_progress > resv->region_cache_count) {
369 		struct file_region *trg;
370 
371 		VM_BUG_ON(resv->adds_in_progress - resv->region_cache_count > 1);
372 		/* Must drop lock to allocate a new descriptor. */
373 		resv->adds_in_progress--;
374 		spin_unlock(&resv->lock);
375 
376 		trg = kmalloc(sizeof(*trg), GFP_KERNEL);
377 		if (!trg) {
378 			kfree(nrg);
379 			return -ENOMEM;
380 		}
381 
382 		spin_lock(&resv->lock);
383 		list_add(&trg->link, &resv->region_cache);
384 		resv->region_cache_count++;
385 		goto retry_locked;
386 	}
387 
388 	/* Locate the region we are before or in. */
389 	list_for_each_entry(rg, head, link)
390 		if (f <= rg->to)
391 			break;
392 
393 	/* If we are below the current region then a new region is required.
394 	 * Subtle, allocate a new region at the position but make it zero
395 	 * size such that we can guarantee to record the reservation. */
396 	if (&rg->link == head || t < rg->from) {
397 		if (!nrg) {
398 			resv->adds_in_progress--;
399 			spin_unlock(&resv->lock);
400 			nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
401 			if (!nrg)
402 				return -ENOMEM;
403 
404 			nrg->from = f;
405 			nrg->to   = f;
406 			INIT_LIST_HEAD(&nrg->link);
407 			goto retry;
408 		}
409 
410 		list_add(&nrg->link, rg->link.prev);
411 		chg = t - f;
412 		goto out_nrg;
413 	}
414 
415 	/* Round our left edge to the current segment if it encloses us. */
416 	if (f > rg->from)
417 		f = rg->from;
418 	chg = t - f;
419 
420 	/* Check for and consume any regions we now overlap with. */
421 	list_for_each_entry(rg, rg->link.prev, link) {
422 		if (&rg->link == head)
423 			break;
424 		if (rg->from > t)
425 			goto out;
426 
427 		/* We overlap with this area, if it extends further than
428 		 * us then we must extend ourselves.  Account for its
429 		 * existing reservation. */
430 		if (rg->to > t) {
431 			chg += rg->to - t;
432 			t = rg->to;
433 		}
434 		chg -= rg->to - rg->from;
435 	}
436 
437 out:
438 	spin_unlock(&resv->lock);
439 	/*  We already know we raced and no longer need the new region */
440 	kfree(nrg);
441 	return chg;
442 out_nrg:
443 	spin_unlock(&resv->lock);
444 	return chg;
445 }
446 
447 /*
448  * Abort the in progress add operation.  The adds_in_progress field
449  * of the resv_map keeps track of the operations in progress between
450  * calls to region_chg and region_add.  Operations are sometimes
451  * aborted after the call to region_chg.  In such cases, region_abort
452  * is called to decrement the adds_in_progress counter.
453  *
454  * NOTE: The range arguments [f, t) are not needed or used in this
455  * routine.  They are kept to make reading the calling code easier as
456  * arguments will match the associated region_chg call.
457  */
458 static void region_abort(struct resv_map *resv, long f, long t)
459 {
460 	spin_lock(&resv->lock);
461 	VM_BUG_ON(!resv->region_cache_count);
462 	resv->adds_in_progress--;
463 	spin_unlock(&resv->lock);
464 }
465 
466 /*
467  * Delete the specified range [f, t) from the reserve map.  If the
468  * t parameter is LONG_MAX, this indicates that ALL regions after f
469  * should be deleted.  Locate the regions which intersect [f, t)
470  * and either trim, delete or split the existing regions.
471  *
472  * Returns the number of huge pages deleted from the reserve map.
473  * In the normal case, the return value is zero or more.  In the
474  * case where a region must be split, a new region descriptor must
475  * be allocated.  If the allocation fails, -ENOMEM will be returned.
476  * NOTE: If the parameter t == LONG_MAX, then we will never split
477  * a region and possibly return -ENOMEM.  Callers specifying
478  * t == LONG_MAX do not need to check for -ENOMEM error.
479  */
480 static long region_del(struct resv_map *resv, long f, long t)
481 {
482 	struct list_head *head = &resv->regions;
483 	struct file_region *rg, *trg;
484 	struct file_region *nrg = NULL;
485 	long del = 0;
486 
487 retry:
488 	spin_lock(&resv->lock);
489 	list_for_each_entry_safe(rg, trg, head, link) {
490 		/*
491 		 * Skip regions before the range to be deleted.  file_region
492 		 * ranges are normally of the form [from, to).  However, there
493 		 * may be a "placeholder" entry in the map which is of the form
494 		 * (from, to) with from == to.  Check for placeholder entries
495 		 * at the beginning of the range to be deleted.
496 		 */
497 		if (rg->to <= f && (rg->to != rg->from || rg->to != f))
498 			continue;
499 
500 		if (rg->from >= t)
501 			break;
502 
503 		if (f > rg->from && t < rg->to) { /* Must split region */
504 			/*
505 			 * Check for an entry in the cache before dropping
506 			 * lock and attempting allocation.
507 			 */
508 			if (!nrg &&
509 			    resv->region_cache_count > resv->adds_in_progress) {
510 				nrg = list_first_entry(&resv->region_cache,
511 							struct file_region,
512 							link);
513 				list_del(&nrg->link);
514 				resv->region_cache_count--;
515 			}
516 
517 			if (!nrg) {
518 				spin_unlock(&resv->lock);
519 				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
520 				if (!nrg)
521 					return -ENOMEM;
522 				goto retry;
523 			}
524 
525 			del += t - f;
526 
527 			/* New entry for end of split region */
528 			nrg->from = t;
529 			nrg->to = rg->to;
530 			INIT_LIST_HEAD(&nrg->link);
531 
532 			/* Original entry is trimmed */
533 			rg->to = f;
534 
535 			list_add(&nrg->link, &rg->link);
536 			nrg = NULL;
537 			break;
538 		}
539 
540 		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
541 			del += rg->to - rg->from;
542 			list_del(&rg->link);
543 			kfree(rg);
544 			continue;
545 		}
546 
547 		if (f <= rg->from) {	/* Trim beginning of region */
548 			del += t - rg->from;
549 			rg->from = t;
550 		} else {		/* Trim end of region */
551 			del += rg->to - f;
552 			rg->to = f;
553 		}
554 	}
555 
556 	spin_unlock(&resv->lock);
557 	kfree(nrg);
558 	return del;
559 }
560 
561 /*
562  * A rare out of memory error was encountered which prevented removal of
563  * the reserve map region for a page.  The huge page itself was free'ed
564  * and removed from the page cache.  This routine will adjust the subpool
565  * usage count, and the global reserve count if needed.  By incrementing
566  * these counts, the reserve map entry which could not be deleted will
567  * appear as a "reserved" entry instead of simply dangling with incorrect
568  * counts.
569  */
570 void hugetlb_fix_reserve_counts(struct inode *inode, bool restore_reserve)
571 {
572 	struct hugepage_subpool *spool = subpool_inode(inode);
573 	long rsv_adjust;
574 
575 	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
576 	if (restore_reserve && rsv_adjust) {
577 		struct hstate *h = hstate_inode(inode);
578 
579 		hugetlb_acct_memory(h, 1);
580 	}
581 }
582 
583 /*
584  * Count and return the number of huge pages in the reserve map
585  * that intersect with the range [f, t).
586  */
587 static long region_count(struct resv_map *resv, long f, long t)
588 {
589 	struct list_head *head = &resv->regions;
590 	struct file_region *rg;
591 	long chg = 0;
592 
593 	spin_lock(&resv->lock);
594 	/* Locate each segment we overlap with, and count that overlap. */
595 	list_for_each_entry(rg, head, link) {
596 		long seg_from;
597 		long seg_to;
598 
599 		if (rg->to <= f)
600 			continue;
601 		if (rg->from >= t)
602 			break;
603 
604 		seg_from = max(rg->from, f);
605 		seg_to = min(rg->to, t);
606 
607 		chg += seg_to - seg_from;
608 	}
609 	spin_unlock(&resv->lock);
610 
611 	return chg;
612 }
613 
614 /*
615  * Convert the address within this vma to the page offset within
616  * the mapping, in pagecache page units; huge pages here.
617  */
618 static pgoff_t vma_hugecache_offset(struct hstate *h,
619 			struct vm_area_struct *vma, unsigned long address)
620 {
621 	return ((address - vma->vm_start) >> huge_page_shift(h)) +
622 			(vma->vm_pgoff >> huge_page_order(h));
623 }
624 
625 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
626 				     unsigned long address)
627 {
628 	return vma_hugecache_offset(hstate_vma(vma), vma, address);
629 }
630 EXPORT_SYMBOL_GPL(linear_hugepage_index);
631 
632 /*
633  * Return the size of the pages allocated when backing a VMA. In the majority
634  * cases this will be same size as used by the page table entries.
635  */
636 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
637 {
638 	struct hstate *hstate;
639 
640 	if (!is_vm_hugetlb_page(vma))
641 		return PAGE_SIZE;
642 
643 	hstate = hstate_vma(vma);
644 
645 	return 1UL << huge_page_shift(hstate);
646 }
647 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
648 
649 /*
650  * Return the page size being used by the MMU to back a VMA. In the majority
651  * of cases, the page size used by the kernel matches the MMU size. On
652  * architectures where it differs, an architecture-specific version of this
653  * function is required.
654  */
655 #ifndef vma_mmu_pagesize
656 unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
657 {
658 	return vma_kernel_pagesize(vma);
659 }
660 #endif
661 
662 /*
663  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
664  * bits of the reservation map pointer, which are always clear due to
665  * alignment.
666  */
667 #define HPAGE_RESV_OWNER    (1UL << 0)
668 #define HPAGE_RESV_UNMAPPED (1UL << 1)
669 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
670 
671 /*
672  * These helpers are used to track how many pages are reserved for
673  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
674  * is guaranteed to have their future faults succeed.
675  *
676  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
677  * the reserve counters are updated with the hugetlb_lock held. It is safe
678  * to reset the VMA at fork() time as it is not in use yet and there is no
679  * chance of the global counters getting corrupted as a result of the values.
680  *
681  * The private mapping reservation is represented in a subtly different
682  * manner to a shared mapping.  A shared mapping has a region map associated
683  * with the underlying file, this region map represents the backing file
684  * pages which have ever had a reservation assigned which this persists even
685  * after the page is instantiated.  A private mapping has a region map
686  * associated with the original mmap which is attached to all VMAs which
687  * reference it, this region map represents those offsets which have consumed
688  * reservation ie. where pages have been instantiated.
689  */
690 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
691 {
692 	return (unsigned long)vma->vm_private_data;
693 }
694 
695 static void set_vma_private_data(struct vm_area_struct *vma,
696 							unsigned long value)
697 {
698 	vma->vm_private_data = (void *)value;
699 }
700 
701 struct resv_map *resv_map_alloc(void)
702 {
703 	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
704 	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
705 
706 	if (!resv_map || !rg) {
707 		kfree(resv_map);
708 		kfree(rg);
709 		return NULL;
710 	}
711 
712 	kref_init(&resv_map->refs);
713 	spin_lock_init(&resv_map->lock);
714 	INIT_LIST_HEAD(&resv_map->regions);
715 
716 	resv_map->adds_in_progress = 0;
717 
718 	INIT_LIST_HEAD(&resv_map->region_cache);
719 	list_add(&rg->link, &resv_map->region_cache);
720 	resv_map->region_cache_count = 1;
721 
722 	return resv_map;
723 }
724 
725 void resv_map_release(struct kref *ref)
726 {
727 	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
728 	struct list_head *head = &resv_map->region_cache;
729 	struct file_region *rg, *trg;
730 
731 	/* Clear out any active regions before we release the map. */
732 	region_del(resv_map, 0, LONG_MAX);
733 
734 	/* ... and any entries left in the cache */
735 	list_for_each_entry_safe(rg, trg, head, link) {
736 		list_del(&rg->link);
737 		kfree(rg);
738 	}
739 
740 	VM_BUG_ON(resv_map->adds_in_progress);
741 
742 	kfree(resv_map);
743 }
744 
745 static inline struct resv_map *inode_resv_map(struct inode *inode)
746 {
747 	return inode->i_mapping->private_data;
748 }
749 
750 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
751 {
752 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
753 	if (vma->vm_flags & VM_MAYSHARE) {
754 		struct address_space *mapping = vma->vm_file->f_mapping;
755 		struct inode *inode = mapping->host;
756 
757 		return inode_resv_map(inode);
758 
759 	} else {
760 		return (struct resv_map *)(get_vma_private_data(vma) &
761 							~HPAGE_RESV_MASK);
762 	}
763 }
764 
765 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
766 {
767 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
768 	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
769 
770 	set_vma_private_data(vma, (get_vma_private_data(vma) &
771 				HPAGE_RESV_MASK) | (unsigned long)map);
772 }
773 
774 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
775 {
776 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
777 	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
778 
779 	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
780 }
781 
782 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
783 {
784 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
785 
786 	return (get_vma_private_data(vma) & flag) != 0;
787 }
788 
789 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
790 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
791 {
792 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
793 	if (!(vma->vm_flags & VM_MAYSHARE))
794 		vma->vm_private_data = (void *)0;
795 }
796 
797 /* Returns true if the VMA has associated reserve pages */
798 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
799 {
800 	if (vma->vm_flags & VM_NORESERVE) {
801 		/*
802 		 * This address is already reserved by other process(chg == 0),
803 		 * so, we should decrement reserved count. Without decrementing,
804 		 * reserve count remains after releasing inode, because this
805 		 * allocated page will go into page cache and is regarded as
806 		 * coming from reserved pool in releasing step.  Currently, we
807 		 * don't have any other solution to deal with this situation
808 		 * properly, so add work-around here.
809 		 */
810 		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
811 			return true;
812 		else
813 			return false;
814 	}
815 
816 	/* Shared mappings always use reserves */
817 	if (vma->vm_flags & VM_MAYSHARE) {
818 		/*
819 		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
820 		 * be a region map for all pages.  The only situation where
821 		 * there is no region map is if a hole was punched via
822 		 * fallocate.  In this case, there really are no reverves to
823 		 * use.  This situation is indicated if chg != 0.
824 		 */
825 		if (chg)
826 			return false;
827 		else
828 			return true;
829 	}
830 
831 	/*
832 	 * Only the process that called mmap() has reserves for
833 	 * private mappings.
834 	 */
835 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
836 		/*
837 		 * Like the shared case above, a hole punch or truncate
838 		 * could have been performed on the private mapping.
839 		 * Examine the value of chg to determine if reserves
840 		 * actually exist or were previously consumed.
841 		 * Very Subtle - The value of chg comes from a previous
842 		 * call to vma_needs_reserves().  The reserve map for
843 		 * private mappings has different (opposite) semantics
844 		 * than that of shared mappings.  vma_needs_reserves()
845 		 * has already taken this difference in semantics into
846 		 * account.  Therefore, the meaning of chg is the same
847 		 * as in the shared case above.  Code could easily be
848 		 * combined, but keeping it separate draws attention to
849 		 * subtle differences.
850 		 */
851 		if (chg)
852 			return false;
853 		else
854 			return true;
855 	}
856 
857 	return false;
858 }
859 
860 static void enqueue_huge_page(struct hstate *h, struct page *page)
861 {
862 	int nid = page_to_nid(page);
863 	list_move(&page->lru, &h->hugepage_freelists[nid]);
864 	h->free_huge_pages++;
865 	h->free_huge_pages_node[nid]++;
866 }
867 
868 static struct page *dequeue_huge_page_node(struct hstate *h, int nid)
869 {
870 	struct page *page;
871 
872 	list_for_each_entry(page, &h->hugepage_freelists[nid], lru)
873 		if (!is_migrate_isolate_page(page))
874 			break;
875 	/*
876 	 * if 'non-isolated free hugepage' not found on the list,
877 	 * the allocation fails.
878 	 */
879 	if (&h->hugepage_freelists[nid] == &page->lru)
880 		return NULL;
881 	list_move(&page->lru, &h->hugepage_activelist);
882 	set_page_refcounted(page);
883 	h->free_huge_pages--;
884 	h->free_huge_pages_node[nid]--;
885 	return page;
886 }
887 
888 /* Movability of hugepages depends on migration support. */
889 static inline gfp_t htlb_alloc_mask(struct hstate *h)
890 {
891 	if (hugepages_treat_as_movable || hugepage_migration_supported(h))
892 		return GFP_HIGHUSER_MOVABLE;
893 	else
894 		return GFP_HIGHUSER;
895 }
896 
897 static struct page *dequeue_huge_page_vma(struct hstate *h,
898 				struct vm_area_struct *vma,
899 				unsigned long address, int avoid_reserve,
900 				long chg)
901 {
902 	struct page *page = NULL;
903 	struct mempolicy *mpol;
904 	nodemask_t *nodemask;
905 	struct zonelist *zonelist;
906 	struct zone *zone;
907 	struct zoneref *z;
908 	unsigned int cpuset_mems_cookie;
909 
910 	/*
911 	 * A child process with MAP_PRIVATE mappings created by their parent
912 	 * have no page reserves. This check ensures that reservations are
913 	 * not "stolen". The child may still get SIGKILLed
914 	 */
915 	if (!vma_has_reserves(vma, chg) &&
916 			h->free_huge_pages - h->resv_huge_pages == 0)
917 		goto err;
918 
919 	/* If reserves cannot be used, ensure enough pages are in the pool */
920 	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
921 		goto err;
922 
923 retry_cpuset:
924 	cpuset_mems_cookie = read_mems_allowed_begin();
925 	zonelist = huge_zonelist(vma, address,
926 					htlb_alloc_mask(h), &mpol, &nodemask);
927 
928 	for_each_zone_zonelist_nodemask(zone, z, zonelist,
929 						MAX_NR_ZONES - 1, nodemask) {
930 		if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
931 			page = dequeue_huge_page_node(h, zone_to_nid(zone));
932 			if (page) {
933 				if (avoid_reserve)
934 					break;
935 				if (!vma_has_reserves(vma, chg))
936 					break;
937 
938 				SetPagePrivate(page);
939 				h->resv_huge_pages--;
940 				break;
941 			}
942 		}
943 	}
944 
945 	mpol_cond_put(mpol);
946 	if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
947 		goto retry_cpuset;
948 	return page;
949 
950 err:
951 	return NULL;
952 }
953 
954 /*
955  * common helper functions for hstate_next_node_to_{alloc|free}.
956  * We may have allocated or freed a huge page based on a different
957  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
958  * be outside of *nodes_allowed.  Ensure that we use an allowed
959  * node for alloc or free.
960  */
961 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
962 {
963 	nid = next_node_in(nid, *nodes_allowed);
964 	VM_BUG_ON(nid >= MAX_NUMNODES);
965 
966 	return nid;
967 }
968 
969 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
970 {
971 	if (!node_isset(nid, *nodes_allowed))
972 		nid = next_node_allowed(nid, nodes_allowed);
973 	return nid;
974 }
975 
976 /*
977  * returns the previously saved node ["this node"] from which to
978  * allocate a persistent huge page for the pool and advance the
979  * next node from which to allocate, handling wrap at end of node
980  * mask.
981  */
982 static int hstate_next_node_to_alloc(struct hstate *h,
983 					nodemask_t *nodes_allowed)
984 {
985 	int nid;
986 
987 	VM_BUG_ON(!nodes_allowed);
988 
989 	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
990 	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
991 
992 	return nid;
993 }
994 
995 /*
996  * helper for free_pool_huge_page() - return the previously saved
997  * node ["this node"] from which to free a huge page.  Advance the
998  * next node id whether or not we find a free huge page to free so
999  * that the next attempt to free addresses the next node.
1000  */
1001 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1002 {
1003 	int nid;
1004 
1005 	VM_BUG_ON(!nodes_allowed);
1006 
1007 	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1008 	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1009 
1010 	return nid;
1011 }
1012 
1013 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
1014 	for (nr_nodes = nodes_weight(*mask);				\
1015 		nr_nodes > 0 &&						\
1016 		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
1017 		nr_nodes--)
1018 
1019 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
1020 	for (nr_nodes = nodes_weight(*mask);				\
1021 		nr_nodes > 0 &&						\
1022 		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
1023 		nr_nodes--)
1024 
1025 #if (defined(CONFIG_X86_64) || defined(CONFIG_S390)) && \
1026 	((defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || \
1027 	defined(CONFIG_CMA))
1028 static void destroy_compound_gigantic_page(struct page *page,
1029 					unsigned int order)
1030 {
1031 	int i;
1032 	int nr_pages = 1 << order;
1033 	struct page *p = page + 1;
1034 
1035 	atomic_set(compound_mapcount_ptr(page), 0);
1036 	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1037 		clear_compound_head(p);
1038 		set_page_refcounted(p);
1039 	}
1040 
1041 	set_compound_order(page, 0);
1042 	__ClearPageHead(page);
1043 }
1044 
1045 static void free_gigantic_page(struct page *page, unsigned int order)
1046 {
1047 	free_contig_range(page_to_pfn(page), 1 << order);
1048 }
1049 
1050 static int __alloc_gigantic_page(unsigned long start_pfn,
1051 				unsigned long nr_pages)
1052 {
1053 	unsigned long end_pfn = start_pfn + nr_pages;
1054 	return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE);
1055 }
1056 
1057 static bool pfn_range_valid_gigantic(struct zone *z,
1058 			unsigned long start_pfn, unsigned long nr_pages)
1059 {
1060 	unsigned long i, end_pfn = start_pfn + nr_pages;
1061 	struct page *page;
1062 
1063 	for (i = start_pfn; i < end_pfn; i++) {
1064 		if (!pfn_valid(i))
1065 			return false;
1066 
1067 		page = pfn_to_page(i);
1068 
1069 		if (page_zone(page) != z)
1070 			return false;
1071 
1072 		if (PageReserved(page))
1073 			return false;
1074 
1075 		if (page_count(page) > 0)
1076 			return false;
1077 
1078 		if (PageHuge(page))
1079 			return false;
1080 	}
1081 
1082 	return true;
1083 }
1084 
1085 static bool zone_spans_last_pfn(const struct zone *zone,
1086 			unsigned long start_pfn, unsigned long nr_pages)
1087 {
1088 	unsigned long last_pfn = start_pfn + nr_pages - 1;
1089 	return zone_spans_pfn(zone, last_pfn);
1090 }
1091 
1092 static struct page *alloc_gigantic_page(int nid, unsigned int order)
1093 {
1094 	unsigned long nr_pages = 1 << order;
1095 	unsigned long ret, pfn, flags;
1096 	struct zone *z;
1097 
1098 	z = NODE_DATA(nid)->node_zones;
1099 	for (; z - NODE_DATA(nid)->node_zones < MAX_NR_ZONES; z++) {
1100 		spin_lock_irqsave(&z->lock, flags);
1101 
1102 		pfn = ALIGN(z->zone_start_pfn, nr_pages);
1103 		while (zone_spans_last_pfn(z, pfn, nr_pages)) {
1104 			if (pfn_range_valid_gigantic(z, pfn, nr_pages)) {
1105 				/*
1106 				 * We release the zone lock here because
1107 				 * alloc_contig_range() will also lock the zone
1108 				 * at some point. If there's an allocation
1109 				 * spinning on this lock, it may win the race
1110 				 * and cause alloc_contig_range() to fail...
1111 				 */
1112 				spin_unlock_irqrestore(&z->lock, flags);
1113 				ret = __alloc_gigantic_page(pfn, nr_pages);
1114 				if (!ret)
1115 					return pfn_to_page(pfn);
1116 				spin_lock_irqsave(&z->lock, flags);
1117 			}
1118 			pfn += nr_pages;
1119 		}
1120 
1121 		spin_unlock_irqrestore(&z->lock, flags);
1122 	}
1123 
1124 	return NULL;
1125 }
1126 
1127 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid);
1128 static void prep_compound_gigantic_page(struct page *page, unsigned int order);
1129 
1130 static struct page *alloc_fresh_gigantic_page_node(struct hstate *h, int nid)
1131 {
1132 	struct page *page;
1133 
1134 	page = alloc_gigantic_page(nid, huge_page_order(h));
1135 	if (page) {
1136 		prep_compound_gigantic_page(page, huge_page_order(h));
1137 		prep_new_huge_page(h, page, nid);
1138 	}
1139 
1140 	return page;
1141 }
1142 
1143 static int alloc_fresh_gigantic_page(struct hstate *h,
1144 				nodemask_t *nodes_allowed)
1145 {
1146 	struct page *page = NULL;
1147 	int nr_nodes, node;
1148 
1149 	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1150 		page = alloc_fresh_gigantic_page_node(h, node);
1151 		if (page)
1152 			return 1;
1153 	}
1154 
1155 	return 0;
1156 }
1157 
1158 static inline bool gigantic_page_supported(void) { return true; }
1159 #else
1160 static inline bool gigantic_page_supported(void) { return false; }
1161 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1162 static inline void destroy_compound_gigantic_page(struct page *page,
1163 						unsigned int order) { }
1164 static inline int alloc_fresh_gigantic_page(struct hstate *h,
1165 					nodemask_t *nodes_allowed) { return 0; }
1166 #endif
1167 
1168 static void update_and_free_page(struct hstate *h, struct page *page)
1169 {
1170 	int i;
1171 
1172 	if (hstate_is_gigantic(h) && !gigantic_page_supported())
1173 		return;
1174 
1175 	h->nr_huge_pages--;
1176 	h->nr_huge_pages_node[page_to_nid(page)]--;
1177 	for (i = 0; i < pages_per_huge_page(h); i++) {
1178 		page[i].flags &= ~(1 << PG_locked | 1 << PG_error |
1179 				1 << PG_referenced | 1 << PG_dirty |
1180 				1 << PG_active | 1 << PG_private |
1181 				1 << PG_writeback);
1182 	}
1183 	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1184 	set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1185 	set_page_refcounted(page);
1186 	if (hstate_is_gigantic(h)) {
1187 		destroy_compound_gigantic_page(page, huge_page_order(h));
1188 		free_gigantic_page(page, huge_page_order(h));
1189 	} else {
1190 		__free_pages(page, huge_page_order(h));
1191 	}
1192 }
1193 
1194 struct hstate *size_to_hstate(unsigned long size)
1195 {
1196 	struct hstate *h;
1197 
1198 	for_each_hstate(h) {
1199 		if (huge_page_size(h) == size)
1200 			return h;
1201 	}
1202 	return NULL;
1203 }
1204 
1205 /*
1206  * Test to determine whether the hugepage is "active/in-use" (i.e. being linked
1207  * to hstate->hugepage_activelist.)
1208  *
1209  * This function can be called for tail pages, but never returns true for them.
1210  */
1211 bool page_huge_active(struct page *page)
1212 {
1213 	VM_BUG_ON_PAGE(!PageHuge(page), page);
1214 	return PageHead(page) && PagePrivate(&page[1]);
1215 }
1216 
1217 /* never called for tail page */
1218 static void set_page_huge_active(struct page *page)
1219 {
1220 	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1221 	SetPagePrivate(&page[1]);
1222 }
1223 
1224 static void clear_page_huge_active(struct page *page)
1225 {
1226 	VM_BUG_ON_PAGE(!PageHeadHuge(page), page);
1227 	ClearPagePrivate(&page[1]);
1228 }
1229 
1230 void free_huge_page(struct page *page)
1231 {
1232 	/*
1233 	 * Can't pass hstate in here because it is called from the
1234 	 * compound page destructor.
1235 	 */
1236 	struct hstate *h = page_hstate(page);
1237 	int nid = page_to_nid(page);
1238 	struct hugepage_subpool *spool =
1239 		(struct hugepage_subpool *)page_private(page);
1240 	bool restore_reserve;
1241 
1242 	set_page_private(page, 0);
1243 	page->mapping = NULL;
1244 	VM_BUG_ON_PAGE(page_count(page), page);
1245 	VM_BUG_ON_PAGE(page_mapcount(page), page);
1246 	restore_reserve = PagePrivate(page);
1247 	ClearPagePrivate(page);
1248 
1249 	/*
1250 	 * A return code of zero implies that the subpool will be under its
1251 	 * minimum size if the reservation is not restored after page is free.
1252 	 * Therefore, force restore_reserve operation.
1253 	 */
1254 	if (hugepage_subpool_put_pages(spool, 1) == 0)
1255 		restore_reserve = true;
1256 
1257 	spin_lock(&hugetlb_lock);
1258 	clear_page_huge_active(page);
1259 	hugetlb_cgroup_uncharge_page(hstate_index(h),
1260 				     pages_per_huge_page(h), page);
1261 	if (restore_reserve)
1262 		h->resv_huge_pages++;
1263 
1264 	if (h->surplus_huge_pages_node[nid]) {
1265 		/* remove the page from active list */
1266 		list_del(&page->lru);
1267 		update_and_free_page(h, page);
1268 		h->surplus_huge_pages--;
1269 		h->surplus_huge_pages_node[nid]--;
1270 	} else {
1271 		arch_clear_hugepage_flags(page);
1272 		enqueue_huge_page(h, page);
1273 	}
1274 	spin_unlock(&hugetlb_lock);
1275 }
1276 
1277 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1278 {
1279 	INIT_LIST_HEAD(&page->lru);
1280 	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1281 	spin_lock(&hugetlb_lock);
1282 	set_hugetlb_cgroup(page, NULL);
1283 	h->nr_huge_pages++;
1284 	h->nr_huge_pages_node[nid]++;
1285 	spin_unlock(&hugetlb_lock);
1286 	put_page(page); /* free it into the hugepage allocator */
1287 }
1288 
1289 static void prep_compound_gigantic_page(struct page *page, unsigned int order)
1290 {
1291 	int i;
1292 	int nr_pages = 1 << order;
1293 	struct page *p = page + 1;
1294 
1295 	/* we rely on prep_new_huge_page to set the destructor */
1296 	set_compound_order(page, order);
1297 	__ClearPageReserved(page);
1298 	__SetPageHead(page);
1299 	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1300 		/*
1301 		 * For gigantic hugepages allocated through bootmem at
1302 		 * boot, it's safer to be consistent with the not-gigantic
1303 		 * hugepages and clear the PG_reserved bit from all tail pages
1304 		 * too.  Otherwse drivers using get_user_pages() to access tail
1305 		 * pages may get the reference counting wrong if they see
1306 		 * PG_reserved set on a tail page (despite the head page not
1307 		 * having PG_reserved set).  Enforcing this consistency between
1308 		 * head and tail pages allows drivers to optimize away a check
1309 		 * on the head page when they need know if put_page() is needed
1310 		 * after get_user_pages().
1311 		 */
1312 		__ClearPageReserved(p);
1313 		set_page_count(p, 0);
1314 		set_compound_head(p, page);
1315 	}
1316 	atomic_set(compound_mapcount_ptr(page), -1);
1317 }
1318 
1319 /*
1320  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1321  * transparent huge pages.  See the PageTransHuge() documentation for more
1322  * details.
1323  */
1324 int PageHuge(struct page *page)
1325 {
1326 	if (!PageCompound(page))
1327 		return 0;
1328 
1329 	page = compound_head(page);
1330 	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1331 }
1332 EXPORT_SYMBOL_GPL(PageHuge);
1333 
1334 /*
1335  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1336  * normal or transparent huge pages.
1337  */
1338 int PageHeadHuge(struct page *page_head)
1339 {
1340 	if (!PageHead(page_head))
1341 		return 0;
1342 
1343 	return get_compound_page_dtor(page_head) == free_huge_page;
1344 }
1345 
1346 pgoff_t __basepage_index(struct page *page)
1347 {
1348 	struct page *page_head = compound_head(page);
1349 	pgoff_t index = page_index(page_head);
1350 	unsigned long compound_idx;
1351 
1352 	if (!PageHuge(page_head))
1353 		return page_index(page);
1354 
1355 	if (compound_order(page_head) >= MAX_ORDER)
1356 		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1357 	else
1358 		compound_idx = page - page_head;
1359 
1360 	return (index << compound_order(page_head)) + compound_idx;
1361 }
1362 
1363 static struct page *alloc_fresh_huge_page_node(struct hstate *h, int nid)
1364 {
1365 	struct page *page;
1366 
1367 	page = __alloc_pages_node(nid,
1368 		htlb_alloc_mask(h)|__GFP_COMP|__GFP_THISNODE|
1369 						__GFP_REPEAT|__GFP_NOWARN,
1370 		huge_page_order(h));
1371 	if (page) {
1372 		prep_new_huge_page(h, page, nid);
1373 	}
1374 
1375 	return page;
1376 }
1377 
1378 static int alloc_fresh_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
1379 {
1380 	struct page *page;
1381 	int nr_nodes, node;
1382 	int ret = 0;
1383 
1384 	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1385 		page = alloc_fresh_huge_page_node(h, node);
1386 		if (page) {
1387 			ret = 1;
1388 			break;
1389 		}
1390 	}
1391 
1392 	if (ret)
1393 		count_vm_event(HTLB_BUDDY_PGALLOC);
1394 	else
1395 		count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1396 
1397 	return ret;
1398 }
1399 
1400 /*
1401  * Free huge page from pool from next node to free.
1402  * Attempt to keep persistent huge pages more or less
1403  * balanced over allowed nodes.
1404  * Called with hugetlb_lock locked.
1405  */
1406 static int free_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1407 							 bool acct_surplus)
1408 {
1409 	int nr_nodes, node;
1410 	int ret = 0;
1411 
1412 	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1413 		/*
1414 		 * If we're returning unused surplus pages, only examine
1415 		 * nodes with surplus pages.
1416 		 */
1417 		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1418 		    !list_empty(&h->hugepage_freelists[node])) {
1419 			struct page *page =
1420 				list_entry(h->hugepage_freelists[node].next,
1421 					  struct page, lru);
1422 			list_del(&page->lru);
1423 			h->free_huge_pages--;
1424 			h->free_huge_pages_node[node]--;
1425 			if (acct_surplus) {
1426 				h->surplus_huge_pages--;
1427 				h->surplus_huge_pages_node[node]--;
1428 			}
1429 			update_and_free_page(h, page);
1430 			ret = 1;
1431 			break;
1432 		}
1433 	}
1434 
1435 	return ret;
1436 }
1437 
1438 /*
1439  * Dissolve a given free hugepage into free buddy pages. This function does
1440  * nothing for in-use (including surplus) hugepages.
1441  */
1442 static void dissolve_free_huge_page(struct page *page)
1443 {
1444 	spin_lock(&hugetlb_lock);
1445 	if (PageHuge(page) && !page_count(page)) {
1446 		struct hstate *h = page_hstate(page);
1447 		int nid = page_to_nid(page);
1448 		list_del(&page->lru);
1449 		h->free_huge_pages--;
1450 		h->free_huge_pages_node[nid]--;
1451 		h->max_huge_pages--;
1452 		update_and_free_page(h, page);
1453 	}
1454 	spin_unlock(&hugetlb_lock);
1455 }
1456 
1457 /*
1458  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1459  * make specified memory blocks removable from the system.
1460  * Note that start_pfn should aligned with (minimum) hugepage size.
1461  */
1462 void dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
1463 {
1464 	unsigned long pfn;
1465 
1466 	if (!hugepages_supported())
1467 		return;
1468 
1469 	VM_BUG_ON(!IS_ALIGNED(start_pfn, 1 << minimum_order));
1470 	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order)
1471 		dissolve_free_huge_page(pfn_to_page(pfn));
1472 }
1473 
1474 /*
1475  * There are 3 ways this can get called:
1476  * 1. With vma+addr: we use the VMA's memory policy
1477  * 2. With !vma, but nid=NUMA_NO_NODE:  We try to allocate a huge
1478  *    page from any node, and let the buddy allocator itself figure
1479  *    it out.
1480  * 3. With !vma, but nid!=NUMA_NO_NODE.  We allocate a huge page
1481  *    strictly from 'nid'
1482  */
1483 static struct page *__hugetlb_alloc_buddy_huge_page(struct hstate *h,
1484 		struct vm_area_struct *vma, unsigned long addr, int nid)
1485 {
1486 	int order = huge_page_order(h);
1487 	gfp_t gfp = htlb_alloc_mask(h)|__GFP_COMP|__GFP_REPEAT|__GFP_NOWARN;
1488 	unsigned int cpuset_mems_cookie;
1489 
1490 	/*
1491 	 * We need a VMA to get a memory policy.  If we do not
1492 	 * have one, we use the 'nid' argument.
1493 	 *
1494 	 * The mempolicy stuff below has some non-inlined bits
1495 	 * and calls ->vm_ops.  That makes it hard to optimize at
1496 	 * compile-time, even when NUMA is off and it does
1497 	 * nothing.  This helps the compiler optimize it out.
1498 	 */
1499 	if (!IS_ENABLED(CONFIG_NUMA) || !vma) {
1500 		/*
1501 		 * If a specific node is requested, make sure to
1502 		 * get memory from there, but only when a node
1503 		 * is explicitly specified.
1504 		 */
1505 		if (nid != NUMA_NO_NODE)
1506 			gfp |= __GFP_THISNODE;
1507 		/*
1508 		 * Make sure to call something that can handle
1509 		 * nid=NUMA_NO_NODE
1510 		 */
1511 		return alloc_pages_node(nid, gfp, order);
1512 	}
1513 
1514 	/*
1515 	 * OK, so we have a VMA.  Fetch the mempolicy and try to
1516 	 * allocate a huge page with it.  We will only reach this
1517 	 * when CONFIG_NUMA=y.
1518 	 */
1519 	do {
1520 		struct page *page;
1521 		struct mempolicy *mpol;
1522 		struct zonelist *zl;
1523 		nodemask_t *nodemask;
1524 
1525 		cpuset_mems_cookie = read_mems_allowed_begin();
1526 		zl = huge_zonelist(vma, addr, gfp, &mpol, &nodemask);
1527 		mpol_cond_put(mpol);
1528 		page = __alloc_pages_nodemask(gfp, order, zl, nodemask);
1529 		if (page)
1530 			return page;
1531 	} while (read_mems_allowed_retry(cpuset_mems_cookie));
1532 
1533 	return NULL;
1534 }
1535 
1536 /*
1537  * There are two ways to allocate a huge page:
1538  * 1. When you have a VMA and an address (like a fault)
1539  * 2. When you have no VMA (like when setting /proc/.../nr_hugepages)
1540  *
1541  * 'vma' and 'addr' are only for (1).  'nid' is always NUMA_NO_NODE in
1542  * this case which signifies that the allocation should be done with
1543  * respect for the VMA's memory policy.
1544  *
1545  * For (2), we ignore 'vma' and 'addr' and use 'nid' exclusively. This
1546  * implies that memory policies will not be taken in to account.
1547  */
1548 static struct page *__alloc_buddy_huge_page(struct hstate *h,
1549 		struct vm_area_struct *vma, unsigned long addr, int nid)
1550 {
1551 	struct page *page;
1552 	unsigned int r_nid;
1553 
1554 	if (hstate_is_gigantic(h))
1555 		return NULL;
1556 
1557 	/*
1558 	 * Make sure that anyone specifying 'nid' is not also specifying a VMA.
1559 	 * This makes sure the caller is picking _one_ of the modes with which
1560 	 * we can call this function, not both.
1561 	 */
1562 	if (vma || (addr != -1)) {
1563 		VM_WARN_ON_ONCE(addr == -1);
1564 		VM_WARN_ON_ONCE(nid != NUMA_NO_NODE);
1565 	}
1566 	/*
1567 	 * Assume we will successfully allocate the surplus page to
1568 	 * prevent racing processes from causing the surplus to exceed
1569 	 * overcommit
1570 	 *
1571 	 * This however introduces a different race, where a process B
1572 	 * tries to grow the static hugepage pool while alloc_pages() is
1573 	 * called by process A. B will only examine the per-node
1574 	 * counters in determining if surplus huge pages can be
1575 	 * converted to normal huge pages in adjust_pool_surplus(). A
1576 	 * won't be able to increment the per-node counter, until the
1577 	 * lock is dropped by B, but B doesn't drop hugetlb_lock until
1578 	 * no more huge pages can be converted from surplus to normal
1579 	 * state (and doesn't try to convert again). Thus, we have a
1580 	 * case where a surplus huge page exists, the pool is grown, and
1581 	 * the surplus huge page still exists after, even though it
1582 	 * should just have been converted to a normal huge page. This
1583 	 * does not leak memory, though, as the hugepage will be freed
1584 	 * once it is out of use. It also does not allow the counters to
1585 	 * go out of whack in adjust_pool_surplus() as we don't modify
1586 	 * the node values until we've gotten the hugepage and only the
1587 	 * per-node value is checked there.
1588 	 */
1589 	spin_lock(&hugetlb_lock);
1590 	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
1591 		spin_unlock(&hugetlb_lock);
1592 		return NULL;
1593 	} else {
1594 		h->nr_huge_pages++;
1595 		h->surplus_huge_pages++;
1596 	}
1597 	spin_unlock(&hugetlb_lock);
1598 
1599 	page = __hugetlb_alloc_buddy_huge_page(h, vma, addr, nid);
1600 
1601 	spin_lock(&hugetlb_lock);
1602 	if (page) {
1603 		INIT_LIST_HEAD(&page->lru);
1604 		r_nid = page_to_nid(page);
1605 		set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1606 		set_hugetlb_cgroup(page, NULL);
1607 		/*
1608 		 * We incremented the global counters already
1609 		 */
1610 		h->nr_huge_pages_node[r_nid]++;
1611 		h->surplus_huge_pages_node[r_nid]++;
1612 		__count_vm_event(HTLB_BUDDY_PGALLOC);
1613 	} else {
1614 		h->nr_huge_pages--;
1615 		h->surplus_huge_pages--;
1616 		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1617 	}
1618 	spin_unlock(&hugetlb_lock);
1619 
1620 	return page;
1621 }
1622 
1623 /*
1624  * Allocate a huge page from 'nid'.  Note, 'nid' may be
1625  * NUMA_NO_NODE, which means that it may be allocated
1626  * anywhere.
1627  */
1628 static
1629 struct page *__alloc_buddy_huge_page_no_mpol(struct hstate *h, int nid)
1630 {
1631 	unsigned long addr = -1;
1632 
1633 	return __alloc_buddy_huge_page(h, NULL, addr, nid);
1634 }
1635 
1636 /*
1637  * Use the VMA's mpolicy to allocate a huge page from the buddy.
1638  */
1639 static
1640 struct page *__alloc_buddy_huge_page_with_mpol(struct hstate *h,
1641 		struct vm_area_struct *vma, unsigned long addr)
1642 {
1643 	return __alloc_buddy_huge_page(h, vma, addr, NUMA_NO_NODE);
1644 }
1645 
1646 /*
1647  * This allocation function is useful in the context where vma is irrelevant.
1648  * E.g. soft-offlining uses this function because it only cares physical
1649  * address of error page.
1650  */
1651 struct page *alloc_huge_page_node(struct hstate *h, int nid)
1652 {
1653 	struct page *page = NULL;
1654 
1655 	spin_lock(&hugetlb_lock);
1656 	if (h->free_huge_pages - h->resv_huge_pages > 0)
1657 		page = dequeue_huge_page_node(h, nid);
1658 	spin_unlock(&hugetlb_lock);
1659 
1660 	if (!page)
1661 		page = __alloc_buddy_huge_page_no_mpol(h, nid);
1662 
1663 	return page;
1664 }
1665 
1666 /*
1667  * Increase the hugetlb pool such that it can accommodate a reservation
1668  * of size 'delta'.
1669  */
1670 static int gather_surplus_pages(struct hstate *h, int delta)
1671 {
1672 	struct list_head surplus_list;
1673 	struct page *page, *tmp;
1674 	int ret, i;
1675 	int needed, allocated;
1676 	bool alloc_ok = true;
1677 
1678 	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
1679 	if (needed <= 0) {
1680 		h->resv_huge_pages += delta;
1681 		return 0;
1682 	}
1683 
1684 	allocated = 0;
1685 	INIT_LIST_HEAD(&surplus_list);
1686 
1687 	ret = -ENOMEM;
1688 retry:
1689 	spin_unlock(&hugetlb_lock);
1690 	for (i = 0; i < needed; i++) {
1691 		page = __alloc_buddy_huge_page_no_mpol(h, NUMA_NO_NODE);
1692 		if (!page) {
1693 			alloc_ok = false;
1694 			break;
1695 		}
1696 		list_add(&page->lru, &surplus_list);
1697 	}
1698 	allocated += i;
1699 
1700 	/*
1701 	 * After retaking hugetlb_lock, we need to recalculate 'needed'
1702 	 * because either resv_huge_pages or free_huge_pages may have changed.
1703 	 */
1704 	spin_lock(&hugetlb_lock);
1705 	needed = (h->resv_huge_pages + delta) -
1706 			(h->free_huge_pages + allocated);
1707 	if (needed > 0) {
1708 		if (alloc_ok)
1709 			goto retry;
1710 		/*
1711 		 * We were not able to allocate enough pages to
1712 		 * satisfy the entire reservation so we free what
1713 		 * we've allocated so far.
1714 		 */
1715 		goto free;
1716 	}
1717 	/*
1718 	 * The surplus_list now contains _at_least_ the number of extra pages
1719 	 * needed to accommodate the reservation.  Add the appropriate number
1720 	 * of pages to the hugetlb pool and free the extras back to the buddy
1721 	 * allocator.  Commit the entire reservation here to prevent another
1722 	 * process from stealing the pages as they are added to the pool but
1723 	 * before they are reserved.
1724 	 */
1725 	needed += allocated;
1726 	h->resv_huge_pages += delta;
1727 	ret = 0;
1728 
1729 	/* Free the needed pages to the hugetlb pool */
1730 	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
1731 		if ((--needed) < 0)
1732 			break;
1733 		/*
1734 		 * This page is now managed by the hugetlb allocator and has
1735 		 * no users -- drop the buddy allocator's reference.
1736 		 */
1737 		put_page_testzero(page);
1738 		VM_BUG_ON_PAGE(page_count(page), page);
1739 		enqueue_huge_page(h, page);
1740 	}
1741 free:
1742 	spin_unlock(&hugetlb_lock);
1743 
1744 	/* Free unnecessary surplus pages to the buddy allocator */
1745 	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
1746 		put_page(page);
1747 	spin_lock(&hugetlb_lock);
1748 
1749 	return ret;
1750 }
1751 
1752 /*
1753  * When releasing a hugetlb pool reservation, any surplus pages that were
1754  * allocated to satisfy the reservation must be explicitly freed if they were
1755  * never used.
1756  * Called with hugetlb_lock held.
1757  */
1758 static void return_unused_surplus_pages(struct hstate *h,
1759 					unsigned long unused_resv_pages)
1760 {
1761 	unsigned long nr_pages;
1762 
1763 	/* Uncommit the reservation */
1764 	h->resv_huge_pages -= unused_resv_pages;
1765 
1766 	/* Cannot return gigantic pages currently */
1767 	if (hstate_is_gigantic(h))
1768 		return;
1769 
1770 	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
1771 
1772 	/*
1773 	 * We want to release as many surplus pages as possible, spread
1774 	 * evenly across all nodes with memory. Iterate across these nodes
1775 	 * until we can no longer free unreserved surplus pages. This occurs
1776 	 * when the nodes with surplus pages have no free pages.
1777 	 * free_pool_huge_page() will balance the the freed pages across the
1778 	 * on-line nodes with memory and will handle the hstate accounting.
1779 	 */
1780 	while (nr_pages--) {
1781 		if (!free_pool_huge_page(h, &node_states[N_MEMORY], 1))
1782 			break;
1783 		cond_resched_lock(&hugetlb_lock);
1784 	}
1785 }
1786 
1787 
1788 /*
1789  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
1790  * are used by the huge page allocation routines to manage reservations.
1791  *
1792  * vma_needs_reservation is called to determine if the huge page at addr
1793  * within the vma has an associated reservation.  If a reservation is
1794  * needed, the value 1 is returned.  The caller is then responsible for
1795  * managing the global reservation and subpool usage counts.  After
1796  * the huge page has been allocated, vma_commit_reservation is called
1797  * to add the page to the reservation map.  If the page allocation fails,
1798  * the reservation must be ended instead of committed.  vma_end_reservation
1799  * is called in such cases.
1800  *
1801  * In the normal case, vma_commit_reservation returns the same value
1802  * as the preceding vma_needs_reservation call.  The only time this
1803  * is not the case is if a reserve map was changed between calls.  It
1804  * is the responsibility of the caller to notice the difference and
1805  * take appropriate action.
1806  */
1807 enum vma_resv_mode {
1808 	VMA_NEEDS_RESV,
1809 	VMA_COMMIT_RESV,
1810 	VMA_END_RESV,
1811 };
1812 static long __vma_reservation_common(struct hstate *h,
1813 				struct vm_area_struct *vma, unsigned long addr,
1814 				enum vma_resv_mode mode)
1815 {
1816 	struct resv_map *resv;
1817 	pgoff_t idx;
1818 	long ret;
1819 
1820 	resv = vma_resv_map(vma);
1821 	if (!resv)
1822 		return 1;
1823 
1824 	idx = vma_hugecache_offset(h, vma, addr);
1825 	switch (mode) {
1826 	case VMA_NEEDS_RESV:
1827 		ret = region_chg(resv, idx, idx + 1);
1828 		break;
1829 	case VMA_COMMIT_RESV:
1830 		ret = region_add(resv, idx, idx + 1);
1831 		break;
1832 	case VMA_END_RESV:
1833 		region_abort(resv, idx, idx + 1);
1834 		ret = 0;
1835 		break;
1836 	default:
1837 		BUG();
1838 	}
1839 
1840 	if (vma->vm_flags & VM_MAYSHARE)
1841 		return ret;
1842 	else if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) && ret >= 0) {
1843 		/*
1844 		 * In most cases, reserves always exist for private mappings.
1845 		 * However, a file associated with mapping could have been
1846 		 * hole punched or truncated after reserves were consumed.
1847 		 * As subsequent fault on such a range will not use reserves.
1848 		 * Subtle - The reserve map for private mappings has the
1849 		 * opposite meaning than that of shared mappings.  If NO
1850 		 * entry is in the reserve map, it means a reservation exists.
1851 		 * If an entry exists in the reserve map, it means the
1852 		 * reservation has already been consumed.  As a result, the
1853 		 * return value of this routine is the opposite of the
1854 		 * value returned from reserve map manipulation routines above.
1855 		 */
1856 		if (ret)
1857 			return 0;
1858 		else
1859 			return 1;
1860 	}
1861 	else
1862 		return ret < 0 ? ret : 0;
1863 }
1864 
1865 static long vma_needs_reservation(struct hstate *h,
1866 			struct vm_area_struct *vma, unsigned long addr)
1867 {
1868 	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
1869 }
1870 
1871 static long vma_commit_reservation(struct hstate *h,
1872 			struct vm_area_struct *vma, unsigned long addr)
1873 {
1874 	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
1875 }
1876 
1877 static void vma_end_reservation(struct hstate *h,
1878 			struct vm_area_struct *vma, unsigned long addr)
1879 {
1880 	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
1881 }
1882 
1883 struct page *alloc_huge_page(struct vm_area_struct *vma,
1884 				    unsigned long addr, int avoid_reserve)
1885 {
1886 	struct hugepage_subpool *spool = subpool_vma(vma);
1887 	struct hstate *h = hstate_vma(vma);
1888 	struct page *page;
1889 	long map_chg, map_commit;
1890 	long gbl_chg;
1891 	int ret, idx;
1892 	struct hugetlb_cgroup *h_cg;
1893 
1894 	idx = hstate_index(h);
1895 	/*
1896 	 * Examine the region/reserve map to determine if the process
1897 	 * has a reservation for the page to be allocated.  A return
1898 	 * code of zero indicates a reservation exists (no change).
1899 	 */
1900 	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
1901 	if (map_chg < 0)
1902 		return ERR_PTR(-ENOMEM);
1903 
1904 	/*
1905 	 * Processes that did not create the mapping will have no
1906 	 * reserves as indicated by the region/reserve map. Check
1907 	 * that the allocation will not exceed the subpool limit.
1908 	 * Allocations for MAP_NORESERVE mappings also need to be
1909 	 * checked against any subpool limit.
1910 	 */
1911 	if (map_chg || avoid_reserve) {
1912 		gbl_chg = hugepage_subpool_get_pages(spool, 1);
1913 		if (gbl_chg < 0) {
1914 			vma_end_reservation(h, vma, addr);
1915 			return ERR_PTR(-ENOSPC);
1916 		}
1917 
1918 		/*
1919 		 * Even though there was no reservation in the region/reserve
1920 		 * map, there could be reservations associated with the
1921 		 * subpool that can be used.  This would be indicated if the
1922 		 * return value of hugepage_subpool_get_pages() is zero.
1923 		 * However, if avoid_reserve is specified we still avoid even
1924 		 * the subpool reservations.
1925 		 */
1926 		if (avoid_reserve)
1927 			gbl_chg = 1;
1928 	}
1929 
1930 	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
1931 	if (ret)
1932 		goto out_subpool_put;
1933 
1934 	spin_lock(&hugetlb_lock);
1935 	/*
1936 	 * glb_chg is passed to indicate whether or not a page must be taken
1937 	 * from the global free pool (global change).  gbl_chg == 0 indicates
1938 	 * a reservation exists for the allocation.
1939 	 */
1940 	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
1941 	if (!page) {
1942 		spin_unlock(&hugetlb_lock);
1943 		page = __alloc_buddy_huge_page_with_mpol(h, vma, addr);
1944 		if (!page)
1945 			goto out_uncharge_cgroup;
1946 		if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
1947 			SetPagePrivate(page);
1948 			h->resv_huge_pages--;
1949 		}
1950 		spin_lock(&hugetlb_lock);
1951 		list_move(&page->lru, &h->hugepage_activelist);
1952 		/* Fall through */
1953 	}
1954 	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
1955 	spin_unlock(&hugetlb_lock);
1956 
1957 	set_page_private(page, (unsigned long)spool);
1958 
1959 	map_commit = vma_commit_reservation(h, vma, addr);
1960 	if (unlikely(map_chg > map_commit)) {
1961 		/*
1962 		 * The page was added to the reservation map between
1963 		 * vma_needs_reservation and vma_commit_reservation.
1964 		 * This indicates a race with hugetlb_reserve_pages.
1965 		 * Adjust for the subpool count incremented above AND
1966 		 * in hugetlb_reserve_pages for the same page.  Also,
1967 		 * the reservation count added in hugetlb_reserve_pages
1968 		 * no longer applies.
1969 		 */
1970 		long rsv_adjust;
1971 
1972 		rsv_adjust = hugepage_subpool_put_pages(spool, 1);
1973 		hugetlb_acct_memory(h, -rsv_adjust);
1974 	}
1975 	return page;
1976 
1977 out_uncharge_cgroup:
1978 	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
1979 out_subpool_put:
1980 	if (map_chg || avoid_reserve)
1981 		hugepage_subpool_put_pages(spool, 1);
1982 	vma_end_reservation(h, vma, addr);
1983 	return ERR_PTR(-ENOSPC);
1984 }
1985 
1986 /*
1987  * alloc_huge_page()'s wrapper which simply returns the page if allocation
1988  * succeeds, otherwise NULL. This function is called from new_vma_page(),
1989  * where no ERR_VALUE is expected to be returned.
1990  */
1991 struct page *alloc_huge_page_noerr(struct vm_area_struct *vma,
1992 				unsigned long addr, int avoid_reserve)
1993 {
1994 	struct page *page = alloc_huge_page(vma, addr, avoid_reserve);
1995 	if (IS_ERR(page))
1996 		page = NULL;
1997 	return page;
1998 }
1999 
2000 int __weak alloc_bootmem_huge_page(struct hstate *h)
2001 {
2002 	struct huge_bootmem_page *m;
2003 	int nr_nodes, node;
2004 
2005 	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2006 		void *addr;
2007 
2008 		addr = memblock_virt_alloc_try_nid_nopanic(
2009 				huge_page_size(h), huge_page_size(h),
2010 				0, BOOTMEM_ALLOC_ACCESSIBLE, node);
2011 		if (addr) {
2012 			/*
2013 			 * Use the beginning of the huge page to store the
2014 			 * huge_bootmem_page struct (until gather_bootmem
2015 			 * puts them into the mem_map).
2016 			 */
2017 			m = addr;
2018 			goto found;
2019 		}
2020 	}
2021 	return 0;
2022 
2023 found:
2024 	BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2025 	/* Put them into a private list first because mem_map is not up yet */
2026 	list_add(&m->list, &huge_boot_pages);
2027 	m->hstate = h;
2028 	return 1;
2029 }
2030 
2031 static void __init prep_compound_huge_page(struct page *page,
2032 		unsigned int order)
2033 {
2034 	if (unlikely(order > (MAX_ORDER - 1)))
2035 		prep_compound_gigantic_page(page, order);
2036 	else
2037 		prep_compound_page(page, order);
2038 }
2039 
2040 /* Put bootmem huge pages into the standard lists after mem_map is up */
2041 static void __init gather_bootmem_prealloc(void)
2042 {
2043 	struct huge_bootmem_page *m;
2044 
2045 	list_for_each_entry(m, &huge_boot_pages, list) {
2046 		struct hstate *h = m->hstate;
2047 		struct page *page;
2048 
2049 #ifdef CONFIG_HIGHMEM
2050 		page = pfn_to_page(m->phys >> PAGE_SHIFT);
2051 		memblock_free_late(__pa(m),
2052 				   sizeof(struct huge_bootmem_page));
2053 #else
2054 		page = virt_to_page(m);
2055 #endif
2056 		WARN_ON(page_count(page) != 1);
2057 		prep_compound_huge_page(page, h->order);
2058 		WARN_ON(PageReserved(page));
2059 		prep_new_huge_page(h, page, page_to_nid(page));
2060 		/*
2061 		 * If we had gigantic hugepages allocated at boot time, we need
2062 		 * to restore the 'stolen' pages to totalram_pages in order to
2063 		 * fix confusing memory reports from free(1) and another
2064 		 * side-effects, like CommitLimit going negative.
2065 		 */
2066 		if (hstate_is_gigantic(h))
2067 			adjust_managed_page_count(page, 1 << h->order);
2068 	}
2069 }
2070 
2071 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2072 {
2073 	unsigned long i;
2074 
2075 	for (i = 0; i < h->max_huge_pages; ++i) {
2076 		if (hstate_is_gigantic(h)) {
2077 			if (!alloc_bootmem_huge_page(h))
2078 				break;
2079 		} else if (!alloc_fresh_huge_page(h,
2080 					 &node_states[N_MEMORY]))
2081 			break;
2082 	}
2083 	h->max_huge_pages = i;
2084 }
2085 
2086 static void __init hugetlb_init_hstates(void)
2087 {
2088 	struct hstate *h;
2089 
2090 	for_each_hstate(h) {
2091 		if (minimum_order > huge_page_order(h))
2092 			minimum_order = huge_page_order(h);
2093 
2094 		/* oversize hugepages were init'ed in early boot */
2095 		if (!hstate_is_gigantic(h))
2096 			hugetlb_hstate_alloc_pages(h);
2097 	}
2098 	VM_BUG_ON(minimum_order == UINT_MAX);
2099 }
2100 
2101 static char * __init memfmt(char *buf, unsigned long n)
2102 {
2103 	if (n >= (1UL << 30))
2104 		sprintf(buf, "%lu GB", n >> 30);
2105 	else if (n >= (1UL << 20))
2106 		sprintf(buf, "%lu MB", n >> 20);
2107 	else
2108 		sprintf(buf, "%lu KB", n >> 10);
2109 	return buf;
2110 }
2111 
2112 static void __init report_hugepages(void)
2113 {
2114 	struct hstate *h;
2115 
2116 	for_each_hstate(h) {
2117 		char buf[32];
2118 		pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2119 			memfmt(buf, huge_page_size(h)),
2120 			h->free_huge_pages);
2121 	}
2122 }
2123 
2124 #ifdef CONFIG_HIGHMEM
2125 static void try_to_free_low(struct hstate *h, unsigned long count,
2126 						nodemask_t *nodes_allowed)
2127 {
2128 	int i;
2129 
2130 	if (hstate_is_gigantic(h))
2131 		return;
2132 
2133 	for_each_node_mask(i, *nodes_allowed) {
2134 		struct page *page, *next;
2135 		struct list_head *freel = &h->hugepage_freelists[i];
2136 		list_for_each_entry_safe(page, next, freel, lru) {
2137 			if (count >= h->nr_huge_pages)
2138 				return;
2139 			if (PageHighMem(page))
2140 				continue;
2141 			list_del(&page->lru);
2142 			update_and_free_page(h, page);
2143 			h->free_huge_pages--;
2144 			h->free_huge_pages_node[page_to_nid(page)]--;
2145 		}
2146 	}
2147 }
2148 #else
2149 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2150 						nodemask_t *nodes_allowed)
2151 {
2152 }
2153 #endif
2154 
2155 /*
2156  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2157  * balanced by operating on them in a round-robin fashion.
2158  * Returns 1 if an adjustment was made.
2159  */
2160 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2161 				int delta)
2162 {
2163 	int nr_nodes, node;
2164 
2165 	VM_BUG_ON(delta != -1 && delta != 1);
2166 
2167 	if (delta < 0) {
2168 		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2169 			if (h->surplus_huge_pages_node[node])
2170 				goto found;
2171 		}
2172 	} else {
2173 		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2174 			if (h->surplus_huge_pages_node[node] <
2175 					h->nr_huge_pages_node[node])
2176 				goto found;
2177 		}
2178 	}
2179 	return 0;
2180 
2181 found:
2182 	h->surplus_huge_pages += delta;
2183 	h->surplus_huge_pages_node[node] += delta;
2184 	return 1;
2185 }
2186 
2187 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2188 static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
2189 						nodemask_t *nodes_allowed)
2190 {
2191 	unsigned long min_count, ret;
2192 
2193 	if (hstate_is_gigantic(h) && !gigantic_page_supported())
2194 		return h->max_huge_pages;
2195 
2196 	/*
2197 	 * Increase the pool size
2198 	 * First take pages out of surplus state.  Then make up the
2199 	 * remaining difference by allocating fresh huge pages.
2200 	 *
2201 	 * We might race with __alloc_buddy_huge_page() here and be unable
2202 	 * to convert a surplus huge page to a normal huge page. That is
2203 	 * not critical, though, it just means the overall size of the
2204 	 * pool might be one hugepage larger than it needs to be, but
2205 	 * within all the constraints specified by the sysctls.
2206 	 */
2207 	spin_lock(&hugetlb_lock);
2208 	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
2209 		if (!adjust_pool_surplus(h, nodes_allowed, -1))
2210 			break;
2211 	}
2212 
2213 	while (count > persistent_huge_pages(h)) {
2214 		/*
2215 		 * If this allocation races such that we no longer need the
2216 		 * page, free_huge_page will handle it by freeing the page
2217 		 * and reducing the surplus.
2218 		 */
2219 		spin_unlock(&hugetlb_lock);
2220 
2221 		/* yield cpu to avoid soft lockup */
2222 		cond_resched();
2223 
2224 		if (hstate_is_gigantic(h))
2225 			ret = alloc_fresh_gigantic_page(h, nodes_allowed);
2226 		else
2227 			ret = alloc_fresh_huge_page(h, nodes_allowed);
2228 		spin_lock(&hugetlb_lock);
2229 		if (!ret)
2230 			goto out;
2231 
2232 		/* Bail for signals. Probably ctrl-c from user */
2233 		if (signal_pending(current))
2234 			goto out;
2235 	}
2236 
2237 	/*
2238 	 * Decrease the pool size
2239 	 * First return free pages to the buddy allocator (being careful
2240 	 * to keep enough around to satisfy reservations).  Then place
2241 	 * pages into surplus state as needed so the pool will shrink
2242 	 * to the desired size as pages become free.
2243 	 *
2244 	 * By placing pages into the surplus state independent of the
2245 	 * overcommit value, we are allowing the surplus pool size to
2246 	 * exceed overcommit. There are few sane options here. Since
2247 	 * __alloc_buddy_huge_page() is checking the global counter,
2248 	 * though, we'll note that we're not allowed to exceed surplus
2249 	 * and won't grow the pool anywhere else. Not until one of the
2250 	 * sysctls are changed, or the surplus pages go out of use.
2251 	 */
2252 	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
2253 	min_count = max(count, min_count);
2254 	try_to_free_low(h, min_count, nodes_allowed);
2255 	while (min_count < persistent_huge_pages(h)) {
2256 		if (!free_pool_huge_page(h, nodes_allowed, 0))
2257 			break;
2258 		cond_resched_lock(&hugetlb_lock);
2259 	}
2260 	while (count < persistent_huge_pages(h)) {
2261 		if (!adjust_pool_surplus(h, nodes_allowed, 1))
2262 			break;
2263 	}
2264 out:
2265 	ret = persistent_huge_pages(h);
2266 	spin_unlock(&hugetlb_lock);
2267 	return ret;
2268 }
2269 
2270 #define HSTATE_ATTR_RO(_name) \
2271 	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
2272 
2273 #define HSTATE_ATTR(_name) \
2274 	static struct kobj_attribute _name##_attr = \
2275 		__ATTR(_name, 0644, _name##_show, _name##_store)
2276 
2277 static struct kobject *hugepages_kobj;
2278 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
2279 
2280 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
2281 
2282 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
2283 {
2284 	int i;
2285 
2286 	for (i = 0; i < HUGE_MAX_HSTATE; i++)
2287 		if (hstate_kobjs[i] == kobj) {
2288 			if (nidp)
2289 				*nidp = NUMA_NO_NODE;
2290 			return &hstates[i];
2291 		}
2292 
2293 	return kobj_to_node_hstate(kobj, nidp);
2294 }
2295 
2296 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
2297 					struct kobj_attribute *attr, char *buf)
2298 {
2299 	struct hstate *h;
2300 	unsigned long nr_huge_pages;
2301 	int nid;
2302 
2303 	h = kobj_to_hstate(kobj, &nid);
2304 	if (nid == NUMA_NO_NODE)
2305 		nr_huge_pages = h->nr_huge_pages;
2306 	else
2307 		nr_huge_pages = h->nr_huge_pages_node[nid];
2308 
2309 	return sprintf(buf, "%lu\n", nr_huge_pages);
2310 }
2311 
2312 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
2313 					   struct hstate *h, int nid,
2314 					   unsigned long count, size_t len)
2315 {
2316 	int err;
2317 	NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
2318 
2319 	if (hstate_is_gigantic(h) && !gigantic_page_supported()) {
2320 		err = -EINVAL;
2321 		goto out;
2322 	}
2323 
2324 	if (nid == NUMA_NO_NODE) {
2325 		/*
2326 		 * global hstate attribute
2327 		 */
2328 		if (!(obey_mempolicy &&
2329 				init_nodemask_of_mempolicy(nodes_allowed))) {
2330 			NODEMASK_FREE(nodes_allowed);
2331 			nodes_allowed = &node_states[N_MEMORY];
2332 		}
2333 	} else if (nodes_allowed) {
2334 		/*
2335 		 * per node hstate attribute: adjust count to global,
2336 		 * but restrict alloc/free to the specified node.
2337 		 */
2338 		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
2339 		init_nodemask_of_node(nodes_allowed, nid);
2340 	} else
2341 		nodes_allowed = &node_states[N_MEMORY];
2342 
2343 	h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
2344 
2345 	if (nodes_allowed != &node_states[N_MEMORY])
2346 		NODEMASK_FREE(nodes_allowed);
2347 
2348 	return len;
2349 out:
2350 	NODEMASK_FREE(nodes_allowed);
2351 	return err;
2352 }
2353 
2354 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
2355 					 struct kobject *kobj, const char *buf,
2356 					 size_t len)
2357 {
2358 	struct hstate *h;
2359 	unsigned long count;
2360 	int nid;
2361 	int err;
2362 
2363 	err = kstrtoul(buf, 10, &count);
2364 	if (err)
2365 		return err;
2366 
2367 	h = kobj_to_hstate(kobj, &nid);
2368 	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
2369 }
2370 
2371 static ssize_t nr_hugepages_show(struct kobject *kobj,
2372 				       struct kobj_attribute *attr, char *buf)
2373 {
2374 	return nr_hugepages_show_common(kobj, attr, buf);
2375 }
2376 
2377 static ssize_t nr_hugepages_store(struct kobject *kobj,
2378 	       struct kobj_attribute *attr, const char *buf, size_t len)
2379 {
2380 	return nr_hugepages_store_common(false, kobj, buf, len);
2381 }
2382 HSTATE_ATTR(nr_hugepages);
2383 
2384 #ifdef CONFIG_NUMA
2385 
2386 /*
2387  * hstate attribute for optionally mempolicy-based constraint on persistent
2388  * huge page alloc/free.
2389  */
2390 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
2391 				       struct kobj_attribute *attr, char *buf)
2392 {
2393 	return nr_hugepages_show_common(kobj, attr, buf);
2394 }
2395 
2396 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
2397 	       struct kobj_attribute *attr, const char *buf, size_t len)
2398 {
2399 	return nr_hugepages_store_common(true, kobj, buf, len);
2400 }
2401 HSTATE_ATTR(nr_hugepages_mempolicy);
2402 #endif
2403 
2404 
2405 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
2406 					struct kobj_attribute *attr, char *buf)
2407 {
2408 	struct hstate *h = kobj_to_hstate(kobj, NULL);
2409 	return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
2410 }
2411 
2412 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
2413 		struct kobj_attribute *attr, const char *buf, size_t count)
2414 {
2415 	int err;
2416 	unsigned long input;
2417 	struct hstate *h = kobj_to_hstate(kobj, NULL);
2418 
2419 	if (hstate_is_gigantic(h))
2420 		return -EINVAL;
2421 
2422 	err = kstrtoul(buf, 10, &input);
2423 	if (err)
2424 		return err;
2425 
2426 	spin_lock(&hugetlb_lock);
2427 	h->nr_overcommit_huge_pages = input;
2428 	spin_unlock(&hugetlb_lock);
2429 
2430 	return count;
2431 }
2432 HSTATE_ATTR(nr_overcommit_hugepages);
2433 
2434 static ssize_t free_hugepages_show(struct kobject *kobj,
2435 					struct kobj_attribute *attr, char *buf)
2436 {
2437 	struct hstate *h;
2438 	unsigned long free_huge_pages;
2439 	int nid;
2440 
2441 	h = kobj_to_hstate(kobj, &nid);
2442 	if (nid == NUMA_NO_NODE)
2443 		free_huge_pages = h->free_huge_pages;
2444 	else
2445 		free_huge_pages = h->free_huge_pages_node[nid];
2446 
2447 	return sprintf(buf, "%lu\n", free_huge_pages);
2448 }
2449 HSTATE_ATTR_RO(free_hugepages);
2450 
2451 static ssize_t resv_hugepages_show(struct kobject *kobj,
2452 					struct kobj_attribute *attr, char *buf)
2453 {
2454 	struct hstate *h = kobj_to_hstate(kobj, NULL);
2455 	return sprintf(buf, "%lu\n", h->resv_huge_pages);
2456 }
2457 HSTATE_ATTR_RO(resv_hugepages);
2458 
2459 static ssize_t surplus_hugepages_show(struct kobject *kobj,
2460 					struct kobj_attribute *attr, char *buf)
2461 {
2462 	struct hstate *h;
2463 	unsigned long surplus_huge_pages;
2464 	int nid;
2465 
2466 	h = kobj_to_hstate(kobj, &nid);
2467 	if (nid == NUMA_NO_NODE)
2468 		surplus_huge_pages = h->surplus_huge_pages;
2469 	else
2470 		surplus_huge_pages = h->surplus_huge_pages_node[nid];
2471 
2472 	return sprintf(buf, "%lu\n", surplus_huge_pages);
2473 }
2474 HSTATE_ATTR_RO(surplus_hugepages);
2475 
2476 static struct attribute *hstate_attrs[] = {
2477 	&nr_hugepages_attr.attr,
2478 	&nr_overcommit_hugepages_attr.attr,
2479 	&free_hugepages_attr.attr,
2480 	&resv_hugepages_attr.attr,
2481 	&surplus_hugepages_attr.attr,
2482 #ifdef CONFIG_NUMA
2483 	&nr_hugepages_mempolicy_attr.attr,
2484 #endif
2485 	NULL,
2486 };
2487 
2488 static struct attribute_group hstate_attr_group = {
2489 	.attrs = hstate_attrs,
2490 };
2491 
2492 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
2493 				    struct kobject **hstate_kobjs,
2494 				    struct attribute_group *hstate_attr_group)
2495 {
2496 	int retval;
2497 	int hi = hstate_index(h);
2498 
2499 	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
2500 	if (!hstate_kobjs[hi])
2501 		return -ENOMEM;
2502 
2503 	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
2504 	if (retval)
2505 		kobject_put(hstate_kobjs[hi]);
2506 
2507 	return retval;
2508 }
2509 
2510 static void __init hugetlb_sysfs_init(void)
2511 {
2512 	struct hstate *h;
2513 	int err;
2514 
2515 	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
2516 	if (!hugepages_kobj)
2517 		return;
2518 
2519 	for_each_hstate(h) {
2520 		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
2521 					 hstate_kobjs, &hstate_attr_group);
2522 		if (err)
2523 			pr_err("Hugetlb: Unable to add hstate %s", h->name);
2524 	}
2525 }
2526 
2527 #ifdef CONFIG_NUMA
2528 
2529 /*
2530  * node_hstate/s - associate per node hstate attributes, via their kobjects,
2531  * with node devices in node_devices[] using a parallel array.  The array
2532  * index of a node device or _hstate == node id.
2533  * This is here to avoid any static dependency of the node device driver, in
2534  * the base kernel, on the hugetlb module.
2535  */
2536 struct node_hstate {
2537 	struct kobject		*hugepages_kobj;
2538 	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
2539 };
2540 static struct node_hstate node_hstates[MAX_NUMNODES];
2541 
2542 /*
2543  * A subset of global hstate attributes for node devices
2544  */
2545 static struct attribute *per_node_hstate_attrs[] = {
2546 	&nr_hugepages_attr.attr,
2547 	&free_hugepages_attr.attr,
2548 	&surplus_hugepages_attr.attr,
2549 	NULL,
2550 };
2551 
2552 static struct attribute_group per_node_hstate_attr_group = {
2553 	.attrs = per_node_hstate_attrs,
2554 };
2555 
2556 /*
2557  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
2558  * Returns node id via non-NULL nidp.
2559  */
2560 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2561 {
2562 	int nid;
2563 
2564 	for (nid = 0; nid < nr_node_ids; nid++) {
2565 		struct node_hstate *nhs = &node_hstates[nid];
2566 		int i;
2567 		for (i = 0; i < HUGE_MAX_HSTATE; i++)
2568 			if (nhs->hstate_kobjs[i] == kobj) {
2569 				if (nidp)
2570 					*nidp = nid;
2571 				return &hstates[i];
2572 			}
2573 	}
2574 
2575 	BUG();
2576 	return NULL;
2577 }
2578 
2579 /*
2580  * Unregister hstate attributes from a single node device.
2581  * No-op if no hstate attributes attached.
2582  */
2583 static void hugetlb_unregister_node(struct node *node)
2584 {
2585 	struct hstate *h;
2586 	struct node_hstate *nhs = &node_hstates[node->dev.id];
2587 
2588 	if (!nhs->hugepages_kobj)
2589 		return;		/* no hstate attributes */
2590 
2591 	for_each_hstate(h) {
2592 		int idx = hstate_index(h);
2593 		if (nhs->hstate_kobjs[idx]) {
2594 			kobject_put(nhs->hstate_kobjs[idx]);
2595 			nhs->hstate_kobjs[idx] = NULL;
2596 		}
2597 	}
2598 
2599 	kobject_put(nhs->hugepages_kobj);
2600 	nhs->hugepages_kobj = NULL;
2601 }
2602 
2603 
2604 /*
2605  * Register hstate attributes for a single node device.
2606  * No-op if attributes already registered.
2607  */
2608 static void hugetlb_register_node(struct node *node)
2609 {
2610 	struct hstate *h;
2611 	struct node_hstate *nhs = &node_hstates[node->dev.id];
2612 	int err;
2613 
2614 	if (nhs->hugepages_kobj)
2615 		return;		/* already allocated */
2616 
2617 	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
2618 							&node->dev.kobj);
2619 	if (!nhs->hugepages_kobj)
2620 		return;
2621 
2622 	for_each_hstate(h) {
2623 		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
2624 						nhs->hstate_kobjs,
2625 						&per_node_hstate_attr_group);
2626 		if (err) {
2627 			pr_err("Hugetlb: Unable to add hstate %s for node %d\n",
2628 				h->name, node->dev.id);
2629 			hugetlb_unregister_node(node);
2630 			break;
2631 		}
2632 	}
2633 }
2634 
2635 /*
2636  * hugetlb init time:  register hstate attributes for all registered node
2637  * devices of nodes that have memory.  All on-line nodes should have
2638  * registered their associated device by this time.
2639  */
2640 static void __init hugetlb_register_all_nodes(void)
2641 {
2642 	int nid;
2643 
2644 	for_each_node_state(nid, N_MEMORY) {
2645 		struct node *node = node_devices[nid];
2646 		if (node->dev.id == nid)
2647 			hugetlb_register_node(node);
2648 	}
2649 
2650 	/*
2651 	 * Let the node device driver know we're here so it can
2652 	 * [un]register hstate attributes on node hotplug.
2653 	 */
2654 	register_hugetlbfs_with_node(hugetlb_register_node,
2655 				     hugetlb_unregister_node);
2656 }
2657 #else	/* !CONFIG_NUMA */
2658 
2659 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
2660 {
2661 	BUG();
2662 	if (nidp)
2663 		*nidp = -1;
2664 	return NULL;
2665 }
2666 
2667 static void hugetlb_register_all_nodes(void) { }
2668 
2669 #endif
2670 
2671 static int __init hugetlb_init(void)
2672 {
2673 	int i;
2674 
2675 	if (!hugepages_supported())
2676 		return 0;
2677 
2678 	if (!size_to_hstate(default_hstate_size)) {
2679 		default_hstate_size = HPAGE_SIZE;
2680 		if (!size_to_hstate(default_hstate_size))
2681 			hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
2682 	}
2683 	default_hstate_idx = hstate_index(size_to_hstate(default_hstate_size));
2684 	if (default_hstate_max_huge_pages) {
2685 		if (!default_hstate.max_huge_pages)
2686 			default_hstate.max_huge_pages = default_hstate_max_huge_pages;
2687 	}
2688 
2689 	hugetlb_init_hstates();
2690 	gather_bootmem_prealloc();
2691 	report_hugepages();
2692 
2693 	hugetlb_sysfs_init();
2694 	hugetlb_register_all_nodes();
2695 	hugetlb_cgroup_file_init();
2696 
2697 #ifdef CONFIG_SMP
2698 	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
2699 #else
2700 	num_fault_mutexes = 1;
2701 #endif
2702 	hugetlb_fault_mutex_table =
2703 		kmalloc(sizeof(struct mutex) * num_fault_mutexes, GFP_KERNEL);
2704 	BUG_ON(!hugetlb_fault_mutex_table);
2705 
2706 	for (i = 0; i < num_fault_mutexes; i++)
2707 		mutex_init(&hugetlb_fault_mutex_table[i]);
2708 	return 0;
2709 }
2710 subsys_initcall(hugetlb_init);
2711 
2712 /* Should be called on processing a hugepagesz=... option */
2713 void __init hugetlb_bad_size(void)
2714 {
2715 	parsed_valid_hugepagesz = false;
2716 }
2717 
2718 void __init hugetlb_add_hstate(unsigned int order)
2719 {
2720 	struct hstate *h;
2721 	unsigned long i;
2722 
2723 	if (size_to_hstate(PAGE_SIZE << order)) {
2724 		pr_warn("hugepagesz= specified twice, ignoring\n");
2725 		return;
2726 	}
2727 	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
2728 	BUG_ON(order == 0);
2729 	h = &hstates[hugetlb_max_hstate++];
2730 	h->order = order;
2731 	h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1);
2732 	h->nr_huge_pages = 0;
2733 	h->free_huge_pages = 0;
2734 	for (i = 0; i < MAX_NUMNODES; ++i)
2735 		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
2736 	INIT_LIST_HEAD(&h->hugepage_activelist);
2737 	h->next_nid_to_alloc = first_memory_node;
2738 	h->next_nid_to_free = first_memory_node;
2739 	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
2740 					huge_page_size(h)/1024);
2741 
2742 	parsed_hstate = h;
2743 }
2744 
2745 static int __init hugetlb_nrpages_setup(char *s)
2746 {
2747 	unsigned long *mhp;
2748 	static unsigned long *last_mhp;
2749 
2750 	if (!parsed_valid_hugepagesz) {
2751 		pr_warn("hugepages = %s preceded by "
2752 			"an unsupported hugepagesz, ignoring\n", s);
2753 		parsed_valid_hugepagesz = true;
2754 		return 1;
2755 	}
2756 	/*
2757 	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter yet,
2758 	 * so this hugepages= parameter goes to the "default hstate".
2759 	 */
2760 	else if (!hugetlb_max_hstate)
2761 		mhp = &default_hstate_max_huge_pages;
2762 	else
2763 		mhp = &parsed_hstate->max_huge_pages;
2764 
2765 	if (mhp == last_mhp) {
2766 		pr_warn("hugepages= specified twice without interleaving hugepagesz=, ignoring\n");
2767 		return 1;
2768 	}
2769 
2770 	if (sscanf(s, "%lu", mhp) <= 0)
2771 		*mhp = 0;
2772 
2773 	/*
2774 	 * Global state is always initialized later in hugetlb_init.
2775 	 * But we need to allocate >= MAX_ORDER hstates here early to still
2776 	 * use the bootmem allocator.
2777 	 */
2778 	if (hugetlb_max_hstate && parsed_hstate->order >= MAX_ORDER)
2779 		hugetlb_hstate_alloc_pages(parsed_hstate);
2780 
2781 	last_mhp = mhp;
2782 
2783 	return 1;
2784 }
2785 __setup("hugepages=", hugetlb_nrpages_setup);
2786 
2787 static int __init hugetlb_default_setup(char *s)
2788 {
2789 	default_hstate_size = memparse(s, &s);
2790 	return 1;
2791 }
2792 __setup("default_hugepagesz=", hugetlb_default_setup);
2793 
2794 static unsigned int cpuset_mems_nr(unsigned int *array)
2795 {
2796 	int node;
2797 	unsigned int nr = 0;
2798 
2799 	for_each_node_mask(node, cpuset_current_mems_allowed)
2800 		nr += array[node];
2801 
2802 	return nr;
2803 }
2804 
2805 #ifdef CONFIG_SYSCTL
2806 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
2807 			 struct ctl_table *table, int write,
2808 			 void __user *buffer, size_t *length, loff_t *ppos)
2809 {
2810 	struct hstate *h = &default_hstate;
2811 	unsigned long tmp = h->max_huge_pages;
2812 	int ret;
2813 
2814 	if (!hugepages_supported())
2815 		return -EOPNOTSUPP;
2816 
2817 	table->data = &tmp;
2818 	table->maxlen = sizeof(unsigned long);
2819 	ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2820 	if (ret)
2821 		goto out;
2822 
2823 	if (write)
2824 		ret = __nr_hugepages_store_common(obey_mempolicy, h,
2825 						  NUMA_NO_NODE, tmp, *length);
2826 out:
2827 	return ret;
2828 }
2829 
2830 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
2831 			  void __user *buffer, size_t *length, loff_t *ppos)
2832 {
2833 
2834 	return hugetlb_sysctl_handler_common(false, table, write,
2835 							buffer, length, ppos);
2836 }
2837 
2838 #ifdef CONFIG_NUMA
2839 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
2840 			  void __user *buffer, size_t *length, loff_t *ppos)
2841 {
2842 	return hugetlb_sysctl_handler_common(true, table, write,
2843 							buffer, length, ppos);
2844 }
2845 #endif /* CONFIG_NUMA */
2846 
2847 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
2848 			void __user *buffer,
2849 			size_t *length, loff_t *ppos)
2850 {
2851 	struct hstate *h = &default_hstate;
2852 	unsigned long tmp;
2853 	int ret;
2854 
2855 	if (!hugepages_supported())
2856 		return -EOPNOTSUPP;
2857 
2858 	tmp = h->nr_overcommit_huge_pages;
2859 
2860 	if (write && hstate_is_gigantic(h))
2861 		return -EINVAL;
2862 
2863 	table->data = &tmp;
2864 	table->maxlen = sizeof(unsigned long);
2865 	ret = proc_doulongvec_minmax(table, write, buffer, length, ppos);
2866 	if (ret)
2867 		goto out;
2868 
2869 	if (write) {
2870 		spin_lock(&hugetlb_lock);
2871 		h->nr_overcommit_huge_pages = tmp;
2872 		spin_unlock(&hugetlb_lock);
2873 	}
2874 out:
2875 	return ret;
2876 }
2877 
2878 #endif /* CONFIG_SYSCTL */
2879 
2880 void hugetlb_report_meminfo(struct seq_file *m)
2881 {
2882 	struct hstate *h = &default_hstate;
2883 	if (!hugepages_supported())
2884 		return;
2885 	seq_printf(m,
2886 			"HugePages_Total:   %5lu\n"
2887 			"HugePages_Free:    %5lu\n"
2888 			"HugePages_Rsvd:    %5lu\n"
2889 			"HugePages_Surp:    %5lu\n"
2890 			"Hugepagesize:   %8lu kB\n",
2891 			h->nr_huge_pages,
2892 			h->free_huge_pages,
2893 			h->resv_huge_pages,
2894 			h->surplus_huge_pages,
2895 			1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2896 }
2897 
2898 int hugetlb_report_node_meminfo(int nid, char *buf)
2899 {
2900 	struct hstate *h = &default_hstate;
2901 	if (!hugepages_supported())
2902 		return 0;
2903 	return sprintf(buf,
2904 		"Node %d HugePages_Total: %5u\n"
2905 		"Node %d HugePages_Free:  %5u\n"
2906 		"Node %d HugePages_Surp:  %5u\n",
2907 		nid, h->nr_huge_pages_node[nid],
2908 		nid, h->free_huge_pages_node[nid],
2909 		nid, h->surplus_huge_pages_node[nid]);
2910 }
2911 
2912 void hugetlb_show_meminfo(void)
2913 {
2914 	struct hstate *h;
2915 	int nid;
2916 
2917 	if (!hugepages_supported())
2918 		return;
2919 
2920 	for_each_node_state(nid, N_MEMORY)
2921 		for_each_hstate(h)
2922 			pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
2923 				nid,
2924 				h->nr_huge_pages_node[nid],
2925 				h->free_huge_pages_node[nid],
2926 				h->surplus_huge_pages_node[nid],
2927 				1UL << (huge_page_order(h) + PAGE_SHIFT - 10));
2928 }
2929 
2930 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
2931 {
2932 	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
2933 		   atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
2934 }
2935 
2936 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
2937 unsigned long hugetlb_total_pages(void)
2938 {
2939 	struct hstate *h;
2940 	unsigned long nr_total_pages = 0;
2941 
2942 	for_each_hstate(h)
2943 		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
2944 	return nr_total_pages;
2945 }
2946 
2947 static int hugetlb_acct_memory(struct hstate *h, long delta)
2948 {
2949 	int ret = -ENOMEM;
2950 
2951 	spin_lock(&hugetlb_lock);
2952 	/*
2953 	 * When cpuset is configured, it breaks the strict hugetlb page
2954 	 * reservation as the accounting is done on a global variable. Such
2955 	 * reservation is completely rubbish in the presence of cpuset because
2956 	 * the reservation is not checked against page availability for the
2957 	 * current cpuset. Application can still potentially OOM'ed by kernel
2958 	 * with lack of free htlb page in cpuset that the task is in.
2959 	 * Attempt to enforce strict accounting with cpuset is almost
2960 	 * impossible (or too ugly) because cpuset is too fluid that
2961 	 * task or memory node can be dynamically moved between cpusets.
2962 	 *
2963 	 * The change of semantics for shared hugetlb mapping with cpuset is
2964 	 * undesirable. However, in order to preserve some of the semantics,
2965 	 * we fall back to check against current free page availability as
2966 	 * a best attempt and hopefully to minimize the impact of changing
2967 	 * semantics that cpuset has.
2968 	 */
2969 	if (delta > 0) {
2970 		if (gather_surplus_pages(h, delta) < 0)
2971 			goto out;
2972 
2973 		if (delta > cpuset_mems_nr(h->free_huge_pages_node)) {
2974 			return_unused_surplus_pages(h, delta);
2975 			goto out;
2976 		}
2977 	}
2978 
2979 	ret = 0;
2980 	if (delta < 0)
2981 		return_unused_surplus_pages(h, (unsigned long) -delta);
2982 
2983 out:
2984 	spin_unlock(&hugetlb_lock);
2985 	return ret;
2986 }
2987 
2988 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
2989 {
2990 	struct resv_map *resv = vma_resv_map(vma);
2991 
2992 	/*
2993 	 * This new VMA should share its siblings reservation map if present.
2994 	 * The VMA will only ever have a valid reservation map pointer where
2995 	 * it is being copied for another still existing VMA.  As that VMA
2996 	 * has a reference to the reservation map it cannot disappear until
2997 	 * after this open call completes.  It is therefore safe to take a
2998 	 * new reference here without additional locking.
2999 	 */
3000 	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3001 		kref_get(&resv->refs);
3002 }
3003 
3004 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
3005 {
3006 	struct hstate *h = hstate_vma(vma);
3007 	struct resv_map *resv = vma_resv_map(vma);
3008 	struct hugepage_subpool *spool = subpool_vma(vma);
3009 	unsigned long reserve, start, end;
3010 	long gbl_reserve;
3011 
3012 	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
3013 		return;
3014 
3015 	start = vma_hugecache_offset(h, vma, vma->vm_start);
3016 	end = vma_hugecache_offset(h, vma, vma->vm_end);
3017 
3018 	reserve = (end - start) - region_count(resv, start, end);
3019 
3020 	kref_put(&resv->refs, resv_map_release);
3021 
3022 	if (reserve) {
3023 		/*
3024 		 * Decrement reserve counts.  The global reserve count may be
3025 		 * adjusted if the subpool has a minimum size.
3026 		 */
3027 		gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
3028 		hugetlb_acct_memory(h, -gbl_reserve);
3029 	}
3030 }
3031 
3032 /*
3033  * We cannot handle pagefaults against hugetlb pages at all.  They cause
3034  * handle_mm_fault() to try to instantiate regular-sized pages in the
3035  * hugegpage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
3036  * this far.
3037  */
3038 static int hugetlb_vm_op_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3039 {
3040 	BUG();
3041 	return 0;
3042 }
3043 
3044 const struct vm_operations_struct hugetlb_vm_ops = {
3045 	.fault = hugetlb_vm_op_fault,
3046 	.open = hugetlb_vm_op_open,
3047 	.close = hugetlb_vm_op_close,
3048 };
3049 
3050 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
3051 				int writable)
3052 {
3053 	pte_t entry;
3054 
3055 	if (writable) {
3056 		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
3057 					 vma->vm_page_prot)));
3058 	} else {
3059 		entry = huge_pte_wrprotect(mk_huge_pte(page,
3060 					   vma->vm_page_prot));
3061 	}
3062 	entry = pte_mkyoung(entry);
3063 	entry = pte_mkhuge(entry);
3064 	entry = arch_make_huge_pte(entry, vma, page, writable);
3065 
3066 	return entry;
3067 }
3068 
3069 static void set_huge_ptep_writable(struct vm_area_struct *vma,
3070 				   unsigned long address, pte_t *ptep)
3071 {
3072 	pte_t entry;
3073 
3074 	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
3075 	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
3076 		update_mmu_cache(vma, address, ptep);
3077 }
3078 
3079 static int is_hugetlb_entry_migration(pte_t pte)
3080 {
3081 	swp_entry_t swp;
3082 
3083 	if (huge_pte_none(pte) || pte_present(pte))
3084 		return 0;
3085 	swp = pte_to_swp_entry(pte);
3086 	if (non_swap_entry(swp) && is_migration_entry(swp))
3087 		return 1;
3088 	else
3089 		return 0;
3090 }
3091 
3092 static int is_hugetlb_entry_hwpoisoned(pte_t pte)
3093 {
3094 	swp_entry_t swp;
3095 
3096 	if (huge_pte_none(pte) || pte_present(pte))
3097 		return 0;
3098 	swp = pte_to_swp_entry(pte);
3099 	if (non_swap_entry(swp) && is_hwpoison_entry(swp))
3100 		return 1;
3101 	else
3102 		return 0;
3103 }
3104 
3105 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
3106 			    struct vm_area_struct *vma)
3107 {
3108 	pte_t *src_pte, *dst_pte, entry;
3109 	struct page *ptepage;
3110 	unsigned long addr;
3111 	int cow;
3112 	struct hstate *h = hstate_vma(vma);
3113 	unsigned long sz = huge_page_size(h);
3114 	unsigned long mmun_start;	/* For mmu_notifiers */
3115 	unsigned long mmun_end;		/* For mmu_notifiers */
3116 	int ret = 0;
3117 
3118 	cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
3119 
3120 	mmun_start = vma->vm_start;
3121 	mmun_end = vma->vm_end;
3122 	if (cow)
3123 		mmu_notifier_invalidate_range_start(src, mmun_start, mmun_end);
3124 
3125 	for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
3126 		spinlock_t *src_ptl, *dst_ptl;
3127 		src_pte = huge_pte_offset(src, addr);
3128 		if (!src_pte)
3129 			continue;
3130 		dst_pte = huge_pte_alloc(dst, addr, sz);
3131 		if (!dst_pte) {
3132 			ret = -ENOMEM;
3133 			break;
3134 		}
3135 
3136 		/* If the pagetables are shared don't copy or take references */
3137 		if (dst_pte == src_pte)
3138 			continue;
3139 
3140 		dst_ptl = huge_pte_lock(h, dst, dst_pte);
3141 		src_ptl = huge_pte_lockptr(h, src, src_pte);
3142 		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
3143 		entry = huge_ptep_get(src_pte);
3144 		if (huge_pte_none(entry)) { /* skip none entry */
3145 			;
3146 		} else if (unlikely(is_hugetlb_entry_migration(entry) ||
3147 				    is_hugetlb_entry_hwpoisoned(entry))) {
3148 			swp_entry_t swp_entry = pte_to_swp_entry(entry);
3149 
3150 			if (is_write_migration_entry(swp_entry) && cow) {
3151 				/*
3152 				 * COW mappings require pages in both
3153 				 * parent and child to be set to read.
3154 				 */
3155 				make_migration_entry_read(&swp_entry);
3156 				entry = swp_entry_to_pte(swp_entry);
3157 				set_huge_pte_at(src, addr, src_pte, entry);
3158 			}
3159 			set_huge_pte_at(dst, addr, dst_pte, entry);
3160 		} else {
3161 			if (cow) {
3162 				huge_ptep_set_wrprotect(src, addr, src_pte);
3163 				mmu_notifier_invalidate_range(src, mmun_start,
3164 								   mmun_end);
3165 			}
3166 			entry = huge_ptep_get(src_pte);
3167 			ptepage = pte_page(entry);
3168 			get_page(ptepage);
3169 			page_dup_rmap(ptepage, true);
3170 			set_huge_pte_at(dst, addr, dst_pte, entry);
3171 			hugetlb_count_add(pages_per_huge_page(h), dst);
3172 		}
3173 		spin_unlock(src_ptl);
3174 		spin_unlock(dst_ptl);
3175 	}
3176 
3177 	if (cow)
3178 		mmu_notifier_invalidate_range_end(src, mmun_start, mmun_end);
3179 
3180 	return ret;
3181 }
3182 
3183 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
3184 			    unsigned long start, unsigned long end,
3185 			    struct page *ref_page)
3186 {
3187 	struct mm_struct *mm = vma->vm_mm;
3188 	unsigned long address;
3189 	pte_t *ptep;
3190 	pte_t pte;
3191 	spinlock_t *ptl;
3192 	struct page *page;
3193 	struct hstate *h = hstate_vma(vma);
3194 	unsigned long sz = huge_page_size(h);
3195 	const unsigned long mmun_start = start;	/* For mmu_notifiers */
3196 	const unsigned long mmun_end   = end;	/* For mmu_notifiers */
3197 
3198 	WARN_ON(!is_vm_hugetlb_page(vma));
3199 	BUG_ON(start & ~huge_page_mask(h));
3200 	BUG_ON(end & ~huge_page_mask(h));
3201 
3202 	tlb_start_vma(tlb, vma);
3203 	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3204 	address = start;
3205 	for (; address < end; address += sz) {
3206 		ptep = huge_pte_offset(mm, address);
3207 		if (!ptep)
3208 			continue;
3209 
3210 		ptl = huge_pte_lock(h, mm, ptep);
3211 		if (huge_pmd_unshare(mm, &address, ptep)) {
3212 			spin_unlock(ptl);
3213 			continue;
3214 		}
3215 
3216 		pte = huge_ptep_get(ptep);
3217 		if (huge_pte_none(pte)) {
3218 			spin_unlock(ptl);
3219 			continue;
3220 		}
3221 
3222 		/*
3223 		 * Migrating hugepage or HWPoisoned hugepage is already
3224 		 * unmapped and its refcount is dropped, so just clear pte here.
3225 		 */
3226 		if (unlikely(!pte_present(pte))) {
3227 			huge_pte_clear(mm, address, ptep);
3228 			spin_unlock(ptl);
3229 			continue;
3230 		}
3231 
3232 		page = pte_page(pte);
3233 		/*
3234 		 * If a reference page is supplied, it is because a specific
3235 		 * page is being unmapped, not a range. Ensure the page we
3236 		 * are about to unmap is the actual page of interest.
3237 		 */
3238 		if (ref_page) {
3239 			if (page != ref_page) {
3240 				spin_unlock(ptl);
3241 				continue;
3242 			}
3243 			/*
3244 			 * Mark the VMA as having unmapped its page so that
3245 			 * future faults in this VMA will fail rather than
3246 			 * looking like data was lost
3247 			 */
3248 			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
3249 		}
3250 
3251 		pte = huge_ptep_get_and_clear(mm, address, ptep);
3252 		tlb_remove_tlb_entry(tlb, ptep, address);
3253 		if (huge_pte_dirty(pte))
3254 			set_page_dirty(page);
3255 
3256 		hugetlb_count_sub(pages_per_huge_page(h), mm);
3257 		page_remove_rmap(page, true);
3258 
3259 		spin_unlock(ptl);
3260 		tlb_remove_page_size(tlb, page, huge_page_size(h));
3261 		/*
3262 		 * Bail out after unmapping reference page if supplied
3263 		 */
3264 		if (ref_page)
3265 			break;
3266 	}
3267 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3268 	tlb_end_vma(tlb, vma);
3269 }
3270 
3271 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
3272 			  struct vm_area_struct *vma, unsigned long start,
3273 			  unsigned long end, struct page *ref_page)
3274 {
3275 	__unmap_hugepage_range(tlb, vma, start, end, ref_page);
3276 
3277 	/*
3278 	 * Clear this flag so that x86's huge_pmd_share page_table_shareable
3279 	 * test will fail on a vma being torn down, and not grab a page table
3280 	 * on its way out.  We're lucky that the flag has such an appropriate
3281 	 * name, and can in fact be safely cleared here. We could clear it
3282 	 * before the __unmap_hugepage_range above, but all that's necessary
3283 	 * is to clear it before releasing the i_mmap_rwsem. This works
3284 	 * because in the context this is called, the VMA is about to be
3285 	 * destroyed and the i_mmap_rwsem is held.
3286 	 */
3287 	vma->vm_flags &= ~VM_MAYSHARE;
3288 }
3289 
3290 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
3291 			  unsigned long end, struct page *ref_page)
3292 {
3293 	struct mm_struct *mm;
3294 	struct mmu_gather tlb;
3295 
3296 	mm = vma->vm_mm;
3297 
3298 	tlb_gather_mmu(&tlb, mm, start, end);
3299 	__unmap_hugepage_range(&tlb, vma, start, end, ref_page);
3300 	tlb_finish_mmu(&tlb, start, end);
3301 }
3302 
3303 /*
3304  * This is called when the original mapper is failing to COW a MAP_PRIVATE
3305  * mappping it owns the reserve page for. The intention is to unmap the page
3306  * from other VMAs and let the children be SIGKILLed if they are faulting the
3307  * same region.
3308  */
3309 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
3310 			      struct page *page, unsigned long address)
3311 {
3312 	struct hstate *h = hstate_vma(vma);
3313 	struct vm_area_struct *iter_vma;
3314 	struct address_space *mapping;
3315 	pgoff_t pgoff;
3316 
3317 	/*
3318 	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
3319 	 * from page cache lookup which is in HPAGE_SIZE units.
3320 	 */
3321 	address = address & huge_page_mask(h);
3322 	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
3323 			vma->vm_pgoff;
3324 	mapping = vma->vm_file->f_mapping;
3325 
3326 	/*
3327 	 * Take the mapping lock for the duration of the table walk. As
3328 	 * this mapping should be shared between all the VMAs,
3329 	 * __unmap_hugepage_range() is called as the lock is already held
3330 	 */
3331 	i_mmap_lock_write(mapping);
3332 	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
3333 		/* Do not unmap the current VMA */
3334 		if (iter_vma == vma)
3335 			continue;
3336 
3337 		/*
3338 		 * Shared VMAs have their own reserves and do not affect
3339 		 * MAP_PRIVATE accounting but it is possible that a shared
3340 		 * VMA is using the same page so check and skip such VMAs.
3341 		 */
3342 		if (iter_vma->vm_flags & VM_MAYSHARE)
3343 			continue;
3344 
3345 		/*
3346 		 * Unmap the page from other VMAs without their own reserves.
3347 		 * They get marked to be SIGKILLed if they fault in these
3348 		 * areas. This is because a future no-page fault on this VMA
3349 		 * could insert a zeroed page instead of the data existing
3350 		 * from the time of fork. This would look like data corruption
3351 		 */
3352 		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
3353 			unmap_hugepage_range(iter_vma, address,
3354 					     address + huge_page_size(h), page);
3355 	}
3356 	i_mmap_unlock_write(mapping);
3357 }
3358 
3359 /*
3360  * Hugetlb_cow() should be called with page lock of the original hugepage held.
3361  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
3362  * cannot race with other handlers or page migration.
3363  * Keep the pte_same checks anyway to make transition from the mutex easier.
3364  */
3365 static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
3366 			unsigned long address, pte_t *ptep, pte_t pte,
3367 			struct page *pagecache_page, spinlock_t *ptl)
3368 {
3369 	struct hstate *h = hstate_vma(vma);
3370 	struct page *old_page, *new_page;
3371 	int ret = 0, outside_reserve = 0;
3372 	unsigned long mmun_start;	/* For mmu_notifiers */
3373 	unsigned long mmun_end;		/* For mmu_notifiers */
3374 
3375 	old_page = pte_page(pte);
3376 
3377 retry_avoidcopy:
3378 	/* If no-one else is actually using this page, avoid the copy
3379 	 * and just make the page writable */
3380 	if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
3381 		page_move_anon_rmap(old_page, vma);
3382 		set_huge_ptep_writable(vma, address, ptep);
3383 		return 0;
3384 	}
3385 
3386 	/*
3387 	 * If the process that created a MAP_PRIVATE mapping is about to
3388 	 * perform a COW due to a shared page count, attempt to satisfy
3389 	 * the allocation without using the existing reserves. The pagecache
3390 	 * page is used to determine if the reserve at this address was
3391 	 * consumed or not. If reserves were used, a partial faulted mapping
3392 	 * at the time of fork() could consume its reserves on COW instead
3393 	 * of the full address range.
3394 	 */
3395 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
3396 			old_page != pagecache_page)
3397 		outside_reserve = 1;
3398 
3399 	get_page(old_page);
3400 
3401 	/*
3402 	 * Drop page table lock as buddy allocator may be called. It will
3403 	 * be acquired again before returning to the caller, as expected.
3404 	 */
3405 	spin_unlock(ptl);
3406 	new_page = alloc_huge_page(vma, address, outside_reserve);
3407 
3408 	if (IS_ERR(new_page)) {
3409 		/*
3410 		 * If a process owning a MAP_PRIVATE mapping fails to COW,
3411 		 * it is due to references held by a child and an insufficient
3412 		 * huge page pool. To guarantee the original mappers
3413 		 * reliability, unmap the page from child processes. The child
3414 		 * may get SIGKILLed if it later faults.
3415 		 */
3416 		if (outside_reserve) {
3417 			put_page(old_page);
3418 			BUG_ON(huge_pte_none(pte));
3419 			unmap_ref_private(mm, vma, old_page, address);
3420 			BUG_ON(huge_pte_none(pte));
3421 			spin_lock(ptl);
3422 			ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3423 			if (likely(ptep &&
3424 				   pte_same(huge_ptep_get(ptep), pte)))
3425 				goto retry_avoidcopy;
3426 			/*
3427 			 * race occurs while re-acquiring page table
3428 			 * lock, and our job is done.
3429 			 */
3430 			return 0;
3431 		}
3432 
3433 		ret = (PTR_ERR(new_page) == -ENOMEM) ?
3434 			VM_FAULT_OOM : VM_FAULT_SIGBUS;
3435 		goto out_release_old;
3436 	}
3437 
3438 	/*
3439 	 * When the original hugepage is shared one, it does not have
3440 	 * anon_vma prepared.
3441 	 */
3442 	if (unlikely(anon_vma_prepare(vma))) {
3443 		ret = VM_FAULT_OOM;
3444 		goto out_release_all;
3445 	}
3446 
3447 	copy_user_huge_page(new_page, old_page, address, vma,
3448 			    pages_per_huge_page(h));
3449 	__SetPageUptodate(new_page);
3450 	set_page_huge_active(new_page);
3451 
3452 	mmun_start = address & huge_page_mask(h);
3453 	mmun_end = mmun_start + huge_page_size(h);
3454 	mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
3455 
3456 	/*
3457 	 * Retake the page table lock to check for racing updates
3458 	 * before the page tables are altered
3459 	 */
3460 	spin_lock(ptl);
3461 	ptep = huge_pte_offset(mm, address & huge_page_mask(h));
3462 	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
3463 		ClearPagePrivate(new_page);
3464 
3465 		/* Break COW */
3466 		huge_ptep_clear_flush(vma, address, ptep);
3467 		mmu_notifier_invalidate_range(mm, mmun_start, mmun_end);
3468 		set_huge_pte_at(mm, address, ptep,
3469 				make_huge_pte(vma, new_page, 1));
3470 		page_remove_rmap(old_page, true);
3471 		hugepage_add_new_anon_rmap(new_page, vma, address);
3472 		/* Make the old page be freed below */
3473 		new_page = old_page;
3474 	}
3475 	spin_unlock(ptl);
3476 	mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
3477 out_release_all:
3478 	put_page(new_page);
3479 out_release_old:
3480 	put_page(old_page);
3481 
3482 	spin_lock(ptl); /* Caller expects lock to be held */
3483 	return ret;
3484 }
3485 
3486 /* Return the pagecache page at a given address within a VMA */
3487 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
3488 			struct vm_area_struct *vma, unsigned long address)
3489 {
3490 	struct address_space *mapping;
3491 	pgoff_t idx;
3492 
3493 	mapping = vma->vm_file->f_mapping;
3494 	idx = vma_hugecache_offset(h, vma, address);
3495 
3496 	return find_lock_page(mapping, idx);
3497 }
3498 
3499 /*
3500  * Return whether there is a pagecache page to back given address within VMA.
3501  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
3502  */
3503 static bool hugetlbfs_pagecache_present(struct hstate *h,
3504 			struct vm_area_struct *vma, unsigned long address)
3505 {
3506 	struct address_space *mapping;
3507 	pgoff_t idx;
3508 	struct page *page;
3509 
3510 	mapping = vma->vm_file->f_mapping;
3511 	idx = vma_hugecache_offset(h, vma, address);
3512 
3513 	page = find_get_page(mapping, idx);
3514 	if (page)
3515 		put_page(page);
3516 	return page != NULL;
3517 }
3518 
3519 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
3520 			   pgoff_t idx)
3521 {
3522 	struct inode *inode = mapping->host;
3523 	struct hstate *h = hstate_inode(inode);
3524 	int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
3525 
3526 	if (err)
3527 		return err;
3528 	ClearPagePrivate(page);
3529 
3530 	spin_lock(&inode->i_lock);
3531 	inode->i_blocks += blocks_per_huge_page(h);
3532 	spin_unlock(&inode->i_lock);
3533 	return 0;
3534 }
3535 
3536 static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
3537 			   struct address_space *mapping, pgoff_t idx,
3538 			   unsigned long address, pte_t *ptep, unsigned int flags)
3539 {
3540 	struct hstate *h = hstate_vma(vma);
3541 	int ret = VM_FAULT_SIGBUS;
3542 	int anon_rmap = 0;
3543 	unsigned long size;
3544 	struct page *page;
3545 	pte_t new_pte;
3546 	spinlock_t *ptl;
3547 
3548 	/*
3549 	 * Currently, we are forced to kill the process in the event the
3550 	 * original mapper has unmapped pages from the child due to a failed
3551 	 * COW. Warn that such a situation has occurred as it may not be obvious
3552 	 */
3553 	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
3554 		pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
3555 			   current->pid);
3556 		return ret;
3557 	}
3558 
3559 	/*
3560 	 * Use page lock to guard against racing truncation
3561 	 * before we get page_table_lock.
3562 	 */
3563 retry:
3564 	page = find_lock_page(mapping, idx);
3565 	if (!page) {
3566 		size = i_size_read(mapping->host) >> huge_page_shift(h);
3567 		if (idx >= size)
3568 			goto out;
3569 		page = alloc_huge_page(vma, address, 0);
3570 		if (IS_ERR(page)) {
3571 			ret = PTR_ERR(page);
3572 			if (ret == -ENOMEM)
3573 				ret = VM_FAULT_OOM;
3574 			else
3575 				ret = VM_FAULT_SIGBUS;
3576 			goto out;
3577 		}
3578 		clear_huge_page(page, address, pages_per_huge_page(h));
3579 		__SetPageUptodate(page);
3580 		set_page_huge_active(page);
3581 
3582 		if (vma->vm_flags & VM_MAYSHARE) {
3583 			int err = huge_add_to_page_cache(page, mapping, idx);
3584 			if (err) {
3585 				put_page(page);
3586 				if (err == -EEXIST)
3587 					goto retry;
3588 				goto out;
3589 			}
3590 		} else {
3591 			lock_page(page);
3592 			if (unlikely(anon_vma_prepare(vma))) {
3593 				ret = VM_FAULT_OOM;
3594 				goto backout_unlocked;
3595 			}
3596 			anon_rmap = 1;
3597 		}
3598 	} else {
3599 		/*
3600 		 * If memory error occurs between mmap() and fault, some process
3601 		 * don't have hwpoisoned swap entry for errored virtual address.
3602 		 * So we need to block hugepage fault by PG_hwpoison bit check.
3603 		 */
3604 		if (unlikely(PageHWPoison(page))) {
3605 			ret = VM_FAULT_HWPOISON |
3606 				VM_FAULT_SET_HINDEX(hstate_index(h));
3607 			goto backout_unlocked;
3608 		}
3609 	}
3610 
3611 	/*
3612 	 * If we are going to COW a private mapping later, we examine the
3613 	 * pending reservations for this page now. This will ensure that
3614 	 * any allocations necessary to record that reservation occur outside
3615 	 * the spinlock.
3616 	 */
3617 	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3618 		if (vma_needs_reservation(h, vma, address) < 0) {
3619 			ret = VM_FAULT_OOM;
3620 			goto backout_unlocked;
3621 		}
3622 		/* Just decrements count, does not deallocate */
3623 		vma_end_reservation(h, vma, address);
3624 	}
3625 
3626 	ptl = huge_pte_lockptr(h, mm, ptep);
3627 	spin_lock(ptl);
3628 	size = i_size_read(mapping->host) >> huge_page_shift(h);
3629 	if (idx >= size)
3630 		goto backout;
3631 
3632 	ret = 0;
3633 	if (!huge_pte_none(huge_ptep_get(ptep)))
3634 		goto backout;
3635 
3636 	if (anon_rmap) {
3637 		ClearPagePrivate(page);
3638 		hugepage_add_new_anon_rmap(page, vma, address);
3639 	} else
3640 		page_dup_rmap(page, true);
3641 	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
3642 				&& (vma->vm_flags & VM_SHARED)));
3643 	set_huge_pte_at(mm, address, ptep, new_pte);
3644 
3645 	hugetlb_count_add(pages_per_huge_page(h), mm);
3646 	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
3647 		/* Optimization, do the COW without a second fault */
3648 		ret = hugetlb_cow(mm, vma, address, ptep, new_pte, page, ptl);
3649 	}
3650 
3651 	spin_unlock(ptl);
3652 	unlock_page(page);
3653 out:
3654 	return ret;
3655 
3656 backout:
3657 	spin_unlock(ptl);
3658 backout_unlocked:
3659 	unlock_page(page);
3660 	put_page(page);
3661 	goto out;
3662 }
3663 
3664 #ifdef CONFIG_SMP
3665 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3666 			    struct vm_area_struct *vma,
3667 			    struct address_space *mapping,
3668 			    pgoff_t idx, unsigned long address)
3669 {
3670 	unsigned long key[2];
3671 	u32 hash;
3672 
3673 	if (vma->vm_flags & VM_SHARED) {
3674 		key[0] = (unsigned long) mapping;
3675 		key[1] = idx;
3676 	} else {
3677 		key[0] = (unsigned long) mm;
3678 		key[1] = address >> huge_page_shift(h);
3679 	}
3680 
3681 	hash = jhash2((u32 *)&key, sizeof(key)/sizeof(u32), 0);
3682 
3683 	return hash & (num_fault_mutexes - 1);
3684 }
3685 #else
3686 /*
3687  * For uniprocesor systems we always use a single mutex, so just
3688  * return 0 and avoid the hashing overhead.
3689  */
3690 u32 hugetlb_fault_mutex_hash(struct hstate *h, struct mm_struct *mm,
3691 			    struct vm_area_struct *vma,
3692 			    struct address_space *mapping,
3693 			    pgoff_t idx, unsigned long address)
3694 {
3695 	return 0;
3696 }
3697 #endif
3698 
3699 int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
3700 			unsigned long address, unsigned int flags)
3701 {
3702 	pte_t *ptep, entry;
3703 	spinlock_t *ptl;
3704 	int ret;
3705 	u32 hash;
3706 	pgoff_t idx;
3707 	struct page *page = NULL;
3708 	struct page *pagecache_page = NULL;
3709 	struct hstate *h = hstate_vma(vma);
3710 	struct address_space *mapping;
3711 	int need_wait_lock = 0;
3712 
3713 	address &= huge_page_mask(h);
3714 
3715 	ptep = huge_pte_offset(mm, address);
3716 	if (ptep) {
3717 		entry = huge_ptep_get(ptep);
3718 		if (unlikely(is_hugetlb_entry_migration(entry))) {
3719 			migration_entry_wait_huge(vma, mm, ptep);
3720 			return 0;
3721 		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
3722 			return VM_FAULT_HWPOISON_LARGE |
3723 				VM_FAULT_SET_HINDEX(hstate_index(h));
3724 	} else {
3725 		ptep = huge_pte_alloc(mm, address, huge_page_size(h));
3726 		if (!ptep)
3727 			return VM_FAULT_OOM;
3728 	}
3729 
3730 	mapping = vma->vm_file->f_mapping;
3731 	idx = vma_hugecache_offset(h, vma, address);
3732 
3733 	/*
3734 	 * Serialize hugepage allocation and instantiation, so that we don't
3735 	 * get spurious allocation failures if two CPUs race to instantiate
3736 	 * the same page in the page cache.
3737 	 */
3738 	hash = hugetlb_fault_mutex_hash(h, mm, vma, mapping, idx, address);
3739 	mutex_lock(&hugetlb_fault_mutex_table[hash]);
3740 
3741 	entry = huge_ptep_get(ptep);
3742 	if (huge_pte_none(entry)) {
3743 		ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
3744 		goto out_mutex;
3745 	}
3746 
3747 	ret = 0;
3748 
3749 	/*
3750 	 * entry could be a migration/hwpoison entry at this point, so this
3751 	 * check prevents the kernel from going below assuming that we have
3752 	 * a active hugepage in pagecache. This goto expects the 2nd page fault,
3753 	 * and is_hugetlb_entry_(migration|hwpoisoned) check will properly
3754 	 * handle it.
3755 	 */
3756 	if (!pte_present(entry))
3757 		goto out_mutex;
3758 
3759 	/*
3760 	 * If we are going to COW the mapping later, we examine the pending
3761 	 * reservations for this page now. This will ensure that any
3762 	 * allocations necessary to record that reservation occur outside the
3763 	 * spinlock. For private mappings, we also lookup the pagecache
3764 	 * page now as it is used to determine if a reservation has been
3765 	 * consumed.
3766 	 */
3767 	if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
3768 		if (vma_needs_reservation(h, vma, address) < 0) {
3769 			ret = VM_FAULT_OOM;
3770 			goto out_mutex;
3771 		}
3772 		/* Just decrements count, does not deallocate */
3773 		vma_end_reservation(h, vma, address);
3774 
3775 		if (!(vma->vm_flags & VM_MAYSHARE))
3776 			pagecache_page = hugetlbfs_pagecache_page(h,
3777 								vma, address);
3778 	}
3779 
3780 	ptl = huge_pte_lock(h, mm, ptep);
3781 
3782 	/* Check for a racing update before calling hugetlb_cow */
3783 	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
3784 		goto out_ptl;
3785 
3786 	/*
3787 	 * hugetlb_cow() requires page locks of pte_page(entry) and
3788 	 * pagecache_page, so here we need take the former one
3789 	 * when page != pagecache_page or !pagecache_page.
3790 	 */
3791 	page = pte_page(entry);
3792 	if (page != pagecache_page)
3793 		if (!trylock_page(page)) {
3794 			need_wait_lock = 1;
3795 			goto out_ptl;
3796 		}
3797 
3798 	get_page(page);
3799 
3800 	if (flags & FAULT_FLAG_WRITE) {
3801 		if (!huge_pte_write(entry)) {
3802 			ret = hugetlb_cow(mm, vma, address, ptep, entry,
3803 					pagecache_page, ptl);
3804 			goto out_put_page;
3805 		}
3806 		entry = huge_pte_mkdirty(entry);
3807 	}
3808 	entry = pte_mkyoung(entry);
3809 	if (huge_ptep_set_access_flags(vma, address, ptep, entry,
3810 						flags & FAULT_FLAG_WRITE))
3811 		update_mmu_cache(vma, address, ptep);
3812 out_put_page:
3813 	if (page != pagecache_page)
3814 		unlock_page(page);
3815 	put_page(page);
3816 out_ptl:
3817 	spin_unlock(ptl);
3818 
3819 	if (pagecache_page) {
3820 		unlock_page(pagecache_page);
3821 		put_page(pagecache_page);
3822 	}
3823 out_mutex:
3824 	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
3825 	/*
3826 	 * Generally it's safe to hold refcount during waiting page lock. But
3827 	 * here we just wait to defer the next page fault to avoid busy loop and
3828 	 * the page is not used after unlocked before returning from the current
3829 	 * page fault. So we are safe from accessing freed page, even if we wait
3830 	 * here without taking refcount.
3831 	 */
3832 	if (need_wait_lock)
3833 		wait_on_page_locked(page);
3834 	return ret;
3835 }
3836 
3837 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
3838 			 struct page **pages, struct vm_area_struct **vmas,
3839 			 unsigned long *position, unsigned long *nr_pages,
3840 			 long i, unsigned int flags)
3841 {
3842 	unsigned long pfn_offset;
3843 	unsigned long vaddr = *position;
3844 	unsigned long remainder = *nr_pages;
3845 	struct hstate *h = hstate_vma(vma);
3846 
3847 	while (vaddr < vma->vm_end && remainder) {
3848 		pte_t *pte;
3849 		spinlock_t *ptl = NULL;
3850 		int absent;
3851 		struct page *page;
3852 
3853 		/*
3854 		 * If we have a pending SIGKILL, don't keep faulting pages and
3855 		 * potentially allocating memory.
3856 		 */
3857 		if (unlikely(fatal_signal_pending(current))) {
3858 			remainder = 0;
3859 			break;
3860 		}
3861 
3862 		/*
3863 		 * Some archs (sparc64, sh*) have multiple pte_ts to
3864 		 * each hugepage.  We have to make sure we get the
3865 		 * first, for the page indexing below to work.
3866 		 *
3867 		 * Note that page table lock is not held when pte is null.
3868 		 */
3869 		pte = huge_pte_offset(mm, vaddr & huge_page_mask(h));
3870 		if (pte)
3871 			ptl = huge_pte_lock(h, mm, pte);
3872 		absent = !pte || huge_pte_none(huge_ptep_get(pte));
3873 
3874 		/*
3875 		 * When coredumping, it suits get_dump_page if we just return
3876 		 * an error where there's an empty slot with no huge pagecache
3877 		 * to back it.  This way, we avoid allocating a hugepage, and
3878 		 * the sparse dumpfile avoids allocating disk blocks, but its
3879 		 * huge holes still show up with zeroes where they need to be.
3880 		 */
3881 		if (absent && (flags & FOLL_DUMP) &&
3882 		    !hugetlbfs_pagecache_present(h, vma, vaddr)) {
3883 			if (pte)
3884 				spin_unlock(ptl);
3885 			remainder = 0;
3886 			break;
3887 		}
3888 
3889 		/*
3890 		 * We need call hugetlb_fault for both hugepages under migration
3891 		 * (in which case hugetlb_fault waits for the migration,) and
3892 		 * hwpoisoned hugepages (in which case we need to prevent the
3893 		 * caller from accessing to them.) In order to do this, we use
3894 		 * here is_swap_pte instead of is_hugetlb_entry_migration and
3895 		 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
3896 		 * both cases, and because we can't follow correct pages
3897 		 * directly from any kind of swap entries.
3898 		 */
3899 		if (absent || is_swap_pte(huge_ptep_get(pte)) ||
3900 		    ((flags & FOLL_WRITE) &&
3901 		      !huge_pte_write(huge_ptep_get(pte)))) {
3902 			int ret;
3903 
3904 			if (pte)
3905 				spin_unlock(ptl);
3906 			ret = hugetlb_fault(mm, vma, vaddr,
3907 				(flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
3908 			if (!(ret & VM_FAULT_ERROR))
3909 				continue;
3910 
3911 			remainder = 0;
3912 			break;
3913 		}
3914 
3915 		pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
3916 		page = pte_page(huge_ptep_get(pte));
3917 same_page:
3918 		if (pages) {
3919 			pages[i] = mem_map_offset(page, pfn_offset);
3920 			get_page(pages[i]);
3921 		}
3922 
3923 		if (vmas)
3924 			vmas[i] = vma;
3925 
3926 		vaddr += PAGE_SIZE;
3927 		++pfn_offset;
3928 		--remainder;
3929 		++i;
3930 		if (vaddr < vma->vm_end && remainder &&
3931 				pfn_offset < pages_per_huge_page(h)) {
3932 			/*
3933 			 * We use pfn_offset to avoid touching the pageframes
3934 			 * of this compound page.
3935 			 */
3936 			goto same_page;
3937 		}
3938 		spin_unlock(ptl);
3939 	}
3940 	*nr_pages = remainder;
3941 	*position = vaddr;
3942 
3943 	return i ? i : -EFAULT;
3944 }
3945 
3946 #ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
3947 /*
3948  * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
3949  * implement this.
3950  */
3951 #define flush_hugetlb_tlb_range(vma, addr, end)	flush_tlb_range(vma, addr, end)
3952 #endif
3953 
3954 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
3955 		unsigned long address, unsigned long end, pgprot_t newprot)
3956 {
3957 	struct mm_struct *mm = vma->vm_mm;
3958 	unsigned long start = address;
3959 	pte_t *ptep;
3960 	pte_t pte;
3961 	struct hstate *h = hstate_vma(vma);
3962 	unsigned long pages = 0;
3963 
3964 	BUG_ON(address >= end);
3965 	flush_cache_range(vma, address, end);
3966 
3967 	mmu_notifier_invalidate_range_start(mm, start, end);
3968 	i_mmap_lock_write(vma->vm_file->f_mapping);
3969 	for (; address < end; address += huge_page_size(h)) {
3970 		spinlock_t *ptl;
3971 		ptep = huge_pte_offset(mm, address);
3972 		if (!ptep)
3973 			continue;
3974 		ptl = huge_pte_lock(h, mm, ptep);
3975 		if (huge_pmd_unshare(mm, &address, ptep)) {
3976 			pages++;
3977 			spin_unlock(ptl);
3978 			continue;
3979 		}
3980 		pte = huge_ptep_get(ptep);
3981 		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
3982 			spin_unlock(ptl);
3983 			continue;
3984 		}
3985 		if (unlikely(is_hugetlb_entry_migration(pte))) {
3986 			swp_entry_t entry = pte_to_swp_entry(pte);
3987 
3988 			if (is_write_migration_entry(entry)) {
3989 				pte_t newpte;
3990 
3991 				make_migration_entry_read(&entry);
3992 				newpte = swp_entry_to_pte(entry);
3993 				set_huge_pte_at(mm, address, ptep, newpte);
3994 				pages++;
3995 			}
3996 			spin_unlock(ptl);
3997 			continue;
3998 		}
3999 		if (!huge_pte_none(pte)) {
4000 			pte = huge_ptep_get_and_clear(mm, address, ptep);
4001 			pte = pte_mkhuge(huge_pte_modify(pte, newprot));
4002 			pte = arch_make_huge_pte(pte, vma, NULL, 0);
4003 			set_huge_pte_at(mm, address, ptep, pte);
4004 			pages++;
4005 		}
4006 		spin_unlock(ptl);
4007 	}
4008 	/*
4009 	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
4010 	 * may have cleared our pud entry and done put_page on the page table:
4011 	 * once we release i_mmap_rwsem, another task can do the final put_page
4012 	 * and that page table be reused and filled with junk.
4013 	 */
4014 	flush_hugetlb_tlb_range(vma, start, end);
4015 	mmu_notifier_invalidate_range(mm, start, end);
4016 	i_mmap_unlock_write(vma->vm_file->f_mapping);
4017 	mmu_notifier_invalidate_range_end(mm, start, end);
4018 
4019 	return pages << h->order;
4020 }
4021 
4022 int hugetlb_reserve_pages(struct inode *inode,
4023 					long from, long to,
4024 					struct vm_area_struct *vma,
4025 					vm_flags_t vm_flags)
4026 {
4027 	long ret, chg;
4028 	struct hstate *h = hstate_inode(inode);
4029 	struct hugepage_subpool *spool = subpool_inode(inode);
4030 	struct resv_map *resv_map;
4031 	long gbl_reserve;
4032 
4033 	/*
4034 	 * Only apply hugepage reservation if asked. At fault time, an
4035 	 * attempt will be made for VM_NORESERVE to allocate a page
4036 	 * without using reserves
4037 	 */
4038 	if (vm_flags & VM_NORESERVE)
4039 		return 0;
4040 
4041 	/*
4042 	 * Shared mappings base their reservation on the number of pages that
4043 	 * are already allocated on behalf of the file. Private mappings need
4044 	 * to reserve the full area even if read-only as mprotect() may be
4045 	 * called to make the mapping read-write. Assume !vma is a shm mapping
4046 	 */
4047 	if (!vma || vma->vm_flags & VM_MAYSHARE) {
4048 		resv_map = inode_resv_map(inode);
4049 
4050 		chg = region_chg(resv_map, from, to);
4051 
4052 	} else {
4053 		resv_map = resv_map_alloc();
4054 		if (!resv_map)
4055 			return -ENOMEM;
4056 
4057 		chg = to - from;
4058 
4059 		set_vma_resv_map(vma, resv_map);
4060 		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
4061 	}
4062 
4063 	if (chg < 0) {
4064 		ret = chg;
4065 		goto out_err;
4066 	}
4067 
4068 	/*
4069 	 * There must be enough pages in the subpool for the mapping. If
4070 	 * the subpool has a minimum size, there may be some global
4071 	 * reservations already in place (gbl_reserve).
4072 	 */
4073 	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
4074 	if (gbl_reserve < 0) {
4075 		ret = -ENOSPC;
4076 		goto out_err;
4077 	}
4078 
4079 	/*
4080 	 * Check enough hugepages are available for the reservation.
4081 	 * Hand the pages back to the subpool if there are not
4082 	 */
4083 	ret = hugetlb_acct_memory(h, gbl_reserve);
4084 	if (ret < 0) {
4085 		/* put back original number of pages, chg */
4086 		(void)hugepage_subpool_put_pages(spool, chg);
4087 		goto out_err;
4088 	}
4089 
4090 	/*
4091 	 * Account for the reservations made. Shared mappings record regions
4092 	 * that have reservations as they are shared by multiple VMAs.
4093 	 * When the last VMA disappears, the region map says how much
4094 	 * the reservation was and the page cache tells how much of
4095 	 * the reservation was consumed. Private mappings are per-VMA and
4096 	 * only the consumed reservations are tracked. When the VMA
4097 	 * disappears, the original reservation is the VMA size and the
4098 	 * consumed reservations are stored in the map. Hence, nothing
4099 	 * else has to be done for private mappings here
4100 	 */
4101 	if (!vma || vma->vm_flags & VM_MAYSHARE) {
4102 		long add = region_add(resv_map, from, to);
4103 
4104 		if (unlikely(chg > add)) {
4105 			/*
4106 			 * pages in this range were added to the reserve
4107 			 * map between region_chg and region_add.  This
4108 			 * indicates a race with alloc_huge_page.  Adjust
4109 			 * the subpool and reserve counts modified above
4110 			 * based on the difference.
4111 			 */
4112 			long rsv_adjust;
4113 
4114 			rsv_adjust = hugepage_subpool_put_pages(spool,
4115 								chg - add);
4116 			hugetlb_acct_memory(h, -rsv_adjust);
4117 		}
4118 	}
4119 	return 0;
4120 out_err:
4121 	if (!vma || vma->vm_flags & VM_MAYSHARE)
4122 		region_abort(resv_map, from, to);
4123 	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4124 		kref_put(&resv_map->refs, resv_map_release);
4125 	return ret;
4126 }
4127 
4128 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
4129 								long freed)
4130 {
4131 	struct hstate *h = hstate_inode(inode);
4132 	struct resv_map *resv_map = inode_resv_map(inode);
4133 	long chg = 0;
4134 	struct hugepage_subpool *spool = subpool_inode(inode);
4135 	long gbl_reserve;
4136 
4137 	if (resv_map) {
4138 		chg = region_del(resv_map, start, end);
4139 		/*
4140 		 * region_del() can fail in the rare case where a region
4141 		 * must be split and another region descriptor can not be
4142 		 * allocated.  If end == LONG_MAX, it will not fail.
4143 		 */
4144 		if (chg < 0)
4145 			return chg;
4146 	}
4147 
4148 	spin_lock(&inode->i_lock);
4149 	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
4150 	spin_unlock(&inode->i_lock);
4151 
4152 	/*
4153 	 * If the subpool has a minimum size, the number of global
4154 	 * reservations to be released may be adjusted.
4155 	 */
4156 	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
4157 	hugetlb_acct_memory(h, -gbl_reserve);
4158 
4159 	return 0;
4160 }
4161 
4162 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
4163 static unsigned long page_table_shareable(struct vm_area_struct *svma,
4164 				struct vm_area_struct *vma,
4165 				unsigned long addr, pgoff_t idx)
4166 {
4167 	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
4168 				svma->vm_start;
4169 	unsigned long sbase = saddr & PUD_MASK;
4170 	unsigned long s_end = sbase + PUD_SIZE;
4171 
4172 	/* Allow segments to share if only one is marked locked */
4173 	unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
4174 	unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
4175 
4176 	/*
4177 	 * match the virtual addresses, permission and the alignment of the
4178 	 * page table page.
4179 	 */
4180 	if (pmd_index(addr) != pmd_index(saddr) ||
4181 	    vm_flags != svm_flags ||
4182 	    sbase < svma->vm_start || svma->vm_end < s_end)
4183 		return 0;
4184 
4185 	return saddr;
4186 }
4187 
4188 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
4189 {
4190 	unsigned long base = addr & PUD_MASK;
4191 	unsigned long end = base + PUD_SIZE;
4192 
4193 	/*
4194 	 * check on proper vm_flags and page table alignment
4195 	 */
4196 	if (vma->vm_flags & VM_MAYSHARE &&
4197 	    vma->vm_start <= base && end <= vma->vm_end)
4198 		return true;
4199 	return false;
4200 }
4201 
4202 /*
4203  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
4204  * and returns the corresponding pte. While this is not necessary for the
4205  * !shared pmd case because we can allocate the pmd later as well, it makes the
4206  * code much cleaner. pmd allocation is essential for the shared case because
4207  * pud has to be populated inside the same i_mmap_rwsem section - otherwise
4208  * racing tasks could either miss the sharing (see huge_pte_offset) or select a
4209  * bad pmd for sharing.
4210  */
4211 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4212 {
4213 	struct vm_area_struct *vma = find_vma(mm, addr);
4214 	struct address_space *mapping = vma->vm_file->f_mapping;
4215 	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
4216 			vma->vm_pgoff;
4217 	struct vm_area_struct *svma;
4218 	unsigned long saddr;
4219 	pte_t *spte = NULL;
4220 	pte_t *pte;
4221 	spinlock_t *ptl;
4222 
4223 	if (!vma_shareable(vma, addr))
4224 		return (pte_t *)pmd_alloc(mm, pud, addr);
4225 
4226 	i_mmap_lock_write(mapping);
4227 	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
4228 		if (svma == vma)
4229 			continue;
4230 
4231 		saddr = page_table_shareable(svma, vma, addr, idx);
4232 		if (saddr) {
4233 			spte = huge_pte_offset(svma->vm_mm, saddr);
4234 			if (spte) {
4235 				get_page(virt_to_page(spte));
4236 				break;
4237 			}
4238 		}
4239 	}
4240 
4241 	if (!spte)
4242 		goto out;
4243 
4244 	ptl = huge_pte_lockptr(hstate_vma(vma), mm, spte);
4245 	spin_lock(ptl);
4246 	if (pud_none(*pud)) {
4247 		pud_populate(mm, pud,
4248 				(pmd_t *)((unsigned long)spte & PAGE_MASK));
4249 		mm_inc_nr_pmds(mm);
4250 	} else {
4251 		put_page(virt_to_page(spte));
4252 	}
4253 	spin_unlock(ptl);
4254 out:
4255 	pte = (pte_t *)pmd_alloc(mm, pud, addr);
4256 	i_mmap_unlock_write(mapping);
4257 	return pte;
4258 }
4259 
4260 /*
4261  * unmap huge page backed by shared pte.
4262  *
4263  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
4264  * indicated by page_count > 1, unmap is achieved by clearing pud and
4265  * decrementing the ref count. If count == 1, the pte page is not shared.
4266  *
4267  * called with page table lock held.
4268  *
4269  * returns: 1 successfully unmapped a shared pte page
4270  *	    0 the underlying pte page is not shared, or it is the last user
4271  */
4272 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4273 {
4274 	pgd_t *pgd = pgd_offset(mm, *addr);
4275 	pud_t *pud = pud_offset(pgd, *addr);
4276 
4277 	BUG_ON(page_count(virt_to_page(ptep)) == 0);
4278 	if (page_count(virt_to_page(ptep)) == 1)
4279 		return 0;
4280 
4281 	pud_clear(pud);
4282 	put_page(virt_to_page(ptep));
4283 	mm_dec_nr_pmds(mm);
4284 	*addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
4285 	return 1;
4286 }
4287 #define want_pmd_share()	(1)
4288 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4289 pte_t *huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
4290 {
4291 	return NULL;
4292 }
4293 
4294 int huge_pmd_unshare(struct mm_struct *mm, unsigned long *addr, pte_t *ptep)
4295 {
4296 	return 0;
4297 }
4298 #define want_pmd_share()	(0)
4299 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
4300 
4301 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
4302 pte_t *huge_pte_alloc(struct mm_struct *mm,
4303 			unsigned long addr, unsigned long sz)
4304 {
4305 	pgd_t *pgd;
4306 	pud_t *pud;
4307 	pte_t *pte = NULL;
4308 
4309 	pgd = pgd_offset(mm, addr);
4310 	pud = pud_alloc(mm, pgd, addr);
4311 	if (pud) {
4312 		if (sz == PUD_SIZE) {
4313 			pte = (pte_t *)pud;
4314 		} else {
4315 			BUG_ON(sz != PMD_SIZE);
4316 			if (want_pmd_share() && pud_none(*pud))
4317 				pte = huge_pmd_share(mm, addr, pud);
4318 			else
4319 				pte = (pte_t *)pmd_alloc(mm, pud, addr);
4320 		}
4321 	}
4322 	BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
4323 
4324 	return pte;
4325 }
4326 
4327 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
4328 {
4329 	pgd_t *pgd;
4330 	pud_t *pud;
4331 	pmd_t *pmd = NULL;
4332 
4333 	pgd = pgd_offset(mm, addr);
4334 	if (pgd_present(*pgd)) {
4335 		pud = pud_offset(pgd, addr);
4336 		if (pud_present(*pud)) {
4337 			if (pud_huge(*pud))
4338 				return (pte_t *)pud;
4339 			pmd = pmd_offset(pud, addr);
4340 		}
4341 	}
4342 	return (pte_t *) pmd;
4343 }
4344 
4345 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
4346 
4347 /*
4348  * These functions are overwritable if your architecture needs its own
4349  * behavior.
4350  */
4351 struct page * __weak
4352 follow_huge_addr(struct mm_struct *mm, unsigned long address,
4353 			      int write)
4354 {
4355 	return ERR_PTR(-EINVAL);
4356 }
4357 
4358 struct page * __weak
4359 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
4360 		pmd_t *pmd, int flags)
4361 {
4362 	struct page *page = NULL;
4363 	spinlock_t *ptl;
4364 retry:
4365 	ptl = pmd_lockptr(mm, pmd);
4366 	spin_lock(ptl);
4367 	/*
4368 	 * make sure that the address range covered by this pmd is not
4369 	 * unmapped from other threads.
4370 	 */
4371 	if (!pmd_huge(*pmd))
4372 		goto out;
4373 	if (pmd_present(*pmd)) {
4374 		page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
4375 		if (flags & FOLL_GET)
4376 			get_page(page);
4377 	} else {
4378 		if (is_hugetlb_entry_migration(huge_ptep_get((pte_t *)pmd))) {
4379 			spin_unlock(ptl);
4380 			__migration_entry_wait(mm, (pte_t *)pmd, ptl);
4381 			goto retry;
4382 		}
4383 		/*
4384 		 * hwpoisoned entry is treated as no_page_table in
4385 		 * follow_page_mask().
4386 		 */
4387 	}
4388 out:
4389 	spin_unlock(ptl);
4390 	return page;
4391 }
4392 
4393 struct page * __weak
4394 follow_huge_pud(struct mm_struct *mm, unsigned long address,
4395 		pud_t *pud, int flags)
4396 {
4397 	if (flags & FOLL_GET)
4398 		return NULL;
4399 
4400 	return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
4401 }
4402 
4403 #ifdef CONFIG_MEMORY_FAILURE
4404 
4405 /*
4406  * This function is called from memory failure code.
4407  */
4408 int dequeue_hwpoisoned_huge_page(struct page *hpage)
4409 {
4410 	struct hstate *h = page_hstate(hpage);
4411 	int nid = page_to_nid(hpage);
4412 	int ret = -EBUSY;
4413 
4414 	spin_lock(&hugetlb_lock);
4415 	/*
4416 	 * Just checking !page_huge_active is not enough, because that could be
4417 	 * an isolated/hwpoisoned hugepage (which have >0 refcount).
4418 	 */
4419 	if (!page_huge_active(hpage) && !page_count(hpage)) {
4420 		/*
4421 		 * Hwpoisoned hugepage isn't linked to activelist or freelist,
4422 		 * but dangling hpage->lru can trigger list-debug warnings
4423 		 * (this happens when we call unpoison_memory() on it),
4424 		 * so let it point to itself with list_del_init().
4425 		 */
4426 		list_del_init(&hpage->lru);
4427 		set_page_refcounted(hpage);
4428 		h->free_huge_pages--;
4429 		h->free_huge_pages_node[nid]--;
4430 		ret = 0;
4431 	}
4432 	spin_unlock(&hugetlb_lock);
4433 	return ret;
4434 }
4435 #endif
4436 
4437 bool isolate_huge_page(struct page *page, struct list_head *list)
4438 {
4439 	bool ret = true;
4440 
4441 	VM_BUG_ON_PAGE(!PageHead(page), page);
4442 	spin_lock(&hugetlb_lock);
4443 	if (!page_huge_active(page) || !get_page_unless_zero(page)) {
4444 		ret = false;
4445 		goto unlock;
4446 	}
4447 	clear_page_huge_active(page);
4448 	list_move_tail(&page->lru, list);
4449 unlock:
4450 	spin_unlock(&hugetlb_lock);
4451 	return ret;
4452 }
4453 
4454 void putback_active_hugepage(struct page *page)
4455 {
4456 	VM_BUG_ON_PAGE(!PageHead(page), page);
4457 	spin_lock(&hugetlb_lock);
4458 	set_page_huge_active(page);
4459 	list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
4460 	spin_unlock(&hugetlb_lock);
4461 	put_page(page);
4462 }
4463