xref: /openbmc/linux/mm/hugetlb.c (revision 1d7a0395)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Generic hugetlb support.
4  * (C) Nadia Yvette Chambers, April 2004
5  */
6 #include <linux/list.h>
7 #include <linux/init.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/compiler.h>
17 #include <linux/cpuset.h>
18 #include <linux/mutex.h>
19 #include <linux/memblock.h>
20 #include <linux/sysfs.h>
21 #include <linux/slab.h>
22 #include <linux/sched/mm.h>
23 #include <linux/mmdebug.h>
24 #include <linux/sched/signal.h>
25 #include <linux/rmap.h>
26 #include <linux/string_helpers.h>
27 #include <linux/swap.h>
28 #include <linux/swapops.h>
29 #include <linux/jhash.h>
30 #include <linux/numa.h>
31 #include <linux/llist.h>
32 #include <linux/cma.h>
33 #include <linux/migrate.h>
34 
35 #include <asm/page.h>
36 #include <asm/pgalloc.h>
37 #include <asm/tlb.h>
38 
39 #include <linux/io.h>
40 #include <linux/hugetlb.h>
41 #include <linux/hugetlb_cgroup.h>
42 #include <linux/node.h>
43 #include <linux/page_owner.h>
44 #include "internal.h"
45 #include "hugetlb_vmemmap.h"
46 
47 int hugetlb_max_hstate __read_mostly;
48 unsigned int default_hstate_idx;
49 struct hstate hstates[HUGE_MAX_HSTATE];
50 
51 #ifdef CONFIG_CMA
52 static struct cma *hugetlb_cma[MAX_NUMNODES];
53 #endif
54 static unsigned long hugetlb_cma_size __initdata;
55 
56 /*
57  * Minimum page order among possible hugepage sizes, set to a proper value
58  * at boot time.
59  */
60 static unsigned int minimum_order __read_mostly = UINT_MAX;
61 
62 __initdata LIST_HEAD(huge_boot_pages);
63 
64 /* for command line parsing */
65 static struct hstate * __initdata parsed_hstate;
66 static unsigned long __initdata default_hstate_max_huge_pages;
67 static bool __initdata parsed_valid_hugepagesz = true;
68 static bool __initdata parsed_default_hugepagesz;
69 
70 /*
71  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
72  * free_huge_pages, and surplus_huge_pages.
73  */
74 DEFINE_SPINLOCK(hugetlb_lock);
75 
76 /*
77  * Serializes faults on the same logical page.  This is used to
78  * prevent spurious OOMs when the hugepage pool is fully utilized.
79  */
80 static int num_fault_mutexes;
81 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
82 
83 /* Forward declaration */
84 static int hugetlb_acct_memory(struct hstate *h, long delta);
85 
86 static inline bool subpool_is_free(struct hugepage_subpool *spool)
87 {
88 	if (spool->count)
89 		return false;
90 	if (spool->max_hpages != -1)
91 		return spool->used_hpages == 0;
92 	if (spool->min_hpages != -1)
93 		return spool->rsv_hpages == spool->min_hpages;
94 
95 	return true;
96 }
97 
98 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
99 						unsigned long irq_flags)
100 {
101 	spin_unlock_irqrestore(&spool->lock, irq_flags);
102 
103 	/* If no pages are used, and no other handles to the subpool
104 	 * remain, give up any reservations based on minimum size and
105 	 * free the subpool */
106 	if (subpool_is_free(spool)) {
107 		if (spool->min_hpages != -1)
108 			hugetlb_acct_memory(spool->hstate,
109 						-spool->min_hpages);
110 		kfree(spool);
111 	}
112 }
113 
114 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
115 						long min_hpages)
116 {
117 	struct hugepage_subpool *spool;
118 
119 	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
120 	if (!spool)
121 		return NULL;
122 
123 	spin_lock_init(&spool->lock);
124 	spool->count = 1;
125 	spool->max_hpages = max_hpages;
126 	spool->hstate = h;
127 	spool->min_hpages = min_hpages;
128 
129 	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
130 		kfree(spool);
131 		return NULL;
132 	}
133 	spool->rsv_hpages = min_hpages;
134 
135 	return spool;
136 }
137 
138 void hugepage_put_subpool(struct hugepage_subpool *spool)
139 {
140 	unsigned long flags;
141 
142 	spin_lock_irqsave(&spool->lock, flags);
143 	BUG_ON(!spool->count);
144 	spool->count--;
145 	unlock_or_release_subpool(spool, flags);
146 }
147 
148 /*
149  * Subpool accounting for allocating and reserving pages.
150  * Return -ENOMEM if there are not enough resources to satisfy the
151  * request.  Otherwise, return the number of pages by which the
152  * global pools must be adjusted (upward).  The returned value may
153  * only be different than the passed value (delta) in the case where
154  * a subpool minimum size must be maintained.
155  */
156 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
157 				      long delta)
158 {
159 	long ret = delta;
160 
161 	if (!spool)
162 		return ret;
163 
164 	spin_lock_irq(&spool->lock);
165 
166 	if (spool->max_hpages != -1) {		/* maximum size accounting */
167 		if ((spool->used_hpages + delta) <= spool->max_hpages)
168 			spool->used_hpages += delta;
169 		else {
170 			ret = -ENOMEM;
171 			goto unlock_ret;
172 		}
173 	}
174 
175 	/* minimum size accounting */
176 	if (spool->min_hpages != -1 && spool->rsv_hpages) {
177 		if (delta > spool->rsv_hpages) {
178 			/*
179 			 * Asking for more reserves than those already taken on
180 			 * behalf of subpool.  Return difference.
181 			 */
182 			ret = delta - spool->rsv_hpages;
183 			spool->rsv_hpages = 0;
184 		} else {
185 			ret = 0;	/* reserves already accounted for */
186 			spool->rsv_hpages -= delta;
187 		}
188 	}
189 
190 unlock_ret:
191 	spin_unlock_irq(&spool->lock);
192 	return ret;
193 }
194 
195 /*
196  * Subpool accounting for freeing and unreserving pages.
197  * Return the number of global page reservations that must be dropped.
198  * The return value may only be different than the passed value (delta)
199  * in the case where a subpool minimum size must be maintained.
200  */
201 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
202 				       long delta)
203 {
204 	long ret = delta;
205 	unsigned long flags;
206 
207 	if (!spool)
208 		return delta;
209 
210 	spin_lock_irqsave(&spool->lock, flags);
211 
212 	if (spool->max_hpages != -1)		/* maximum size accounting */
213 		spool->used_hpages -= delta;
214 
215 	 /* minimum size accounting */
216 	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
217 		if (spool->rsv_hpages + delta <= spool->min_hpages)
218 			ret = 0;
219 		else
220 			ret = spool->rsv_hpages + delta - spool->min_hpages;
221 
222 		spool->rsv_hpages += delta;
223 		if (spool->rsv_hpages > spool->min_hpages)
224 			spool->rsv_hpages = spool->min_hpages;
225 	}
226 
227 	/*
228 	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
229 	 * quota reference, free it now.
230 	 */
231 	unlock_or_release_subpool(spool, flags);
232 
233 	return ret;
234 }
235 
236 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
237 {
238 	return HUGETLBFS_SB(inode->i_sb)->spool;
239 }
240 
241 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
242 {
243 	return subpool_inode(file_inode(vma->vm_file));
244 }
245 
246 /* Helper that removes a struct file_region from the resv_map cache and returns
247  * it for use.
248  */
249 static struct file_region *
250 get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
251 {
252 	struct file_region *nrg = NULL;
253 
254 	VM_BUG_ON(resv->region_cache_count <= 0);
255 
256 	resv->region_cache_count--;
257 	nrg = list_first_entry(&resv->region_cache, struct file_region, link);
258 	list_del(&nrg->link);
259 
260 	nrg->from = from;
261 	nrg->to = to;
262 
263 	return nrg;
264 }
265 
266 static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
267 					      struct file_region *rg)
268 {
269 #ifdef CONFIG_CGROUP_HUGETLB
270 	nrg->reservation_counter = rg->reservation_counter;
271 	nrg->css = rg->css;
272 	if (rg->css)
273 		css_get(rg->css);
274 #endif
275 }
276 
277 /* Helper that records hugetlb_cgroup uncharge info. */
278 static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
279 						struct hstate *h,
280 						struct resv_map *resv,
281 						struct file_region *nrg)
282 {
283 #ifdef CONFIG_CGROUP_HUGETLB
284 	if (h_cg) {
285 		nrg->reservation_counter =
286 			&h_cg->rsvd_hugepage[hstate_index(h)];
287 		nrg->css = &h_cg->css;
288 		/*
289 		 * The caller will hold exactly one h_cg->css reference for the
290 		 * whole contiguous reservation region. But this area might be
291 		 * scattered when there are already some file_regions reside in
292 		 * it. As a result, many file_regions may share only one css
293 		 * reference. In order to ensure that one file_region must hold
294 		 * exactly one h_cg->css reference, we should do css_get for
295 		 * each file_region and leave the reference held by caller
296 		 * untouched.
297 		 */
298 		css_get(&h_cg->css);
299 		if (!resv->pages_per_hpage)
300 			resv->pages_per_hpage = pages_per_huge_page(h);
301 		/* pages_per_hpage should be the same for all entries in
302 		 * a resv_map.
303 		 */
304 		VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
305 	} else {
306 		nrg->reservation_counter = NULL;
307 		nrg->css = NULL;
308 	}
309 #endif
310 }
311 
312 static void put_uncharge_info(struct file_region *rg)
313 {
314 #ifdef CONFIG_CGROUP_HUGETLB
315 	if (rg->css)
316 		css_put(rg->css);
317 #endif
318 }
319 
320 static bool has_same_uncharge_info(struct file_region *rg,
321 				   struct file_region *org)
322 {
323 #ifdef CONFIG_CGROUP_HUGETLB
324 	return rg && org &&
325 	       rg->reservation_counter == org->reservation_counter &&
326 	       rg->css == org->css;
327 
328 #else
329 	return true;
330 #endif
331 }
332 
333 static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
334 {
335 	struct file_region *nrg = NULL, *prg = NULL;
336 
337 	prg = list_prev_entry(rg, link);
338 	if (&prg->link != &resv->regions && prg->to == rg->from &&
339 	    has_same_uncharge_info(prg, rg)) {
340 		prg->to = rg->to;
341 
342 		list_del(&rg->link);
343 		put_uncharge_info(rg);
344 		kfree(rg);
345 
346 		rg = prg;
347 	}
348 
349 	nrg = list_next_entry(rg, link);
350 	if (&nrg->link != &resv->regions && nrg->from == rg->to &&
351 	    has_same_uncharge_info(nrg, rg)) {
352 		nrg->from = rg->from;
353 
354 		list_del(&rg->link);
355 		put_uncharge_info(rg);
356 		kfree(rg);
357 	}
358 }
359 
360 static inline long
361 hugetlb_resv_map_add(struct resv_map *map, struct file_region *rg, long from,
362 		     long to, struct hstate *h, struct hugetlb_cgroup *cg,
363 		     long *regions_needed)
364 {
365 	struct file_region *nrg;
366 
367 	if (!regions_needed) {
368 		nrg = get_file_region_entry_from_cache(map, from, to);
369 		record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
370 		list_add(&nrg->link, rg->link.prev);
371 		coalesce_file_region(map, nrg);
372 	} else
373 		*regions_needed += 1;
374 
375 	return to - from;
376 }
377 
378 /*
379  * Must be called with resv->lock held.
380  *
381  * Calling this with regions_needed != NULL will count the number of pages
382  * to be added but will not modify the linked list. And regions_needed will
383  * indicate the number of file_regions needed in the cache to carry out to add
384  * the regions for this range.
385  */
386 static long add_reservation_in_range(struct resv_map *resv, long f, long t,
387 				     struct hugetlb_cgroup *h_cg,
388 				     struct hstate *h, long *regions_needed)
389 {
390 	long add = 0;
391 	struct list_head *head = &resv->regions;
392 	long last_accounted_offset = f;
393 	struct file_region *rg = NULL, *trg = NULL;
394 
395 	if (regions_needed)
396 		*regions_needed = 0;
397 
398 	/* In this loop, we essentially handle an entry for the range
399 	 * [last_accounted_offset, rg->from), at every iteration, with some
400 	 * bounds checking.
401 	 */
402 	list_for_each_entry_safe(rg, trg, head, link) {
403 		/* Skip irrelevant regions that start before our range. */
404 		if (rg->from < f) {
405 			/* If this region ends after the last accounted offset,
406 			 * then we need to update last_accounted_offset.
407 			 */
408 			if (rg->to > last_accounted_offset)
409 				last_accounted_offset = rg->to;
410 			continue;
411 		}
412 
413 		/* When we find a region that starts beyond our range, we've
414 		 * finished.
415 		 */
416 		if (rg->from >= t)
417 			break;
418 
419 		/* Add an entry for last_accounted_offset -> rg->from, and
420 		 * update last_accounted_offset.
421 		 */
422 		if (rg->from > last_accounted_offset)
423 			add += hugetlb_resv_map_add(resv, rg,
424 						    last_accounted_offset,
425 						    rg->from, h, h_cg,
426 						    regions_needed);
427 
428 		last_accounted_offset = rg->to;
429 	}
430 
431 	/* Handle the case where our range extends beyond
432 	 * last_accounted_offset.
433 	 */
434 	if (last_accounted_offset < t)
435 		add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
436 					    t, h, h_cg, regions_needed);
437 
438 	VM_BUG_ON(add < 0);
439 	return add;
440 }
441 
442 /* Must be called with resv->lock acquired. Will drop lock to allocate entries.
443  */
444 static int allocate_file_region_entries(struct resv_map *resv,
445 					int regions_needed)
446 	__must_hold(&resv->lock)
447 {
448 	struct list_head allocated_regions;
449 	int to_allocate = 0, i = 0;
450 	struct file_region *trg = NULL, *rg = NULL;
451 
452 	VM_BUG_ON(regions_needed < 0);
453 
454 	INIT_LIST_HEAD(&allocated_regions);
455 
456 	/*
457 	 * Check for sufficient descriptors in the cache to accommodate
458 	 * the number of in progress add operations plus regions_needed.
459 	 *
460 	 * This is a while loop because when we drop the lock, some other call
461 	 * to region_add or region_del may have consumed some region_entries,
462 	 * so we keep looping here until we finally have enough entries for
463 	 * (adds_in_progress + regions_needed).
464 	 */
465 	while (resv->region_cache_count <
466 	       (resv->adds_in_progress + regions_needed)) {
467 		to_allocate = resv->adds_in_progress + regions_needed -
468 			      resv->region_cache_count;
469 
470 		/* At this point, we should have enough entries in the cache
471 		 * for all the existing adds_in_progress. We should only be
472 		 * needing to allocate for regions_needed.
473 		 */
474 		VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);
475 
476 		spin_unlock(&resv->lock);
477 		for (i = 0; i < to_allocate; i++) {
478 			trg = kmalloc(sizeof(*trg), GFP_KERNEL);
479 			if (!trg)
480 				goto out_of_memory;
481 			list_add(&trg->link, &allocated_regions);
482 		}
483 
484 		spin_lock(&resv->lock);
485 
486 		list_splice(&allocated_regions, &resv->region_cache);
487 		resv->region_cache_count += to_allocate;
488 	}
489 
490 	return 0;
491 
492 out_of_memory:
493 	list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
494 		list_del(&rg->link);
495 		kfree(rg);
496 	}
497 	return -ENOMEM;
498 }
499 
500 /*
501  * Add the huge page range represented by [f, t) to the reserve
502  * map.  Regions will be taken from the cache to fill in this range.
503  * Sufficient regions should exist in the cache due to the previous
504  * call to region_chg with the same range, but in some cases the cache will not
505  * have sufficient entries due to races with other code doing region_add or
506  * region_del.  The extra needed entries will be allocated.
507  *
508  * regions_needed is the out value provided by a previous call to region_chg.
509  *
510  * Return the number of new huge pages added to the map.  This number is greater
511  * than or equal to zero.  If file_region entries needed to be allocated for
512  * this operation and we were not able to allocate, it returns -ENOMEM.
513  * region_add of regions of length 1 never allocate file_regions and cannot
514  * fail; region_chg will always allocate at least 1 entry and a region_add for
515  * 1 page will only require at most 1 entry.
516  */
517 static long region_add(struct resv_map *resv, long f, long t,
518 		       long in_regions_needed, struct hstate *h,
519 		       struct hugetlb_cgroup *h_cg)
520 {
521 	long add = 0, actual_regions_needed = 0;
522 
523 	spin_lock(&resv->lock);
524 retry:
525 
526 	/* Count how many regions are actually needed to execute this add. */
527 	add_reservation_in_range(resv, f, t, NULL, NULL,
528 				 &actual_regions_needed);
529 
530 	/*
531 	 * Check for sufficient descriptors in the cache to accommodate
532 	 * this add operation. Note that actual_regions_needed may be greater
533 	 * than in_regions_needed, as the resv_map may have been modified since
534 	 * the region_chg call. In this case, we need to make sure that we
535 	 * allocate extra entries, such that we have enough for all the
536 	 * existing adds_in_progress, plus the excess needed for this
537 	 * operation.
538 	 */
539 	if (actual_regions_needed > in_regions_needed &&
540 	    resv->region_cache_count <
541 		    resv->adds_in_progress +
542 			    (actual_regions_needed - in_regions_needed)) {
543 		/* region_add operation of range 1 should never need to
544 		 * allocate file_region entries.
545 		 */
546 		VM_BUG_ON(t - f <= 1);
547 
548 		if (allocate_file_region_entries(
549 			    resv, actual_regions_needed - in_regions_needed)) {
550 			return -ENOMEM;
551 		}
552 
553 		goto retry;
554 	}
555 
556 	add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
557 
558 	resv->adds_in_progress -= in_regions_needed;
559 
560 	spin_unlock(&resv->lock);
561 	return add;
562 }
563 
564 /*
565  * Examine the existing reserve map and determine how many
566  * huge pages in the specified range [f, t) are NOT currently
567  * represented.  This routine is called before a subsequent
568  * call to region_add that will actually modify the reserve
569  * map to add the specified range [f, t).  region_chg does
570  * not change the number of huge pages represented by the
571  * map.  A number of new file_region structures is added to the cache as a
572  * placeholder, for the subsequent region_add call to use. At least 1
573  * file_region structure is added.
574  *
575  * out_regions_needed is the number of regions added to the
576  * resv->adds_in_progress.  This value needs to be provided to a follow up call
577  * to region_add or region_abort for proper accounting.
578  *
579  * Returns the number of huge pages that need to be added to the existing
580  * reservation map for the range [f, t).  This number is greater or equal to
581  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
582  * is needed and can not be allocated.
583  */
584 static long region_chg(struct resv_map *resv, long f, long t,
585 		       long *out_regions_needed)
586 {
587 	long chg = 0;
588 
589 	spin_lock(&resv->lock);
590 
591 	/* Count how many hugepages in this range are NOT represented. */
592 	chg = add_reservation_in_range(resv, f, t, NULL, NULL,
593 				       out_regions_needed);
594 
595 	if (*out_regions_needed == 0)
596 		*out_regions_needed = 1;
597 
598 	if (allocate_file_region_entries(resv, *out_regions_needed))
599 		return -ENOMEM;
600 
601 	resv->adds_in_progress += *out_regions_needed;
602 
603 	spin_unlock(&resv->lock);
604 	return chg;
605 }
606 
607 /*
608  * Abort the in progress add operation.  The adds_in_progress field
609  * of the resv_map keeps track of the operations in progress between
610  * calls to region_chg and region_add.  Operations are sometimes
611  * aborted after the call to region_chg.  In such cases, region_abort
612  * is called to decrement the adds_in_progress counter. regions_needed
613  * is the value returned by the region_chg call, it is used to decrement
614  * the adds_in_progress counter.
615  *
616  * NOTE: The range arguments [f, t) are not needed or used in this
617  * routine.  They are kept to make reading the calling code easier as
618  * arguments will match the associated region_chg call.
619  */
620 static void region_abort(struct resv_map *resv, long f, long t,
621 			 long regions_needed)
622 {
623 	spin_lock(&resv->lock);
624 	VM_BUG_ON(!resv->region_cache_count);
625 	resv->adds_in_progress -= regions_needed;
626 	spin_unlock(&resv->lock);
627 }
628 
629 /*
630  * Delete the specified range [f, t) from the reserve map.  If the
631  * t parameter is LONG_MAX, this indicates that ALL regions after f
632  * should be deleted.  Locate the regions which intersect [f, t)
633  * and either trim, delete or split the existing regions.
634  *
635  * Returns the number of huge pages deleted from the reserve map.
636  * In the normal case, the return value is zero or more.  In the
637  * case where a region must be split, a new region descriptor must
638  * be allocated.  If the allocation fails, -ENOMEM will be returned.
639  * NOTE: If the parameter t == LONG_MAX, then we will never split
640  * a region and possibly return -ENOMEM.  Callers specifying
641  * t == LONG_MAX do not need to check for -ENOMEM error.
642  */
643 static long region_del(struct resv_map *resv, long f, long t)
644 {
645 	struct list_head *head = &resv->regions;
646 	struct file_region *rg, *trg;
647 	struct file_region *nrg = NULL;
648 	long del = 0;
649 
650 retry:
651 	spin_lock(&resv->lock);
652 	list_for_each_entry_safe(rg, trg, head, link) {
653 		/*
654 		 * Skip regions before the range to be deleted.  file_region
655 		 * ranges are normally of the form [from, to).  However, there
656 		 * may be a "placeholder" entry in the map which is of the form
657 		 * (from, to) with from == to.  Check for placeholder entries
658 		 * at the beginning of the range to be deleted.
659 		 */
660 		if (rg->to <= f && (rg->to != rg->from || rg->to != f))
661 			continue;
662 
663 		if (rg->from >= t)
664 			break;
665 
666 		if (f > rg->from && t < rg->to) { /* Must split region */
667 			/*
668 			 * Check for an entry in the cache before dropping
669 			 * lock and attempting allocation.
670 			 */
671 			if (!nrg &&
672 			    resv->region_cache_count > resv->adds_in_progress) {
673 				nrg = list_first_entry(&resv->region_cache,
674 							struct file_region,
675 							link);
676 				list_del(&nrg->link);
677 				resv->region_cache_count--;
678 			}
679 
680 			if (!nrg) {
681 				spin_unlock(&resv->lock);
682 				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
683 				if (!nrg)
684 					return -ENOMEM;
685 				goto retry;
686 			}
687 
688 			del += t - f;
689 			hugetlb_cgroup_uncharge_file_region(
690 				resv, rg, t - f, false);
691 
692 			/* New entry for end of split region */
693 			nrg->from = t;
694 			nrg->to = rg->to;
695 
696 			copy_hugetlb_cgroup_uncharge_info(nrg, rg);
697 
698 			INIT_LIST_HEAD(&nrg->link);
699 
700 			/* Original entry is trimmed */
701 			rg->to = f;
702 
703 			list_add(&nrg->link, &rg->link);
704 			nrg = NULL;
705 			break;
706 		}
707 
708 		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
709 			del += rg->to - rg->from;
710 			hugetlb_cgroup_uncharge_file_region(resv, rg,
711 							    rg->to - rg->from, true);
712 			list_del(&rg->link);
713 			kfree(rg);
714 			continue;
715 		}
716 
717 		if (f <= rg->from) {	/* Trim beginning of region */
718 			hugetlb_cgroup_uncharge_file_region(resv, rg,
719 							    t - rg->from, false);
720 
721 			del += t - rg->from;
722 			rg->from = t;
723 		} else {		/* Trim end of region */
724 			hugetlb_cgroup_uncharge_file_region(resv, rg,
725 							    rg->to - f, false);
726 
727 			del += rg->to - f;
728 			rg->to = f;
729 		}
730 	}
731 
732 	spin_unlock(&resv->lock);
733 	kfree(nrg);
734 	return del;
735 }
736 
737 /*
738  * A rare out of memory error was encountered which prevented removal of
739  * the reserve map region for a page.  The huge page itself was free'ed
740  * and removed from the page cache.  This routine will adjust the subpool
741  * usage count, and the global reserve count if needed.  By incrementing
742  * these counts, the reserve map entry which could not be deleted will
743  * appear as a "reserved" entry instead of simply dangling with incorrect
744  * counts.
745  */
746 void hugetlb_fix_reserve_counts(struct inode *inode)
747 {
748 	struct hugepage_subpool *spool = subpool_inode(inode);
749 	long rsv_adjust;
750 	bool reserved = false;
751 
752 	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
753 	if (rsv_adjust > 0) {
754 		struct hstate *h = hstate_inode(inode);
755 
756 		if (!hugetlb_acct_memory(h, 1))
757 			reserved = true;
758 	} else if (!rsv_adjust) {
759 		reserved = true;
760 	}
761 
762 	if (!reserved)
763 		pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
764 }
765 
766 /*
767  * Count and return the number of huge pages in the reserve map
768  * that intersect with the range [f, t).
769  */
770 static long region_count(struct resv_map *resv, long f, long t)
771 {
772 	struct list_head *head = &resv->regions;
773 	struct file_region *rg;
774 	long chg = 0;
775 
776 	spin_lock(&resv->lock);
777 	/* Locate each segment we overlap with, and count that overlap. */
778 	list_for_each_entry(rg, head, link) {
779 		long seg_from;
780 		long seg_to;
781 
782 		if (rg->to <= f)
783 			continue;
784 		if (rg->from >= t)
785 			break;
786 
787 		seg_from = max(rg->from, f);
788 		seg_to = min(rg->to, t);
789 
790 		chg += seg_to - seg_from;
791 	}
792 	spin_unlock(&resv->lock);
793 
794 	return chg;
795 }
796 
797 /*
798  * Convert the address within this vma to the page offset within
799  * the mapping, in pagecache page units; huge pages here.
800  */
801 static pgoff_t vma_hugecache_offset(struct hstate *h,
802 			struct vm_area_struct *vma, unsigned long address)
803 {
804 	return ((address - vma->vm_start) >> huge_page_shift(h)) +
805 			(vma->vm_pgoff >> huge_page_order(h));
806 }
807 
808 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
809 				     unsigned long address)
810 {
811 	return vma_hugecache_offset(hstate_vma(vma), vma, address);
812 }
813 EXPORT_SYMBOL_GPL(linear_hugepage_index);
814 
815 /*
816  * Return the size of the pages allocated when backing a VMA. In the majority
817  * cases this will be same size as used by the page table entries.
818  */
819 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
820 {
821 	if (vma->vm_ops && vma->vm_ops->pagesize)
822 		return vma->vm_ops->pagesize(vma);
823 	return PAGE_SIZE;
824 }
825 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
826 
827 /*
828  * Return the page size being used by the MMU to back a VMA. In the majority
829  * of cases, the page size used by the kernel matches the MMU size. On
830  * architectures where it differs, an architecture-specific 'strong'
831  * version of this symbol is required.
832  */
833 __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
834 {
835 	return vma_kernel_pagesize(vma);
836 }
837 
838 /*
839  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
840  * bits of the reservation map pointer, which are always clear due to
841  * alignment.
842  */
843 #define HPAGE_RESV_OWNER    (1UL << 0)
844 #define HPAGE_RESV_UNMAPPED (1UL << 1)
845 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
846 
847 /*
848  * These helpers are used to track how many pages are reserved for
849  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
850  * is guaranteed to have their future faults succeed.
851  *
852  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
853  * the reserve counters are updated with the hugetlb_lock held. It is safe
854  * to reset the VMA at fork() time as it is not in use yet and there is no
855  * chance of the global counters getting corrupted as a result of the values.
856  *
857  * The private mapping reservation is represented in a subtly different
858  * manner to a shared mapping.  A shared mapping has a region map associated
859  * with the underlying file, this region map represents the backing file
860  * pages which have ever had a reservation assigned which this persists even
861  * after the page is instantiated.  A private mapping has a region map
862  * associated with the original mmap which is attached to all VMAs which
863  * reference it, this region map represents those offsets which have consumed
864  * reservation ie. where pages have been instantiated.
865  */
866 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
867 {
868 	return (unsigned long)vma->vm_private_data;
869 }
870 
871 static void set_vma_private_data(struct vm_area_struct *vma,
872 							unsigned long value)
873 {
874 	vma->vm_private_data = (void *)value;
875 }
876 
877 static void
878 resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
879 					  struct hugetlb_cgroup *h_cg,
880 					  struct hstate *h)
881 {
882 #ifdef CONFIG_CGROUP_HUGETLB
883 	if (!h_cg || !h) {
884 		resv_map->reservation_counter = NULL;
885 		resv_map->pages_per_hpage = 0;
886 		resv_map->css = NULL;
887 	} else {
888 		resv_map->reservation_counter =
889 			&h_cg->rsvd_hugepage[hstate_index(h)];
890 		resv_map->pages_per_hpage = pages_per_huge_page(h);
891 		resv_map->css = &h_cg->css;
892 	}
893 #endif
894 }
895 
896 struct resv_map *resv_map_alloc(void)
897 {
898 	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
899 	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
900 
901 	if (!resv_map || !rg) {
902 		kfree(resv_map);
903 		kfree(rg);
904 		return NULL;
905 	}
906 
907 	kref_init(&resv_map->refs);
908 	spin_lock_init(&resv_map->lock);
909 	INIT_LIST_HEAD(&resv_map->regions);
910 
911 	resv_map->adds_in_progress = 0;
912 	/*
913 	 * Initialize these to 0. On shared mappings, 0's here indicate these
914 	 * fields don't do cgroup accounting. On private mappings, these will be
915 	 * re-initialized to the proper values, to indicate that hugetlb cgroup
916 	 * reservations are to be un-charged from here.
917 	 */
918 	resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
919 
920 	INIT_LIST_HEAD(&resv_map->region_cache);
921 	list_add(&rg->link, &resv_map->region_cache);
922 	resv_map->region_cache_count = 1;
923 
924 	return resv_map;
925 }
926 
927 void resv_map_release(struct kref *ref)
928 {
929 	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
930 	struct list_head *head = &resv_map->region_cache;
931 	struct file_region *rg, *trg;
932 
933 	/* Clear out any active regions before we release the map. */
934 	region_del(resv_map, 0, LONG_MAX);
935 
936 	/* ... and any entries left in the cache */
937 	list_for_each_entry_safe(rg, trg, head, link) {
938 		list_del(&rg->link);
939 		kfree(rg);
940 	}
941 
942 	VM_BUG_ON(resv_map->adds_in_progress);
943 
944 	kfree(resv_map);
945 }
946 
947 static inline struct resv_map *inode_resv_map(struct inode *inode)
948 {
949 	/*
950 	 * At inode evict time, i_mapping may not point to the original
951 	 * address space within the inode.  This original address space
952 	 * contains the pointer to the resv_map.  So, always use the
953 	 * address space embedded within the inode.
954 	 * The VERY common case is inode->mapping == &inode->i_data but,
955 	 * this may not be true for device special inodes.
956 	 */
957 	return (struct resv_map *)(&inode->i_data)->private_data;
958 }
959 
960 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
961 {
962 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
963 	if (vma->vm_flags & VM_MAYSHARE) {
964 		struct address_space *mapping = vma->vm_file->f_mapping;
965 		struct inode *inode = mapping->host;
966 
967 		return inode_resv_map(inode);
968 
969 	} else {
970 		return (struct resv_map *)(get_vma_private_data(vma) &
971 							~HPAGE_RESV_MASK);
972 	}
973 }
974 
975 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
976 {
977 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
978 	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
979 
980 	set_vma_private_data(vma, (get_vma_private_data(vma) &
981 				HPAGE_RESV_MASK) | (unsigned long)map);
982 }
983 
984 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
985 {
986 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
987 	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
988 
989 	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
990 }
991 
992 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
993 {
994 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
995 
996 	return (get_vma_private_data(vma) & flag) != 0;
997 }
998 
999 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
1000 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
1001 {
1002 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1003 	if (!(vma->vm_flags & VM_MAYSHARE))
1004 		vma->vm_private_data = (void *)0;
1005 }
1006 
1007 /* Returns true if the VMA has associated reserve pages */
1008 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
1009 {
1010 	if (vma->vm_flags & VM_NORESERVE) {
1011 		/*
1012 		 * This address is already reserved by other process(chg == 0),
1013 		 * so, we should decrement reserved count. Without decrementing,
1014 		 * reserve count remains after releasing inode, because this
1015 		 * allocated page will go into page cache and is regarded as
1016 		 * coming from reserved pool in releasing step.  Currently, we
1017 		 * don't have any other solution to deal with this situation
1018 		 * properly, so add work-around here.
1019 		 */
1020 		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
1021 			return true;
1022 		else
1023 			return false;
1024 	}
1025 
1026 	/* Shared mappings always use reserves */
1027 	if (vma->vm_flags & VM_MAYSHARE) {
1028 		/*
1029 		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
1030 		 * be a region map for all pages.  The only situation where
1031 		 * there is no region map is if a hole was punched via
1032 		 * fallocate.  In this case, there really are no reserves to
1033 		 * use.  This situation is indicated if chg != 0.
1034 		 */
1035 		if (chg)
1036 			return false;
1037 		else
1038 			return true;
1039 	}
1040 
1041 	/*
1042 	 * Only the process that called mmap() has reserves for
1043 	 * private mappings.
1044 	 */
1045 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1046 		/*
1047 		 * Like the shared case above, a hole punch or truncate
1048 		 * could have been performed on the private mapping.
1049 		 * Examine the value of chg to determine if reserves
1050 		 * actually exist or were previously consumed.
1051 		 * Very Subtle - The value of chg comes from a previous
1052 		 * call to vma_needs_reserves().  The reserve map for
1053 		 * private mappings has different (opposite) semantics
1054 		 * than that of shared mappings.  vma_needs_reserves()
1055 		 * has already taken this difference in semantics into
1056 		 * account.  Therefore, the meaning of chg is the same
1057 		 * as in the shared case above.  Code could easily be
1058 		 * combined, but keeping it separate draws attention to
1059 		 * subtle differences.
1060 		 */
1061 		if (chg)
1062 			return false;
1063 		else
1064 			return true;
1065 	}
1066 
1067 	return false;
1068 }
1069 
1070 static void enqueue_huge_page(struct hstate *h, struct page *page)
1071 {
1072 	int nid = page_to_nid(page);
1073 
1074 	lockdep_assert_held(&hugetlb_lock);
1075 	list_move(&page->lru, &h->hugepage_freelists[nid]);
1076 	h->free_huge_pages++;
1077 	h->free_huge_pages_node[nid]++;
1078 	SetHPageFreed(page);
1079 }
1080 
1081 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
1082 {
1083 	struct page *page;
1084 	bool pin = !!(current->flags & PF_MEMALLOC_PIN);
1085 
1086 	lockdep_assert_held(&hugetlb_lock);
1087 	list_for_each_entry(page, &h->hugepage_freelists[nid], lru) {
1088 		if (pin && !is_pinnable_page(page))
1089 			continue;
1090 
1091 		if (PageHWPoison(page))
1092 			continue;
1093 
1094 		list_move(&page->lru, &h->hugepage_activelist);
1095 		set_page_refcounted(page);
1096 		ClearHPageFreed(page);
1097 		h->free_huge_pages--;
1098 		h->free_huge_pages_node[nid]--;
1099 		return page;
1100 	}
1101 
1102 	return NULL;
1103 }
1104 
1105 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
1106 		nodemask_t *nmask)
1107 {
1108 	unsigned int cpuset_mems_cookie;
1109 	struct zonelist *zonelist;
1110 	struct zone *zone;
1111 	struct zoneref *z;
1112 	int node = NUMA_NO_NODE;
1113 
1114 	zonelist = node_zonelist(nid, gfp_mask);
1115 
1116 retry_cpuset:
1117 	cpuset_mems_cookie = read_mems_allowed_begin();
1118 	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
1119 		struct page *page;
1120 
1121 		if (!cpuset_zone_allowed(zone, gfp_mask))
1122 			continue;
1123 		/*
1124 		 * no need to ask again on the same node. Pool is node rather than
1125 		 * zone aware
1126 		 */
1127 		if (zone_to_nid(zone) == node)
1128 			continue;
1129 		node = zone_to_nid(zone);
1130 
1131 		page = dequeue_huge_page_node_exact(h, node);
1132 		if (page)
1133 			return page;
1134 	}
1135 	if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
1136 		goto retry_cpuset;
1137 
1138 	return NULL;
1139 }
1140 
1141 static struct page *dequeue_huge_page_vma(struct hstate *h,
1142 				struct vm_area_struct *vma,
1143 				unsigned long address, int avoid_reserve,
1144 				long chg)
1145 {
1146 	struct page *page;
1147 	struct mempolicy *mpol;
1148 	gfp_t gfp_mask;
1149 	nodemask_t *nodemask;
1150 	int nid;
1151 
1152 	/*
1153 	 * A child process with MAP_PRIVATE mappings created by their parent
1154 	 * have no page reserves. This check ensures that reservations are
1155 	 * not "stolen". The child may still get SIGKILLed
1156 	 */
1157 	if (!vma_has_reserves(vma, chg) &&
1158 			h->free_huge_pages - h->resv_huge_pages == 0)
1159 		goto err;
1160 
1161 	/* If reserves cannot be used, ensure enough pages are in the pool */
1162 	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
1163 		goto err;
1164 
1165 	gfp_mask = htlb_alloc_mask(h);
1166 	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1167 	page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
1168 	if (page && !avoid_reserve && vma_has_reserves(vma, chg)) {
1169 		SetHPageRestoreReserve(page);
1170 		h->resv_huge_pages--;
1171 	}
1172 
1173 	mpol_cond_put(mpol);
1174 	return page;
1175 
1176 err:
1177 	return NULL;
1178 }
1179 
1180 /*
1181  * common helper functions for hstate_next_node_to_{alloc|free}.
1182  * We may have allocated or freed a huge page based on a different
1183  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1184  * be outside of *nodes_allowed.  Ensure that we use an allowed
1185  * node for alloc or free.
1186  */
1187 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
1188 {
1189 	nid = next_node_in(nid, *nodes_allowed);
1190 	VM_BUG_ON(nid >= MAX_NUMNODES);
1191 
1192 	return nid;
1193 }
1194 
1195 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
1196 {
1197 	if (!node_isset(nid, *nodes_allowed))
1198 		nid = next_node_allowed(nid, nodes_allowed);
1199 	return nid;
1200 }
1201 
1202 /*
1203  * returns the previously saved node ["this node"] from which to
1204  * allocate a persistent huge page for the pool and advance the
1205  * next node from which to allocate, handling wrap at end of node
1206  * mask.
1207  */
1208 static int hstate_next_node_to_alloc(struct hstate *h,
1209 					nodemask_t *nodes_allowed)
1210 {
1211 	int nid;
1212 
1213 	VM_BUG_ON(!nodes_allowed);
1214 
1215 	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1216 	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1217 
1218 	return nid;
1219 }
1220 
1221 /*
1222  * helper for remove_pool_huge_page() - return the previously saved
1223  * node ["this node"] from which to free a huge page.  Advance the
1224  * next node id whether or not we find a free huge page to free so
1225  * that the next attempt to free addresses the next node.
1226  */
1227 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1228 {
1229 	int nid;
1230 
1231 	VM_BUG_ON(!nodes_allowed);
1232 
1233 	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1234 	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1235 
1236 	return nid;
1237 }
1238 
1239 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
1240 	for (nr_nodes = nodes_weight(*mask);				\
1241 		nr_nodes > 0 &&						\
1242 		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
1243 		nr_nodes--)
1244 
1245 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
1246 	for (nr_nodes = nodes_weight(*mask);				\
1247 		nr_nodes > 0 &&						\
1248 		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
1249 		nr_nodes--)
1250 
1251 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1252 static void destroy_compound_gigantic_page(struct page *page,
1253 					unsigned int order)
1254 {
1255 	int i;
1256 	int nr_pages = 1 << order;
1257 	struct page *p = page + 1;
1258 
1259 	atomic_set(compound_mapcount_ptr(page), 0);
1260 	atomic_set(compound_pincount_ptr(page), 0);
1261 
1262 	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1263 		clear_compound_head(p);
1264 		set_page_refcounted(p);
1265 	}
1266 
1267 	set_compound_order(page, 0);
1268 	page[1].compound_nr = 0;
1269 	__ClearPageHead(page);
1270 }
1271 
1272 static void free_gigantic_page(struct page *page, unsigned int order)
1273 {
1274 	/*
1275 	 * If the page isn't allocated using the cma allocator,
1276 	 * cma_release() returns false.
1277 	 */
1278 #ifdef CONFIG_CMA
1279 	if (cma_release(hugetlb_cma[page_to_nid(page)], page, 1 << order))
1280 		return;
1281 #endif
1282 
1283 	free_contig_range(page_to_pfn(page), 1 << order);
1284 }
1285 
1286 #ifdef CONFIG_CONTIG_ALLOC
1287 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1288 		int nid, nodemask_t *nodemask)
1289 {
1290 	unsigned long nr_pages = pages_per_huge_page(h);
1291 	if (nid == NUMA_NO_NODE)
1292 		nid = numa_mem_id();
1293 
1294 #ifdef CONFIG_CMA
1295 	{
1296 		struct page *page;
1297 		int node;
1298 
1299 		if (hugetlb_cma[nid]) {
1300 			page = cma_alloc(hugetlb_cma[nid], nr_pages,
1301 					huge_page_order(h), true);
1302 			if (page)
1303 				return page;
1304 		}
1305 
1306 		if (!(gfp_mask & __GFP_THISNODE)) {
1307 			for_each_node_mask(node, *nodemask) {
1308 				if (node == nid || !hugetlb_cma[node])
1309 					continue;
1310 
1311 				page = cma_alloc(hugetlb_cma[node], nr_pages,
1312 						huge_page_order(h), true);
1313 				if (page)
1314 					return page;
1315 			}
1316 		}
1317 	}
1318 #endif
1319 
1320 	return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
1321 }
1322 
1323 #else /* !CONFIG_CONTIG_ALLOC */
1324 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1325 					int nid, nodemask_t *nodemask)
1326 {
1327 	return NULL;
1328 }
1329 #endif /* CONFIG_CONTIG_ALLOC */
1330 
1331 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1332 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1333 					int nid, nodemask_t *nodemask)
1334 {
1335 	return NULL;
1336 }
1337 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1338 static inline void destroy_compound_gigantic_page(struct page *page,
1339 						unsigned int order) { }
1340 #endif
1341 
1342 /*
1343  * Remove hugetlb page from lists, and update dtor so that page appears
1344  * as just a compound page.  A reference is held on the page.
1345  *
1346  * Must be called with hugetlb lock held.
1347  */
1348 static void remove_hugetlb_page(struct hstate *h, struct page *page,
1349 							bool adjust_surplus)
1350 {
1351 	int nid = page_to_nid(page);
1352 
1353 	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1354 	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page);
1355 
1356 	lockdep_assert_held(&hugetlb_lock);
1357 	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1358 		return;
1359 
1360 	list_del(&page->lru);
1361 
1362 	if (HPageFreed(page)) {
1363 		h->free_huge_pages--;
1364 		h->free_huge_pages_node[nid]--;
1365 	}
1366 	if (adjust_surplus) {
1367 		h->surplus_huge_pages--;
1368 		h->surplus_huge_pages_node[nid]--;
1369 	}
1370 
1371 	set_page_refcounted(page);
1372 	set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1373 
1374 	h->nr_huge_pages--;
1375 	h->nr_huge_pages_node[nid]--;
1376 }
1377 
1378 static void add_hugetlb_page(struct hstate *h, struct page *page,
1379 			     bool adjust_surplus)
1380 {
1381 	int zeroed;
1382 	int nid = page_to_nid(page);
1383 
1384 	VM_BUG_ON_PAGE(!HPageVmemmapOptimized(page), page);
1385 
1386 	lockdep_assert_held(&hugetlb_lock);
1387 
1388 	INIT_LIST_HEAD(&page->lru);
1389 	h->nr_huge_pages++;
1390 	h->nr_huge_pages_node[nid]++;
1391 
1392 	if (adjust_surplus) {
1393 		h->surplus_huge_pages++;
1394 		h->surplus_huge_pages_node[nid]++;
1395 	}
1396 
1397 	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1398 	set_page_private(page, 0);
1399 	SetHPageVmemmapOptimized(page);
1400 
1401 	/*
1402 	 * This page is now managed by the hugetlb allocator and has
1403 	 * no users -- drop the last reference.
1404 	 */
1405 	zeroed = put_page_testzero(page);
1406 	VM_BUG_ON_PAGE(!zeroed, page);
1407 	arch_clear_hugepage_flags(page);
1408 	enqueue_huge_page(h, page);
1409 }
1410 
1411 static void __update_and_free_page(struct hstate *h, struct page *page)
1412 {
1413 	int i;
1414 	struct page *subpage = page;
1415 
1416 	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1417 		return;
1418 
1419 	if (alloc_huge_page_vmemmap(h, page)) {
1420 		spin_lock_irq(&hugetlb_lock);
1421 		/*
1422 		 * If we cannot allocate vmemmap pages, just refuse to free the
1423 		 * page and put the page back on the hugetlb free list and treat
1424 		 * as a surplus page.
1425 		 */
1426 		add_hugetlb_page(h, page, true);
1427 		spin_unlock_irq(&hugetlb_lock);
1428 		return;
1429 	}
1430 
1431 	for (i = 0; i < pages_per_huge_page(h);
1432 	     i++, subpage = mem_map_next(subpage, page, i)) {
1433 		subpage->flags &= ~(1 << PG_locked | 1 << PG_error |
1434 				1 << PG_referenced | 1 << PG_dirty |
1435 				1 << PG_active | 1 << PG_private |
1436 				1 << PG_writeback);
1437 	}
1438 	if (hstate_is_gigantic(h)) {
1439 		destroy_compound_gigantic_page(page, huge_page_order(h));
1440 		free_gigantic_page(page, huge_page_order(h));
1441 	} else {
1442 		__free_pages(page, huge_page_order(h));
1443 	}
1444 }
1445 
1446 /*
1447  * As update_and_free_page() can be called under any context, so we cannot
1448  * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
1449  * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
1450  * the vmemmap pages.
1451  *
1452  * free_hpage_workfn() locklessly retrieves the linked list of pages to be
1453  * freed and frees them one-by-one. As the page->mapping pointer is going
1454  * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
1455  * structure of a lockless linked list of huge pages to be freed.
1456  */
1457 static LLIST_HEAD(hpage_freelist);
1458 
1459 static void free_hpage_workfn(struct work_struct *work)
1460 {
1461 	struct llist_node *node;
1462 
1463 	node = llist_del_all(&hpage_freelist);
1464 
1465 	while (node) {
1466 		struct page *page;
1467 		struct hstate *h;
1468 
1469 		page = container_of((struct address_space **)node,
1470 				     struct page, mapping);
1471 		node = node->next;
1472 		page->mapping = NULL;
1473 		/*
1474 		 * The VM_BUG_ON_PAGE(!PageHuge(page), page) in page_hstate()
1475 		 * is going to trigger because a previous call to
1476 		 * remove_hugetlb_page() will set_compound_page_dtor(page,
1477 		 * NULL_COMPOUND_DTOR), so do not use page_hstate() directly.
1478 		 */
1479 		h = size_to_hstate(page_size(page));
1480 
1481 		__update_and_free_page(h, page);
1482 
1483 		cond_resched();
1484 	}
1485 }
1486 static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
1487 
1488 static inline void flush_free_hpage_work(struct hstate *h)
1489 {
1490 	if (free_vmemmap_pages_per_hpage(h))
1491 		flush_work(&free_hpage_work);
1492 }
1493 
1494 static void update_and_free_page(struct hstate *h, struct page *page,
1495 				 bool atomic)
1496 {
1497 	if (!HPageVmemmapOptimized(page) || !atomic) {
1498 		__update_and_free_page(h, page);
1499 		return;
1500 	}
1501 
1502 	/*
1503 	 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
1504 	 *
1505 	 * Only call schedule_work() if hpage_freelist is previously
1506 	 * empty. Otherwise, schedule_work() had been called but the workfn
1507 	 * hasn't retrieved the list yet.
1508 	 */
1509 	if (llist_add((struct llist_node *)&page->mapping, &hpage_freelist))
1510 		schedule_work(&free_hpage_work);
1511 }
1512 
1513 static void update_and_free_pages_bulk(struct hstate *h, struct list_head *list)
1514 {
1515 	struct page *page, *t_page;
1516 
1517 	list_for_each_entry_safe(page, t_page, list, lru) {
1518 		update_and_free_page(h, page, false);
1519 		cond_resched();
1520 	}
1521 }
1522 
1523 struct hstate *size_to_hstate(unsigned long size)
1524 {
1525 	struct hstate *h;
1526 
1527 	for_each_hstate(h) {
1528 		if (huge_page_size(h) == size)
1529 			return h;
1530 	}
1531 	return NULL;
1532 }
1533 
1534 void free_huge_page(struct page *page)
1535 {
1536 	/*
1537 	 * Can't pass hstate in here because it is called from the
1538 	 * compound page destructor.
1539 	 */
1540 	struct hstate *h = page_hstate(page);
1541 	int nid = page_to_nid(page);
1542 	struct hugepage_subpool *spool = hugetlb_page_subpool(page);
1543 	bool restore_reserve;
1544 	unsigned long flags;
1545 
1546 	VM_BUG_ON_PAGE(page_count(page), page);
1547 	VM_BUG_ON_PAGE(page_mapcount(page), page);
1548 
1549 	hugetlb_set_page_subpool(page, NULL);
1550 	page->mapping = NULL;
1551 	restore_reserve = HPageRestoreReserve(page);
1552 	ClearHPageRestoreReserve(page);
1553 
1554 	/*
1555 	 * If HPageRestoreReserve was set on page, page allocation consumed a
1556 	 * reservation.  If the page was associated with a subpool, there
1557 	 * would have been a page reserved in the subpool before allocation
1558 	 * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1559 	 * reservation, do not call hugepage_subpool_put_pages() as this will
1560 	 * remove the reserved page from the subpool.
1561 	 */
1562 	if (!restore_reserve) {
1563 		/*
1564 		 * A return code of zero implies that the subpool will be
1565 		 * under its minimum size if the reservation is not restored
1566 		 * after page is free.  Therefore, force restore_reserve
1567 		 * operation.
1568 		 */
1569 		if (hugepage_subpool_put_pages(spool, 1) == 0)
1570 			restore_reserve = true;
1571 	}
1572 
1573 	spin_lock_irqsave(&hugetlb_lock, flags);
1574 	ClearHPageMigratable(page);
1575 	hugetlb_cgroup_uncharge_page(hstate_index(h),
1576 				     pages_per_huge_page(h), page);
1577 	hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h),
1578 					  pages_per_huge_page(h), page);
1579 	if (restore_reserve)
1580 		h->resv_huge_pages++;
1581 
1582 	if (HPageTemporary(page)) {
1583 		remove_hugetlb_page(h, page, false);
1584 		spin_unlock_irqrestore(&hugetlb_lock, flags);
1585 		update_and_free_page(h, page, true);
1586 	} else if (h->surplus_huge_pages_node[nid]) {
1587 		/* remove the page from active list */
1588 		remove_hugetlb_page(h, page, true);
1589 		spin_unlock_irqrestore(&hugetlb_lock, flags);
1590 		update_and_free_page(h, page, true);
1591 	} else {
1592 		arch_clear_hugepage_flags(page);
1593 		enqueue_huge_page(h, page);
1594 		spin_unlock_irqrestore(&hugetlb_lock, flags);
1595 	}
1596 }
1597 
1598 /*
1599  * Must be called with the hugetlb lock held
1600  */
1601 static void __prep_account_new_huge_page(struct hstate *h, int nid)
1602 {
1603 	lockdep_assert_held(&hugetlb_lock);
1604 	h->nr_huge_pages++;
1605 	h->nr_huge_pages_node[nid]++;
1606 }
1607 
1608 static void __prep_new_huge_page(struct hstate *h, struct page *page)
1609 {
1610 	free_huge_page_vmemmap(h, page);
1611 	INIT_LIST_HEAD(&page->lru);
1612 	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1613 	hugetlb_set_page_subpool(page, NULL);
1614 	set_hugetlb_cgroup(page, NULL);
1615 	set_hugetlb_cgroup_rsvd(page, NULL);
1616 }
1617 
1618 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1619 {
1620 	__prep_new_huge_page(h, page);
1621 	spin_lock_irq(&hugetlb_lock);
1622 	__prep_account_new_huge_page(h, nid);
1623 	spin_unlock_irq(&hugetlb_lock);
1624 }
1625 
1626 static bool prep_compound_gigantic_page(struct page *page, unsigned int order)
1627 {
1628 	int i, j;
1629 	int nr_pages = 1 << order;
1630 	struct page *p = page + 1;
1631 
1632 	/* we rely on prep_new_huge_page to set the destructor */
1633 	set_compound_order(page, order);
1634 	__ClearPageReserved(page);
1635 	__SetPageHead(page);
1636 	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1637 		/*
1638 		 * For gigantic hugepages allocated through bootmem at
1639 		 * boot, it's safer to be consistent with the not-gigantic
1640 		 * hugepages and clear the PG_reserved bit from all tail pages
1641 		 * too.  Otherwise drivers using get_user_pages() to access tail
1642 		 * pages may get the reference counting wrong if they see
1643 		 * PG_reserved set on a tail page (despite the head page not
1644 		 * having PG_reserved set).  Enforcing this consistency between
1645 		 * head and tail pages allows drivers to optimize away a check
1646 		 * on the head page when they need know if put_page() is needed
1647 		 * after get_user_pages().
1648 		 */
1649 		__ClearPageReserved(p);
1650 		/*
1651 		 * Subtle and very unlikely
1652 		 *
1653 		 * Gigantic 'page allocators' such as memblock or cma will
1654 		 * return a set of pages with each page ref counted.  We need
1655 		 * to turn this set of pages into a compound page with tail
1656 		 * page ref counts set to zero.  Code such as speculative page
1657 		 * cache adding could take a ref on a 'to be' tail page.
1658 		 * We need to respect any increased ref count, and only set
1659 		 * the ref count to zero if count is currently 1.  If count
1660 		 * is not 1, we call synchronize_rcu in the hope that a rcu
1661 		 * grace period will cause ref count to drop and then retry.
1662 		 * If count is still inflated on retry we return an error and
1663 		 * must discard the pages.
1664 		 */
1665 		if (!page_ref_freeze(p, 1)) {
1666 			pr_info("HugeTLB unexpected inflated ref count on freshly allocated page\n");
1667 			synchronize_rcu();
1668 			if (!page_ref_freeze(p, 1))
1669 				goto out_error;
1670 		}
1671 		set_page_count(p, 0);
1672 		set_compound_head(p, page);
1673 	}
1674 	atomic_set(compound_mapcount_ptr(page), -1);
1675 	atomic_set(compound_pincount_ptr(page), 0);
1676 	return true;
1677 
1678 out_error:
1679 	/* undo tail page modifications made above */
1680 	p = page + 1;
1681 	for (j = 1; j < i; j++, p = mem_map_next(p, page, j)) {
1682 		clear_compound_head(p);
1683 		set_page_refcounted(p);
1684 	}
1685 	/* need to clear PG_reserved on remaining tail pages  */
1686 	for (; j < nr_pages; j++, p = mem_map_next(p, page, j))
1687 		__ClearPageReserved(p);
1688 	set_compound_order(page, 0);
1689 	page[1].compound_nr = 0;
1690 	__ClearPageHead(page);
1691 	return false;
1692 }
1693 
1694 /*
1695  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1696  * transparent huge pages.  See the PageTransHuge() documentation for more
1697  * details.
1698  */
1699 int PageHuge(struct page *page)
1700 {
1701 	if (!PageCompound(page))
1702 		return 0;
1703 
1704 	page = compound_head(page);
1705 	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1706 }
1707 EXPORT_SYMBOL_GPL(PageHuge);
1708 
1709 /*
1710  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1711  * normal or transparent huge pages.
1712  */
1713 int PageHeadHuge(struct page *page_head)
1714 {
1715 	if (!PageHead(page_head))
1716 		return 0;
1717 
1718 	return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR;
1719 }
1720 
1721 /*
1722  * Find and lock address space (mapping) in write mode.
1723  *
1724  * Upon entry, the page is locked which means that page_mapping() is
1725  * stable.  Due to locking order, we can only trylock_write.  If we can
1726  * not get the lock, simply return NULL to caller.
1727  */
1728 struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage)
1729 {
1730 	struct address_space *mapping = page_mapping(hpage);
1731 
1732 	if (!mapping)
1733 		return mapping;
1734 
1735 	if (i_mmap_trylock_write(mapping))
1736 		return mapping;
1737 
1738 	return NULL;
1739 }
1740 
1741 pgoff_t hugetlb_basepage_index(struct page *page)
1742 {
1743 	struct page *page_head = compound_head(page);
1744 	pgoff_t index = page_index(page_head);
1745 	unsigned long compound_idx;
1746 
1747 	if (compound_order(page_head) >= MAX_ORDER)
1748 		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1749 	else
1750 		compound_idx = page - page_head;
1751 
1752 	return (index << compound_order(page_head)) + compound_idx;
1753 }
1754 
1755 static struct page *alloc_buddy_huge_page(struct hstate *h,
1756 		gfp_t gfp_mask, int nid, nodemask_t *nmask,
1757 		nodemask_t *node_alloc_noretry)
1758 {
1759 	int order = huge_page_order(h);
1760 	struct page *page;
1761 	bool alloc_try_hard = true;
1762 
1763 	/*
1764 	 * By default we always try hard to allocate the page with
1765 	 * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
1766 	 * a loop (to adjust global huge page counts) and previous allocation
1767 	 * failed, do not continue to try hard on the same node.  Use the
1768 	 * node_alloc_noretry bitmap to manage this state information.
1769 	 */
1770 	if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
1771 		alloc_try_hard = false;
1772 	gfp_mask |= __GFP_COMP|__GFP_NOWARN;
1773 	if (alloc_try_hard)
1774 		gfp_mask |= __GFP_RETRY_MAYFAIL;
1775 	if (nid == NUMA_NO_NODE)
1776 		nid = numa_mem_id();
1777 	page = __alloc_pages(gfp_mask, order, nid, nmask);
1778 	if (page)
1779 		__count_vm_event(HTLB_BUDDY_PGALLOC);
1780 	else
1781 		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1782 
1783 	/*
1784 	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
1785 	 * indicates an overall state change.  Clear bit so that we resume
1786 	 * normal 'try hard' allocations.
1787 	 */
1788 	if (node_alloc_noretry && page && !alloc_try_hard)
1789 		node_clear(nid, *node_alloc_noretry);
1790 
1791 	/*
1792 	 * If we tried hard to get a page but failed, set bit so that
1793 	 * subsequent attempts will not try as hard until there is an
1794 	 * overall state change.
1795 	 */
1796 	if (node_alloc_noretry && !page && alloc_try_hard)
1797 		node_set(nid, *node_alloc_noretry);
1798 
1799 	return page;
1800 }
1801 
1802 /*
1803  * Common helper to allocate a fresh hugetlb page. All specific allocators
1804  * should use this function to get new hugetlb pages
1805  */
1806 static struct page *alloc_fresh_huge_page(struct hstate *h,
1807 		gfp_t gfp_mask, int nid, nodemask_t *nmask,
1808 		nodemask_t *node_alloc_noretry)
1809 {
1810 	struct page *page;
1811 	bool retry = false;
1812 
1813 retry:
1814 	if (hstate_is_gigantic(h))
1815 		page = alloc_gigantic_page(h, gfp_mask, nid, nmask);
1816 	else
1817 		page = alloc_buddy_huge_page(h, gfp_mask,
1818 				nid, nmask, node_alloc_noretry);
1819 	if (!page)
1820 		return NULL;
1821 
1822 	if (hstate_is_gigantic(h)) {
1823 		if (!prep_compound_gigantic_page(page, huge_page_order(h))) {
1824 			/*
1825 			 * Rare failure to convert pages to compound page.
1826 			 * Free pages and try again - ONCE!
1827 			 */
1828 			free_gigantic_page(page, huge_page_order(h));
1829 			if (!retry) {
1830 				retry = true;
1831 				goto retry;
1832 			}
1833 			pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n");
1834 			return NULL;
1835 		}
1836 	}
1837 	prep_new_huge_page(h, page, page_to_nid(page));
1838 
1839 	return page;
1840 }
1841 
1842 /*
1843  * Allocates a fresh page to the hugetlb allocator pool in the node interleaved
1844  * manner.
1845  */
1846 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1847 				nodemask_t *node_alloc_noretry)
1848 {
1849 	struct page *page;
1850 	int nr_nodes, node;
1851 	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
1852 
1853 	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
1854 		page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed,
1855 						node_alloc_noretry);
1856 		if (page)
1857 			break;
1858 	}
1859 
1860 	if (!page)
1861 		return 0;
1862 
1863 	put_page(page); /* free it into the hugepage allocator */
1864 
1865 	return 1;
1866 }
1867 
1868 /*
1869  * Remove huge page from pool from next node to free.  Attempt to keep
1870  * persistent huge pages more or less balanced over allowed nodes.
1871  * This routine only 'removes' the hugetlb page.  The caller must make
1872  * an additional call to free the page to low level allocators.
1873  * Called with hugetlb_lock locked.
1874  */
1875 static struct page *remove_pool_huge_page(struct hstate *h,
1876 						nodemask_t *nodes_allowed,
1877 						 bool acct_surplus)
1878 {
1879 	int nr_nodes, node;
1880 	struct page *page = NULL;
1881 
1882 	lockdep_assert_held(&hugetlb_lock);
1883 	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
1884 		/*
1885 		 * If we're returning unused surplus pages, only examine
1886 		 * nodes with surplus pages.
1887 		 */
1888 		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
1889 		    !list_empty(&h->hugepage_freelists[node])) {
1890 			page = list_entry(h->hugepage_freelists[node].next,
1891 					  struct page, lru);
1892 			remove_hugetlb_page(h, page, acct_surplus);
1893 			break;
1894 		}
1895 	}
1896 
1897 	return page;
1898 }
1899 
1900 /*
1901  * Dissolve a given free hugepage into free buddy pages. This function does
1902  * nothing for in-use hugepages and non-hugepages.
1903  * This function returns values like below:
1904  *
1905  *  -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
1906  *           when the system is under memory pressure and the feature of
1907  *           freeing unused vmemmap pages associated with each hugetlb page
1908  *           is enabled.
1909  *  -EBUSY:  failed to dissolved free hugepages or the hugepage is in-use
1910  *           (allocated or reserved.)
1911  *       0:  successfully dissolved free hugepages or the page is not a
1912  *           hugepage (considered as already dissolved)
1913  */
1914 int dissolve_free_huge_page(struct page *page)
1915 {
1916 	int rc = -EBUSY;
1917 
1918 retry:
1919 	/* Not to disrupt normal path by vainly holding hugetlb_lock */
1920 	if (!PageHuge(page))
1921 		return 0;
1922 
1923 	spin_lock_irq(&hugetlb_lock);
1924 	if (!PageHuge(page)) {
1925 		rc = 0;
1926 		goto out;
1927 	}
1928 
1929 	if (!page_count(page)) {
1930 		struct page *head = compound_head(page);
1931 		struct hstate *h = page_hstate(head);
1932 		if (h->free_huge_pages - h->resv_huge_pages == 0)
1933 			goto out;
1934 
1935 		/*
1936 		 * We should make sure that the page is already on the free list
1937 		 * when it is dissolved.
1938 		 */
1939 		if (unlikely(!HPageFreed(head))) {
1940 			spin_unlock_irq(&hugetlb_lock);
1941 			cond_resched();
1942 
1943 			/*
1944 			 * Theoretically, we should return -EBUSY when we
1945 			 * encounter this race. In fact, we have a chance
1946 			 * to successfully dissolve the page if we do a
1947 			 * retry. Because the race window is quite small.
1948 			 * If we seize this opportunity, it is an optimization
1949 			 * for increasing the success rate of dissolving page.
1950 			 */
1951 			goto retry;
1952 		}
1953 
1954 		remove_hugetlb_page(h, head, false);
1955 		h->max_huge_pages--;
1956 		spin_unlock_irq(&hugetlb_lock);
1957 
1958 		/*
1959 		 * Normally update_and_free_page will allocate required vmemmmap
1960 		 * before freeing the page.  update_and_free_page will fail to
1961 		 * free the page if it can not allocate required vmemmap.  We
1962 		 * need to adjust max_huge_pages if the page is not freed.
1963 		 * Attempt to allocate vmemmmap here so that we can take
1964 		 * appropriate action on failure.
1965 		 */
1966 		rc = alloc_huge_page_vmemmap(h, head);
1967 		if (!rc) {
1968 			/*
1969 			 * Move PageHWPoison flag from head page to the raw
1970 			 * error page, which makes any subpages rather than
1971 			 * the error page reusable.
1972 			 */
1973 			if (PageHWPoison(head) && page != head) {
1974 				SetPageHWPoison(page);
1975 				ClearPageHWPoison(head);
1976 			}
1977 			update_and_free_page(h, head, false);
1978 		} else {
1979 			spin_lock_irq(&hugetlb_lock);
1980 			add_hugetlb_page(h, head, false);
1981 			h->max_huge_pages++;
1982 			spin_unlock_irq(&hugetlb_lock);
1983 		}
1984 
1985 		return rc;
1986 	}
1987 out:
1988 	spin_unlock_irq(&hugetlb_lock);
1989 	return rc;
1990 }
1991 
1992 /*
1993  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
1994  * make specified memory blocks removable from the system.
1995  * Note that this will dissolve a free gigantic hugepage completely, if any
1996  * part of it lies within the given range.
1997  * Also note that if dissolve_free_huge_page() returns with an error, all
1998  * free hugepages that were dissolved before that error are lost.
1999  */
2000 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
2001 {
2002 	unsigned long pfn;
2003 	struct page *page;
2004 	int rc = 0;
2005 
2006 	if (!hugepages_supported())
2007 		return rc;
2008 
2009 	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
2010 		page = pfn_to_page(pfn);
2011 		rc = dissolve_free_huge_page(page);
2012 		if (rc)
2013 			break;
2014 	}
2015 
2016 	return rc;
2017 }
2018 
2019 /*
2020  * Allocates a fresh surplus page from the page allocator.
2021  */
2022 static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
2023 		int nid, nodemask_t *nmask)
2024 {
2025 	struct page *page = NULL;
2026 
2027 	if (hstate_is_gigantic(h))
2028 		return NULL;
2029 
2030 	spin_lock_irq(&hugetlb_lock);
2031 	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
2032 		goto out_unlock;
2033 	spin_unlock_irq(&hugetlb_lock);
2034 
2035 	page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
2036 	if (!page)
2037 		return NULL;
2038 
2039 	spin_lock_irq(&hugetlb_lock);
2040 	/*
2041 	 * We could have raced with the pool size change.
2042 	 * Double check that and simply deallocate the new page
2043 	 * if we would end up overcommiting the surpluses. Abuse
2044 	 * temporary page to workaround the nasty free_huge_page
2045 	 * codeflow
2046 	 */
2047 	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
2048 		SetHPageTemporary(page);
2049 		spin_unlock_irq(&hugetlb_lock);
2050 		put_page(page);
2051 		return NULL;
2052 	} else {
2053 		h->surplus_huge_pages++;
2054 		h->surplus_huge_pages_node[page_to_nid(page)]++;
2055 	}
2056 
2057 out_unlock:
2058 	spin_unlock_irq(&hugetlb_lock);
2059 
2060 	return page;
2061 }
2062 
2063 static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
2064 				     int nid, nodemask_t *nmask)
2065 {
2066 	struct page *page;
2067 
2068 	if (hstate_is_gigantic(h))
2069 		return NULL;
2070 
2071 	page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
2072 	if (!page)
2073 		return NULL;
2074 
2075 	/*
2076 	 * We do not account these pages as surplus because they are only
2077 	 * temporary and will be released properly on the last reference
2078 	 */
2079 	SetHPageTemporary(page);
2080 
2081 	return page;
2082 }
2083 
2084 /*
2085  * Use the VMA's mpolicy to allocate a huge page from the buddy.
2086  */
2087 static
2088 struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h,
2089 		struct vm_area_struct *vma, unsigned long addr)
2090 {
2091 	struct page *page;
2092 	struct mempolicy *mpol;
2093 	gfp_t gfp_mask = htlb_alloc_mask(h);
2094 	int nid;
2095 	nodemask_t *nodemask;
2096 
2097 	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
2098 	page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask);
2099 	mpol_cond_put(mpol);
2100 
2101 	return page;
2102 }
2103 
2104 /* page migration callback function */
2105 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
2106 		nodemask_t *nmask, gfp_t gfp_mask)
2107 {
2108 	spin_lock_irq(&hugetlb_lock);
2109 	if (h->free_huge_pages - h->resv_huge_pages > 0) {
2110 		struct page *page;
2111 
2112 		page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
2113 		if (page) {
2114 			spin_unlock_irq(&hugetlb_lock);
2115 			return page;
2116 		}
2117 	}
2118 	spin_unlock_irq(&hugetlb_lock);
2119 
2120 	return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask);
2121 }
2122 
2123 /* mempolicy aware migration callback */
2124 struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
2125 		unsigned long address)
2126 {
2127 	struct mempolicy *mpol;
2128 	nodemask_t *nodemask;
2129 	struct page *page;
2130 	gfp_t gfp_mask;
2131 	int node;
2132 
2133 	gfp_mask = htlb_alloc_mask(h);
2134 	node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
2135 	page = alloc_huge_page_nodemask(h, node, nodemask, gfp_mask);
2136 	mpol_cond_put(mpol);
2137 
2138 	return page;
2139 }
2140 
2141 /*
2142  * Increase the hugetlb pool such that it can accommodate a reservation
2143  * of size 'delta'.
2144  */
2145 static int gather_surplus_pages(struct hstate *h, long delta)
2146 	__must_hold(&hugetlb_lock)
2147 {
2148 	struct list_head surplus_list;
2149 	struct page *page, *tmp;
2150 	int ret;
2151 	long i;
2152 	long needed, allocated;
2153 	bool alloc_ok = true;
2154 
2155 	lockdep_assert_held(&hugetlb_lock);
2156 	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
2157 	if (needed <= 0) {
2158 		h->resv_huge_pages += delta;
2159 		return 0;
2160 	}
2161 
2162 	allocated = 0;
2163 	INIT_LIST_HEAD(&surplus_list);
2164 
2165 	ret = -ENOMEM;
2166 retry:
2167 	spin_unlock_irq(&hugetlb_lock);
2168 	for (i = 0; i < needed; i++) {
2169 		page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),
2170 				NUMA_NO_NODE, NULL);
2171 		if (!page) {
2172 			alloc_ok = false;
2173 			break;
2174 		}
2175 		list_add(&page->lru, &surplus_list);
2176 		cond_resched();
2177 	}
2178 	allocated += i;
2179 
2180 	/*
2181 	 * After retaking hugetlb_lock, we need to recalculate 'needed'
2182 	 * because either resv_huge_pages or free_huge_pages may have changed.
2183 	 */
2184 	spin_lock_irq(&hugetlb_lock);
2185 	needed = (h->resv_huge_pages + delta) -
2186 			(h->free_huge_pages + allocated);
2187 	if (needed > 0) {
2188 		if (alloc_ok)
2189 			goto retry;
2190 		/*
2191 		 * We were not able to allocate enough pages to
2192 		 * satisfy the entire reservation so we free what
2193 		 * we've allocated so far.
2194 		 */
2195 		goto free;
2196 	}
2197 	/*
2198 	 * The surplus_list now contains _at_least_ the number of extra pages
2199 	 * needed to accommodate the reservation.  Add the appropriate number
2200 	 * of pages to the hugetlb pool and free the extras back to the buddy
2201 	 * allocator.  Commit the entire reservation here to prevent another
2202 	 * process from stealing the pages as they are added to the pool but
2203 	 * before they are reserved.
2204 	 */
2205 	needed += allocated;
2206 	h->resv_huge_pages += delta;
2207 	ret = 0;
2208 
2209 	/* Free the needed pages to the hugetlb pool */
2210 	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
2211 		int zeroed;
2212 
2213 		if ((--needed) < 0)
2214 			break;
2215 		/*
2216 		 * This page is now managed by the hugetlb allocator and has
2217 		 * no users -- drop the buddy allocator's reference.
2218 		 */
2219 		zeroed = put_page_testzero(page);
2220 		VM_BUG_ON_PAGE(!zeroed, page);
2221 		enqueue_huge_page(h, page);
2222 	}
2223 free:
2224 	spin_unlock_irq(&hugetlb_lock);
2225 
2226 	/* Free unnecessary surplus pages to the buddy allocator */
2227 	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
2228 		put_page(page);
2229 	spin_lock_irq(&hugetlb_lock);
2230 
2231 	return ret;
2232 }
2233 
2234 /*
2235  * This routine has two main purposes:
2236  * 1) Decrement the reservation count (resv_huge_pages) by the value passed
2237  *    in unused_resv_pages.  This corresponds to the prior adjustments made
2238  *    to the associated reservation map.
2239  * 2) Free any unused surplus pages that may have been allocated to satisfy
2240  *    the reservation.  As many as unused_resv_pages may be freed.
2241  */
2242 static void return_unused_surplus_pages(struct hstate *h,
2243 					unsigned long unused_resv_pages)
2244 {
2245 	unsigned long nr_pages;
2246 	struct page *page;
2247 	LIST_HEAD(page_list);
2248 
2249 	lockdep_assert_held(&hugetlb_lock);
2250 	/* Uncommit the reservation */
2251 	h->resv_huge_pages -= unused_resv_pages;
2252 
2253 	/* Cannot return gigantic pages currently */
2254 	if (hstate_is_gigantic(h))
2255 		goto out;
2256 
2257 	/*
2258 	 * Part (or even all) of the reservation could have been backed
2259 	 * by pre-allocated pages. Only free surplus pages.
2260 	 */
2261 	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
2262 
2263 	/*
2264 	 * We want to release as many surplus pages as possible, spread
2265 	 * evenly across all nodes with memory. Iterate across these nodes
2266 	 * until we can no longer free unreserved surplus pages. This occurs
2267 	 * when the nodes with surplus pages have no free pages.
2268 	 * remove_pool_huge_page() will balance the freed pages across the
2269 	 * on-line nodes with memory and will handle the hstate accounting.
2270 	 */
2271 	while (nr_pages--) {
2272 		page = remove_pool_huge_page(h, &node_states[N_MEMORY], 1);
2273 		if (!page)
2274 			goto out;
2275 
2276 		list_add(&page->lru, &page_list);
2277 	}
2278 
2279 out:
2280 	spin_unlock_irq(&hugetlb_lock);
2281 	update_and_free_pages_bulk(h, &page_list);
2282 	spin_lock_irq(&hugetlb_lock);
2283 }
2284 
2285 
2286 /*
2287  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
2288  * are used by the huge page allocation routines to manage reservations.
2289  *
2290  * vma_needs_reservation is called to determine if the huge page at addr
2291  * within the vma has an associated reservation.  If a reservation is
2292  * needed, the value 1 is returned.  The caller is then responsible for
2293  * managing the global reservation and subpool usage counts.  After
2294  * the huge page has been allocated, vma_commit_reservation is called
2295  * to add the page to the reservation map.  If the page allocation fails,
2296  * the reservation must be ended instead of committed.  vma_end_reservation
2297  * is called in such cases.
2298  *
2299  * In the normal case, vma_commit_reservation returns the same value
2300  * as the preceding vma_needs_reservation call.  The only time this
2301  * is not the case is if a reserve map was changed between calls.  It
2302  * is the responsibility of the caller to notice the difference and
2303  * take appropriate action.
2304  *
2305  * vma_add_reservation is used in error paths where a reservation must
2306  * be restored when a newly allocated huge page must be freed.  It is
2307  * to be called after calling vma_needs_reservation to determine if a
2308  * reservation exists.
2309  *
2310  * vma_del_reservation is used in error paths where an entry in the reserve
2311  * map was created during huge page allocation and must be removed.  It is to
2312  * be called after calling vma_needs_reservation to determine if a reservation
2313  * exists.
2314  */
2315 enum vma_resv_mode {
2316 	VMA_NEEDS_RESV,
2317 	VMA_COMMIT_RESV,
2318 	VMA_END_RESV,
2319 	VMA_ADD_RESV,
2320 	VMA_DEL_RESV,
2321 };
2322 static long __vma_reservation_common(struct hstate *h,
2323 				struct vm_area_struct *vma, unsigned long addr,
2324 				enum vma_resv_mode mode)
2325 {
2326 	struct resv_map *resv;
2327 	pgoff_t idx;
2328 	long ret;
2329 	long dummy_out_regions_needed;
2330 
2331 	resv = vma_resv_map(vma);
2332 	if (!resv)
2333 		return 1;
2334 
2335 	idx = vma_hugecache_offset(h, vma, addr);
2336 	switch (mode) {
2337 	case VMA_NEEDS_RESV:
2338 		ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
2339 		/* We assume that vma_reservation_* routines always operate on
2340 		 * 1 page, and that adding to resv map a 1 page entry can only
2341 		 * ever require 1 region.
2342 		 */
2343 		VM_BUG_ON(dummy_out_regions_needed != 1);
2344 		break;
2345 	case VMA_COMMIT_RESV:
2346 		ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2347 		/* region_add calls of range 1 should never fail. */
2348 		VM_BUG_ON(ret < 0);
2349 		break;
2350 	case VMA_END_RESV:
2351 		region_abort(resv, idx, idx + 1, 1);
2352 		ret = 0;
2353 		break;
2354 	case VMA_ADD_RESV:
2355 		if (vma->vm_flags & VM_MAYSHARE) {
2356 			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2357 			/* region_add calls of range 1 should never fail. */
2358 			VM_BUG_ON(ret < 0);
2359 		} else {
2360 			region_abort(resv, idx, idx + 1, 1);
2361 			ret = region_del(resv, idx, idx + 1);
2362 		}
2363 		break;
2364 	case VMA_DEL_RESV:
2365 		if (vma->vm_flags & VM_MAYSHARE) {
2366 			region_abort(resv, idx, idx + 1, 1);
2367 			ret = region_del(resv, idx, idx + 1);
2368 		} else {
2369 			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2370 			/* region_add calls of range 1 should never fail. */
2371 			VM_BUG_ON(ret < 0);
2372 		}
2373 		break;
2374 	default:
2375 		BUG();
2376 	}
2377 
2378 	if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
2379 		return ret;
2380 	/*
2381 	 * We know private mapping must have HPAGE_RESV_OWNER set.
2382 	 *
2383 	 * In most cases, reserves always exist for private mappings.
2384 	 * However, a file associated with mapping could have been
2385 	 * hole punched or truncated after reserves were consumed.
2386 	 * As subsequent fault on such a range will not use reserves.
2387 	 * Subtle - The reserve map for private mappings has the
2388 	 * opposite meaning than that of shared mappings.  If NO
2389 	 * entry is in the reserve map, it means a reservation exists.
2390 	 * If an entry exists in the reserve map, it means the
2391 	 * reservation has already been consumed.  As a result, the
2392 	 * return value of this routine is the opposite of the
2393 	 * value returned from reserve map manipulation routines above.
2394 	 */
2395 	if (ret > 0)
2396 		return 0;
2397 	if (ret == 0)
2398 		return 1;
2399 	return ret;
2400 }
2401 
2402 static long vma_needs_reservation(struct hstate *h,
2403 			struct vm_area_struct *vma, unsigned long addr)
2404 {
2405 	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2406 }
2407 
2408 static long vma_commit_reservation(struct hstate *h,
2409 			struct vm_area_struct *vma, unsigned long addr)
2410 {
2411 	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
2412 }
2413 
2414 static void vma_end_reservation(struct hstate *h,
2415 			struct vm_area_struct *vma, unsigned long addr)
2416 {
2417 	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2418 }
2419 
2420 static long vma_add_reservation(struct hstate *h,
2421 			struct vm_area_struct *vma, unsigned long addr)
2422 {
2423 	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2424 }
2425 
2426 static long vma_del_reservation(struct hstate *h,
2427 			struct vm_area_struct *vma, unsigned long addr)
2428 {
2429 	return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
2430 }
2431 
2432 /*
2433  * This routine is called to restore reservation information on error paths.
2434  * It should ONLY be called for pages allocated via alloc_huge_page(), and
2435  * the hugetlb mutex should remain held when calling this routine.
2436  *
2437  * It handles two specific cases:
2438  * 1) A reservation was in place and the page consumed the reservation.
2439  *    HPageRestoreReserve is set in the page.
2440  * 2) No reservation was in place for the page, so HPageRestoreReserve is
2441  *    not set.  However, alloc_huge_page always updates the reserve map.
2442  *
2443  * In case 1, free_huge_page later in the error path will increment the
2444  * global reserve count.  But, free_huge_page does not have enough context
2445  * to adjust the reservation map.  This case deals primarily with private
2446  * mappings.  Adjust the reserve map here to be consistent with global
2447  * reserve count adjustments to be made by free_huge_page.  Make sure the
2448  * reserve map indicates there is a reservation present.
2449  *
2450  * In case 2, simply undo reserve map modifications done by alloc_huge_page.
2451  */
2452 void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
2453 			unsigned long address, struct page *page)
2454 {
2455 	long rc = vma_needs_reservation(h, vma, address);
2456 
2457 	if (HPageRestoreReserve(page)) {
2458 		if (unlikely(rc < 0))
2459 			/*
2460 			 * Rare out of memory condition in reserve map
2461 			 * manipulation.  Clear HPageRestoreReserve so that
2462 			 * global reserve count will not be incremented
2463 			 * by free_huge_page.  This will make it appear
2464 			 * as though the reservation for this page was
2465 			 * consumed.  This may prevent the task from
2466 			 * faulting in the page at a later time.  This
2467 			 * is better than inconsistent global huge page
2468 			 * accounting of reserve counts.
2469 			 */
2470 			ClearHPageRestoreReserve(page);
2471 		else if (rc)
2472 			(void)vma_add_reservation(h, vma, address);
2473 		else
2474 			vma_end_reservation(h, vma, address);
2475 	} else {
2476 		if (!rc) {
2477 			/*
2478 			 * This indicates there is an entry in the reserve map
2479 			 * added by alloc_huge_page.  We know it was added
2480 			 * before the alloc_huge_page call, otherwise
2481 			 * HPageRestoreReserve would be set on the page.
2482 			 * Remove the entry so that a subsequent allocation
2483 			 * does not consume a reservation.
2484 			 */
2485 			rc = vma_del_reservation(h, vma, address);
2486 			if (rc < 0)
2487 				/*
2488 				 * VERY rare out of memory condition.  Since
2489 				 * we can not delete the entry, set
2490 				 * HPageRestoreReserve so that the reserve
2491 				 * count will be incremented when the page
2492 				 * is freed.  This reserve will be consumed
2493 				 * on a subsequent allocation.
2494 				 */
2495 				SetHPageRestoreReserve(page);
2496 		} else if (rc < 0) {
2497 			/*
2498 			 * Rare out of memory condition from
2499 			 * vma_needs_reservation call.  Memory allocation is
2500 			 * only attempted if a new entry is needed.  Therefore,
2501 			 * this implies there is not an entry in the
2502 			 * reserve map.
2503 			 *
2504 			 * For shared mappings, no entry in the map indicates
2505 			 * no reservation.  We are done.
2506 			 */
2507 			if (!(vma->vm_flags & VM_MAYSHARE))
2508 				/*
2509 				 * For private mappings, no entry indicates
2510 				 * a reservation is present.  Since we can
2511 				 * not add an entry, set SetHPageRestoreReserve
2512 				 * on the page so reserve count will be
2513 				 * incremented when freed.  This reserve will
2514 				 * be consumed on a subsequent allocation.
2515 				 */
2516 				SetHPageRestoreReserve(page);
2517 		} else
2518 			/*
2519 			 * No reservation present, do nothing
2520 			 */
2521 			 vma_end_reservation(h, vma, address);
2522 	}
2523 }
2524 
2525 /*
2526  * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one
2527  * @h: struct hstate old page belongs to
2528  * @old_page: Old page to dissolve
2529  * @list: List to isolate the page in case we need to
2530  * Returns 0 on success, otherwise negated error.
2531  */
2532 static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
2533 					struct list_head *list)
2534 {
2535 	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2536 	int nid = page_to_nid(old_page);
2537 	struct page *new_page;
2538 	int ret = 0;
2539 
2540 	/*
2541 	 * Before dissolving the page, we need to allocate a new one for the
2542 	 * pool to remain stable.  Here, we allocate the page and 'prep' it
2543 	 * by doing everything but actually updating counters and adding to
2544 	 * the pool.  This simplifies and let us do most of the processing
2545 	 * under the lock.
2546 	 */
2547 	new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL);
2548 	if (!new_page)
2549 		return -ENOMEM;
2550 	__prep_new_huge_page(h, new_page);
2551 
2552 retry:
2553 	spin_lock_irq(&hugetlb_lock);
2554 	if (!PageHuge(old_page)) {
2555 		/*
2556 		 * Freed from under us. Drop new_page too.
2557 		 */
2558 		goto free_new;
2559 	} else if (page_count(old_page)) {
2560 		/*
2561 		 * Someone has grabbed the page, try to isolate it here.
2562 		 * Fail with -EBUSY if not possible.
2563 		 */
2564 		spin_unlock_irq(&hugetlb_lock);
2565 		if (!isolate_huge_page(old_page, list))
2566 			ret = -EBUSY;
2567 		spin_lock_irq(&hugetlb_lock);
2568 		goto free_new;
2569 	} else if (!HPageFreed(old_page)) {
2570 		/*
2571 		 * Page's refcount is 0 but it has not been enqueued in the
2572 		 * freelist yet. Race window is small, so we can succeed here if
2573 		 * we retry.
2574 		 */
2575 		spin_unlock_irq(&hugetlb_lock);
2576 		cond_resched();
2577 		goto retry;
2578 	} else {
2579 		/*
2580 		 * Ok, old_page is still a genuine free hugepage. Remove it from
2581 		 * the freelist and decrease the counters. These will be
2582 		 * incremented again when calling __prep_account_new_huge_page()
2583 		 * and enqueue_huge_page() for new_page. The counters will remain
2584 		 * stable since this happens under the lock.
2585 		 */
2586 		remove_hugetlb_page(h, old_page, false);
2587 
2588 		/*
2589 		 * Reference count trick is needed because allocator gives us
2590 		 * referenced page but the pool requires pages with 0 refcount.
2591 		 */
2592 		__prep_account_new_huge_page(h, nid);
2593 		page_ref_dec(new_page);
2594 		enqueue_huge_page(h, new_page);
2595 
2596 		/*
2597 		 * Pages have been replaced, we can safely free the old one.
2598 		 */
2599 		spin_unlock_irq(&hugetlb_lock);
2600 		update_and_free_page(h, old_page, false);
2601 	}
2602 
2603 	return ret;
2604 
2605 free_new:
2606 	spin_unlock_irq(&hugetlb_lock);
2607 	update_and_free_page(h, new_page, false);
2608 
2609 	return ret;
2610 }
2611 
2612 int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
2613 {
2614 	struct hstate *h;
2615 	struct page *head;
2616 	int ret = -EBUSY;
2617 
2618 	/*
2619 	 * The page might have been dissolved from under our feet, so make sure
2620 	 * to carefully check the state under the lock.
2621 	 * Return success when racing as if we dissolved the page ourselves.
2622 	 */
2623 	spin_lock_irq(&hugetlb_lock);
2624 	if (PageHuge(page)) {
2625 		head = compound_head(page);
2626 		h = page_hstate(head);
2627 	} else {
2628 		spin_unlock_irq(&hugetlb_lock);
2629 		return 0;
2630 	}
2631 	spin_unlock_irq(&hugetlb_lock);
2632 
2633 	/*
2634 	 * Fence off gigantic pages as there is a cyclic dependency between
2635 	 * alloc_contig_range and them. Return -ENOMEM as this has the effect
2636 	 * of bailing out right away without further retrying.
2637 	 */
2638 	if (hstate_is_gigantic(h))
2639 		return -ENOMEM;
2640 
2641 	if (page_count(head) && isolate_huge_page(head, list))
2642 		ret = 0;
2643 	else if (!page_count(head))
2644 		ret = alloc_and_dissolve_huge_page(h, head, list);
2645 
2646 	return ret;
2647 }
2648 
2649 struct page *alloc_huge_page(struct vm_area_struct *vma,
2650 				    unsigned long addr, int avoid_reserve)
2651 {
2652 	struct hugepage_subpool *spool = subpool_vma(vma);
2653 	struct hstate *h = hstate_vma(vma);
2654 	struct page *page;
2655 	long map_chg, map_commit;
2656 	long gbl_chg;
2657 	int ret, idx;
2658 	struct hugetlb_cgroup *h_cg;
2659 	bool deferred_reserve;
2660 
2661 	idx = hstate_index(h);
2662 	/*
2663 	 * Examine the region/reserve map to determine if the process
2664 	 * has a reservation for the page to be allocated.  A return
2665 	 * code of zero indicates a reservation exists (no change).
2666 	 */
2667 	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2668 	if (map_chg < 0)
2669 		return ERR_PTR(-ENOMEM);
2670 
2671 	/*
2672 	 * Processes that did not create the mapping will have no
2673 	 * reserves as indicated by the region/reserve map. Check
2674 	 * that the allocation will not exceed the subpool limit.
2675 	 * Allocations for MAP_NORESERVE mappings also need to be
2676 	 * checked against any subpool limit.
2677 	 */
2678 	if (map_chg || avoid_reserve) {
2679 		gbl_chg = hugepage_subpool_get_pages(spool, 1);
2680 		if (gbl_chg < 0) {
2681 			vma_end_reservation(h, vma, addr);
2682 			return ERR_PTR(-ENOSPC);
2683 		}
2684 
2685 		/*
2686 		 * Even though there was no reservation in the region/reserve
2687 		 * map, there could be reservations associated with the
2688 		 * subpool that can be used.  This would be indicated if the
2689 		 * return value of hugepage_subpool_get_pages() is zero.
2690 		 * However, if avoid_reserve is specified we still avoid even
2691 		 * the subpool reservations.
2692 		 */
2693 		if (avoid_reserve)
2694 			gbl_chg = 1;
2695 	}
2696 
2697 	/* If this allocation is not consuming a reservation, charge it now.
2698 	 */
2699 	deferred_reserve = map_chg || avoid_reserve;
2700 	if (deferred_reserve) {
2701 		ret = hugetlb_cgroup_charge_cgroup_rsvd(
2702 			idx, pages_per_huge_page(h), &h_cg);
2703 		if (ret)
2704 			goto out_subpool_put;
2705 	}
2706 
2707 	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2708 	if (ret)
2709 		goto out_uncharge_cgroup_reservation;
2710 
2711 	spin_lock_irq(&hugetlb_lock);
2712 	/*
2713 	 * glb_chg is passed to indicate whether or not a page must be taken
2714 	 * from the global free pool (global change).  gbl_chg == 0 indicates
2715 	 * a reservation exists for the allocation.
2716 	 */
2717 	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2718 	if (!page) {
2719 		spin_unlock_irq(&hugetlb_lock);
2720 		page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
2721 		if (!page)
2722 			goto out_uncharge_cgroup;
2723 		if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2724 			SetHPageRestoreReserve(page);
2725 			h->resv_huge_pages--;
2726 		}
2727 		spin_lock_irq(&hugetlb_lock);
2728 		list_add(&page->lru, &h->hugepage_activelist);
2729 		/* Fall through */
2730 	}
2731 	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2732 	/* If allocation is not consuming a reservation, also store the
2733 	 * hugetlb_cgroup pointer on the page.
2734 	 */
2735 	if (deferred_reserve) {
2736 		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
2737 						  h_cg, page);
2738 	}
2739 
2740 	spin_unlock_irq(&hugetlb_lock);
2741 
2742 	hugetlb_set_page_subpool(page, spool);
2743 
2744 	map_commit = vma_commit_reservation(h, vma, addr);
2745 	if (unlikely(map_chg > map_commit)) {
2746 		/*
2747 		 * The page was added to the reservation map between
2748 		 * vma_needs_reservation and vma_commit_reservation.
2749 		 * This indicates a race with hugetlb_reserve_pages.
2750 		 * Adjust for the subpool count incremented above AND
2751 		 * in hugetlb_reserve_pages for the same page.  Also,
2752 		 * the reservation count added in hugetlb_reserve_pages
2753 		 * no longer applies.
2754 		 */
2755 		long rsv_adjust;
2756 
2757 		rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2758 		hugetlb_acct_memory(h, -rsv_adjust);
2759 		if (deferred_reserve)
2760 			hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h),
2761 					pages_per_huge_page(h), page);
2762 	}
2763 	return page;
2764 
2765 out_uncharge_cgroup:
2766 	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2767 out_uncharge_cgroup_reservation:
2768 	if (deferred_reserve)
2769 		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
2770 						    h_cg);
2771 out_subpool_put:
2772 	if (map_chg || avoid_reserve)
2773 		hugepage_subpool_put_pages(spool, 1);
2774 	vma_end_reservation(h, vma, addr);
2775 	return ERR_PTR(-ENOSPC);
2776 }
2777 
2778 int alloc_bootmem_huge_page(struct hstate *h)
2779 	__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
2780 int __alloc_bootmem_huge_page(struct hstate *h)
2781 {
2782 	struct huge_bootmem_page *m;
2783 	int nr_nodes, node;
2784 
2785 	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
2786 		void *addr;
2787 
2788 		addr = memblock_alloc_try_nid_raw(
2789 				huge_page_size(h), huge_page_size(h),
2790 				0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
2791 		if (addr) {
2792 			/*
2793 			 * Use the beginning of the huge page to store the
2794 			 * huge_bootmem_page struct (until gather_bootmem
2795 			 * puts them into the mem_map).
2796 			 */
2797 			m = addr;
2798 			goto found;
2799 		}
2800 	}
2801 	return 0;
2802 
2803 found:
2804 	BUG_ON(!IS_ALIGNED(virt_to_phys(m), huge_page_size(h)));
2805 	/* Put them into a private list first because mem_map is not up yet */
2806 	INIT_LIST_HEAD(&m->list);
2807 	list_add(&m->list, &huge_boot_pages);
2808 	m->hstate = h;
2809 	return 1;
2810 }
2811 
2812 /*
2813  * Put bootmem huge pages into the standard lists after mem_map is up.
2814  * Note: This only applies to gigantic (order > MAX_ORDER) pages.
2815  */
2816 static void __init gather_bootmem_prealloc(void)
2817 {
2818 	struct huge_bootmem_page *m;
2819 
2820 	list_for_each_entry(m, &huge_boot_pages, list) {
2821 		struct page *page = virt_to_page(m);
2822 		struct hstate *h = m->hstate;
2823 
2824 		VM_BUG_ON(!hstate_is_gigantic(h));
2825 		WARN_ON(page_count(page) != 1);
2826 		if (prep_compound_gigantic_page(page, huge_page_order(h))) {
2827 			WARN_ON(PageReserved(page));
2828 			prep_new_huge_page(h, page, page_to_nid(page));
2829 			put_page(page); /* add to the hugepage allocator */
2830 		} else {
2831 			free_gigantic_page(page, huge_page_order(h));
2832 			pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n");
2833 		}
2834 
2835 		/*
2836 		 * We need to restore the 'stolen' pages to totalram_pages
2837 		 * in order to fix confusing memory reports from free(1) and
2838 		 * other side-effects, like CommitLimit going negative.
2839 		 */
2840 		adjust_managed_page_count(page, pages_per_huge_page(h));
2841 		cond_resched();
2842 	}
2843 }
2844 
2845 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
2846 {
2847 	unsigned long i;
2848 	nodemask_t *node_alloc_noretry;
2849 
2850 	if (!hstate_is_gigantic(h)) {
2851 		/*
2852 		 * Bit mask controlling how hard we retry per-node allocations.
2853 		 * Ignore errors as lower level routines can deal with
2854 		 * node_alloc_noretry == NULL.  If this kmalloc fails at boot
2855 		 * time, we are likely in bigger trouble.
2856 		 */
2857 		node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry),
2858 						GFP_KERNEL);
2859 	} else {
2860 		/* allocations done at boot time */
2861 		node_alloc_noretry = NULL;
2862 	}
2863 
2864 	/* bit mask controlling how hard we retry per-node allocations */
2865 	if (node_alloc_noretry)
2866 		nodes_clear(*node_alloc_noretry);
2867 
2868 	for (i = 0; i < h->max_huge_pages; ++i) {
2869 		if (hstate_is_gigantic(h)) {
2870 			if (hugetlb_cma_size) {
2871 				pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
2872 				goto free;
2873 			}
2874 			if (!alloc_bootmem_huge_page(h))
2875 				break;
2876 		} else if (!alloc_pool_huge_page(h,
2877 					 &node_states[N_MEMORY],
2878 					 node_alloc_noretry))
2879 			break;
2880 		cond_resched();
2881 	}
2882 	if (i < h->max_huge_pages) {
2883 		char buf[32];
2884 
2885 		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
2886 		pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
2887 			h->max_huge_pages, buf, i);
2888 		h->max_huge_pages = i;
2889 	}
2890 free:
2891 	kfree(node_alloc_noretry);
2892 }
2893 
2894 static void __init hugetlb_init_hstates(void)
2895 {
2896 	struct hstate *h;
2897 
2898 	for_each_hstate(h) {
2899 		if (minimum_order > huge_page_order(h))
2900 			minimum_order = huge_page_order(h);
2901 
2902 		/* oversize hugepages were init'ed in early boot */
2903 		if (!hstate_is_gigantic(h))
2904 			hugetlb_hstate_alloc_pages(h);
2905 	}
2906 	VM_BUG_ON(minimum_order == UINT_MAX);
2907 }
2908 
2909 static void __init report_hugepages(void)
2910 {
2911 	struct hstate *h;
2912 
2913 	for_each_hstate(h) {
2914 		char buf[32];
2915 
2916 		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
2917 		pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
2918 			buf, h->free_huge_pages);
2919 	}
2920 }
2921 
2922 #ifdef CONFIG_HIGHMEM
2923 static void try_to_free_low(struct hstate *h, unsigned long count,
2924 						nodemask_t *nodes_allowed)
2925 {
2926 	int i;
2927 	LIST_HEAD(page_list);
2928 
2929 	lockdep_assert_held(&hugetlb_lock);
2930 	if (hstate_is_gigantic(h))
2931 		return;
2932 
2933 	/*
2934 	 * Collect pages to be freed on a list, and free after dropping lock
2935 	 */
2936 	for_each_node_mask(i, *nodes_allowed) {
2937 		struct page *page, *next;
2938 		struct list_head *freel = &h->hugepage_freelists[i];
2939 		list_for_each_entry_safe(page, next, freel, lru) {
2940 			if (count >= h->nr_huge_pages)
2941 				goto out;
2942 			if (PageHighMem(page))
2943 				continue;
2944 			remove_hugetlb_page(h, page, false);
2945 			list_add(&page->lru, &page_list);
2946 		}
2947 	}
2948 
2949 out:
2950 	spin_unlock_irq(&hugetlb_lock);
2951 	update_and_free_pages_bulk(h, &page_list);
2952 	spin_lock_irq(&hugetlb_lock);
2953 }
2954 #else
2955 static inline void try_to_free_low(struct hstate *h, unsigned long count,
2956 						nodemask_t *nodes_allowed)
2957 {
2958 }
2959 #endif
2960 
2961 /*
2962  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
2963  * balanced by operating on them in a round-robin fashion.
2964  * Returns 1 if an adjustment was made.
2965  */
2966 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
2967 				int delta)
2968 {
2969 	int nr_nodes, node;
2970 
2971 	lockdep_assert_held(&hugetlb_lock);
2972 	VM_BUG_ON(delta != -1 && delta != 1);
2973 
2974 	if (delta < 0) {
2975 		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2976 			if (h->surplus_huge_pages_node[node])
2977 				goto found;
2978 		}
2979 	} else {
2980 		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2981 			if (h->surplus_huge_pages_node[node] <
2982 					h->nr_huge_pages_node[node])
2983 				goto found;
2984 		}
2985 	}
2986 	return 0;
2987 
2988 found:
2989 	h->surplus_huge_pages += delta;
2990 	h->surplus_huge_pages_node[node] += delta;
2991 	return 1;
2992 }
2993 
2994 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
2995 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
2996 			      nodemask_t *nodes_allowed)
2997 {
2998 	unsigned long min_count, ret;
2999 	struct page *page;
3000 	LIST_HEAD(page_list);
3001 	NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
3002 
3003 	/*
3004 	 * Bit mask controlling how hard we retry per-node allocations.
3005 	 * If we can not allocate the bit mask, do not attempt to allocate
3006 	 * the requested huge pages.
3007 	 */
3008 	if (node_alloc_noretry)
3009 		nodes_clear(*node_alloc_noretry);
3010 	else
3011 		return -ENOMEM;
3012 
3013 	/*
3014 	 * resize_lock mutex prevents concurrent adjustments to number of
3015 	 * pages in hstate via the proc/sysfs interfaces.
3016 	 */
3017 	mutex_lock(&h->resize_lock);
3018 	flush_free_hpage_work(h);
3019 	spin_lock_irq(&hugetlb_lock);
3020 
3021 	/*
3022 	 * Check for a node specific request.
3023 	 * Changing node specific huge page count may require a corresponding
3024 	 * change to the global count.  In any case, the passed node mask
3025 	 * (nodes_allowed) will restrict alloc/free to the specified node.
3026 	 */
3027 	if (nid != NUMA_NO_NODE) {
3028 		unsigned long old_count = count;
3029 
3030 		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
3031 		/*
3032 		 * User may have specified a large count value which caused the
3033 		 * above calculation to overflow.  In this case, they wanted
3034 		 * to allocate as many huge pages as possible.  Set count to
3035 		 * largest possible value to align with their intention.
3036 		 */
3037 		if (count < old_count)
3038 			count = ULONG_MAX;
3039 	}
3040 
3041 	/*
3042 	 * Gigantic pages runtime allocation depend on the capability for large
3043 	 * page range allocation.
3044 	 * If the system does not provide this feature, return an error when
3045 	 * the user tries to allocate gigantic pages but let the user free the
3046 	 * boottime allocated gigantic pages.
3047 	 */
3048 	if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
3049 		if (count > persistent_huge_pages(h)) {
3050 			spin_unlock_irq(&hugetlb_lock);
3051 			mutex_unlock(&h->resize_lock);
3052 			NODEMASK_FREE(node_alloc_noretry);
3053 			return -EINVAL;
3054 		}
3055 		/* Fall through to decrease pool */
3056 	}
3057 
3058 	/*
3059 	 * Increase the pool size
3060 	 * First take pages out of surplus state.  Then make up the
3061 	 * remaining difference by allocating fresh huge pages.
3062 	 *
3063 	 * We might race with alloc_surplus_huge_page() here and be unable
3064 	 * to convert a surplus huge page to a normal huge page. That is
3065 	 * not critical, though, it just means the overall size of the
3066 	 * pool might be one hugepage larger than it needs to be, but
3067 	 * within all the constraints specified by the sysctls.
3068 	 */
3069 	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
3070 		if (!adjust_pool_surplus(h, nodes_allowed, -1))
3071 			break;
3072 	}
3073 
3074 	while (count > persistent_huge_pages(h)) {
3075 		/*
3076 		 * If this allocation races such that we no longer need the
3077 		 * page, free_huge_page will handle it by freeing the page
3078 		 * and reducing the surplus.
3079 		 */
3080 		spin_unlock_irq(&hugetlb_lock);
3081 
3082 		/* yield cpu to avoid soft lockup */
3083 		cond_resched();
3084 
3085 		ret = alloc_pool_huge_page(h, nodes_allowed,
3086 						node_alloc_noretry);
3087 		spin_lock_irq(&hugetlb_lock);
3088 		if (!ret)
3089 			goto out;
3090 
3091 		/* Bail for signals. Probably ctrl-c from user */
3092 		if (signal_pending(current))
3093 			goto out;
3094 	}
3095 
3096 	/*
3097 	 * Decrease the pool size
3098 	 * First return free pages to the buddy allocator (being careful
3099 	 * to keep enough around to satisfy reservations).  Then place
3100 	 * pages into surplus state as needed so the pool will shrink
3101 	 * to the desired size as pages become free.
3102 	 *
3103 	 * By placing pages into the surplus state independent of the
3104 	 * overcommit value, we are allowing the surplus pool size to
3105 	 * exceed overcommit. There are few sane options here. Since
3106 	 * alloc_surplus_huge_page() is checking the global counter,
3107 	 * though, we'll note that we're not allowed to exceed surplus
3108 	 * and won't grow the pool anywhere else. Not until one of the
3109 	 * sysctls are changed, or the surplus pages go out of use.
3110 	 */
3111 	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
3112 	min_count = max(count, min_count);
3113 	try_to_free_low(h, min_count, nodes_allowed);
3114 
3115 	/*
3116 	 * Collect pages to be removed on list without dropping lock
3117 	 */
3118 	while (min_count < persistent_huge_pages(h)) {
3119 		page = remove_pool_huge_page(h, nodes_allowed, 0);
3120 		if (!page)
3121 			break;
3122 
3123 		list_add(&page->lru, &page_list);
3124 	}
3125 	/* free the pages after dropping lock */
3126 	spin_unlock_irq(&hugetlb_lock);
3127 	update_and_free_pages_bulk(h, &page_list);
3128 	flush_free_hpage_work(h);
3129 	spin_lock_irq(&hugetlb_lock);
3130 
3131 	while (count < persistent_huge_pages(h)) {
3132 		if (!adjust_pool_surplus(h, nodes_allowed, 1))
3133 			break;
3134 	}
3135 out:
3136 	h->max_huge_pages = persistent_huge_pages(h);
3137 	spin_unlock_irq(&hugetlb_lock);
3138 	mutex_unlock(&h->resize_lock);
3139 
3140 	NODEMASK_FREE(node_alloc_noretry);
3141 
3142 	return 0;
3143 }
3144 
3145 #define HSTATE_ATTR_RO(_name) \
3146 	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
3147 
3148 #define HSTATE_ATTR(_name) \
3149 	static struct kobj_attribute _name##_attr = \
3150 		__ATTR(_name, 0644, _name##_show, _name##_store)
3151 
3152 static struct kobject *hugepages_kobj;
3153 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
3154 
3155 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
3156 
3157 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
3158 {
3159 	int i;
3160 
3161 	for (i = 0; i < HUGE_MAX_HSTATE; i++)
3162 		if (hstate_kobjs[i] == kobj) {
3163 			if (nidp)
3164 				*nidp = NUMA_NO_NODE;
3165 			return &hstates[i];
3166 		}
3167 
3168 	return kobj_to_node_hstate(kobj, nidp);
3169 }
3170 
3171 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
3172 					struct kobj_attribute *attr, char *buf)
3173 {
3174 	struct hstate *h;
3175 	unsigned long nr_huge_pages;
3176 	int nid;
3177 
3178 	h = kobj_to_hstate(kobj, &nid);
3179 	if (nid == NUMA_NO_NODE)
3180 		nr_huge_pages = h->nr_huge_pages;
3181 	else
3182 		nr_huge_pages = h->nr_huge_pages_node[nid];
3183 
3184 	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
3185 }
3186 
3187 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
3188 					   struct hstate *h, int nid,
3189 					   unsigned long count, size_t len)
3190 {
3191 	int err;
3192 	nodemask_t nodes_allowed, *n_mask;
3193 
3194 	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3195 		return -EINVAL;
3196 
3197 	if (nid == NUMA_NO_NODE) {
3198 		/*
3199 		 * global hstate attribute
3200 		 */
3201 		if (!(obey_mempolicy &&
3202 				init_nodemask_of_mempolicy(&nodes_allowed)))
3203 			n_mask = &node_states[N_MEMORY];
3204 		else
3205 			n_mask = &nodes_allowed;
3206 	} else {
3207 		/*
3208 		 * Node specific request.  count adjustment happens in
3209 		 * set_max_huge_pages() after acquiring hugetlb_lock.
3210 		 */
3211 		init_nodemask_of_node(&nodes_allowed, nid);
3212 		n_mask = &nodes_allowed;
3213 	}
3214 
3215 	err = set_max_huge_pages(h, count, nid, n_mask);
3216 
3217 	return err ? err : len;
3218 }
3219 
3220 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
3221 					 struct kobject *kobj, const char *buf,
3222 					 size_t len)
3223 {
3224 	struct hstate *h;
3225 	unsigned long count;
3226 	int nid;
3227 	int err;
3228 
3229 	err = kstrtoul(buf, 10, &count);
3230 	if (err)
3231 		return err;
3232 
3233 	h = kobj_to_hstate(kobj, &nid);
3234 	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
3235 }
3236 
3237 static ssize_t nr_hugepages_show(struct kobject *kobj,
3238 				       struct kobj_attribute *attr, char *buf)
3239 {
3240 	return nr_hugepages_show_common(kobj, attr, buf);
3241 }
3242 
3243 static ssize_t nr_hugepages_store(struct kobject *kobj,
3244 	       struct kobj_attribute *attr, const char *buf, size_t len)
3245 {
3246 	return nr_hugepages_store_common(false, kobj, buf, len);
3247 }
3248 HSTATE_ATTR(nr_hugepages);
3249 
3250 #ifdef CONFIG_NUMA
3251 
3252 /*
3253  * hstate attribute for optionally mempolicy-based constraint on persistent
3254  * huge page alloc/free.
3255  */
3256 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
3257 					   struct kobj_attribute *attr,
3258 					   char *buf)
3259 {
3260 	return nr_hugepages_show_common(kobj, attr, buf);
3261 }
3262 
3263 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
3264 	       struct kobj_attribute *attr, const char *buf, size_t len)
3265 {
3266 	return nr_hugepages_store_common(true, kobj, buf, len);
3267 }
3268 HSTATE_ATTR(nr_hugepages_mempolicy);
3269 #endif
3270 
3271 
3272 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
3273 					struct kobj_attribute *attr, char *buf)
3274 {
3275 	struct hstate *h = kobj_to_hstate(kobj, NULL);
3276 	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
3277 }
3278 
3279 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
3280 		struct kobj_attribute *attr, const char *buf, size_t count)
3281 {
3282 	int err;
3283 	unsigned long input;
3284 	struct hstate *h = kobj_to_hstate(kobj, NULL);
3285 
3286 	if (hstate_is_gigantic(h))
3287 		return -EINVAL;
3288 
3289 	err = kstrtoul(buf, 10, &input);
3290 	if (err)
3291 		return err;
3292 
3293 	spin_lock_irq(&hugetlb_lock);
3294 	h->nr_overcommit_huge_pages = input;
3295 	spin_unlock_irq(&hugetlb_lock);
3296 
3297 	return count;
3298 }
3299 HSTATE_ATTR(nr_overcommit_hugepages);
3300 
3301 static ssize_t free_hugepages_show(struct kobject *kobj,
3302 					struct kobj_attribute *attr, char *buf)
3303 {
3304 	struct hstate *h;
3305 	unsigned long free_huge_pages;
3306 	int nid;
3307 
3308 	h = kobj_to_hstate(kobj, &nid);
3309 	if (nid == NUMA_NO_NODE)
3310 		free_huge_pages = h->free_huge_pages;
3311 	else
3312 		free_huge_pages = h->free_huge_pages_node[nid];
3313 
3314 	return sysfs_emit(buf, "%lu\n", free_huge_pages);
3315 }
3316 HSTATE_ATTR_RO(free_hugepages);
3317 
3318 static ssize_t resv_hugepages_show(struct kobject *kobj,
3319 					struct kobj_attribute *attr, char *buf)
3320 {
3321 	struct hstate *h = kobj_to_hstate(kobj, NULL);
3322 	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
3323 }
3324 HSTATE_ATTR_RO(resv_hugepages);
3325 
3326 static ssize_t surplus_hugepages_show(struct kobject *kobj,
3327 					struct kobj_attribute *attr, char *buf)
3328 {
3329 	struct hstate *h;
3330 	unsigned long surplus_huge_pages;
3331 	int nid;
3332 
3333 	h = kobj_to_hstate(kobj, &nid);
3334 	if (nid == NUMA_NO_NODE)
3335 		surplus_huge_pages = h->surplus_huge_pages;
3336 	else
3337 		surplus_huge_pages = h->surplus_huge_pages_node[nid];
3338 
3339 	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
3340 }
3341 HSTATE_ATTR_RO(surplus_hugepages);
3342 
3343 static struct attribute *hstate_attrs[] = {
3344 	&nr_hugepages_attr.attr,
3345 	&nr_overcommit_hugepages_attr.attr,
3346 	&free_hugepages_attr.attr,
3347 	&resv_hugepages_attr.attr,
3348 	&surplus_hugepages_attr.attr,
3349 #ifdef CONFIG_NUMA
3350 	&nr_hugepages_mempolicy_attr.attr,
3351 #endif
3352 	NULL,
3353 };
3354 
3355 static const struct attribute_group hstate_attr_group = {
3356 	.attrs = hstate_attrs,
3357 };
3358 
3359 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
3360 				    struct kobject **hstate_kobjs,
3361 				    const struct attribute_group *hstate_attr_group)
3362 {
3363 	int retval;
3364 	int hi = hstate_index(h);
3365 
3366 	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
3367 	if (!hstate_kobjs[hi])
3368 		return -ENOMEM;
3369 
3370 	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
3371 	if (retval) {
3372 		kobject_put(hstate_kobjs[hi]);
3373 		hstate_kobjs[hi] = NULL;
3374 	}
3375 
3376 	return retval;
3377 }
3378 
3379 static void __init hugetlb_sysfs_init(void)
3380 {
3381 	struct hstate *h;
3382 	int err;
3383 
3384 	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
3385 	if (!hugepages_kobj)
3386 		return;
3387 
3388 	for_each_hstate(h) {
3389 		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
3390 					 hstate_kobjs, &hstate_attr_group);
3391 		if (err)
3392 			pr_err("HugeTLB: Unable to add hstate %s", h->name);
3393 	}
3394 }
3395 
3396 #ifdef CONFIG_NUMA
3397 
3398 /*
3399  * node_hstate/s - associate per node hstate attributes, via their kobjects,
3400  * with node devices in node_devices[] using a parallel array.  The array
3401  * index of a node device or _hstate == node id.
3402  * This is here to avoid any static dependency of the node device driver, in
3403  * the base kernel, on the hugetlb module.
3404  */
3405 struct node_hstate {
3406 	struct kobject		*hugepages_kobj;
3407 	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
3408 };
3409 static struct node_hstate node_hstates[MAX_NUMNODES];
3410 
3411 /*
3412  * A subset of global hstate attributes for node devices
3413  */
3414 static struct attribute *per_node_hstate_attrs[] = {
3415 	&nr_hugepages_attr.attr,
3416 	&free_hugepages_attr.attr,
3417 	&surplus_hugepages_attr.attr,
3418 	NULL,
3419 };
3420 
3421 static const struct attribute_group per_node_hstate_attr_group = {
3422 	.attrs = per_node_hstate_attrs,
3423 };
3424 
3425 /*
3426  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
3427  * Returns node id via non-NULL nidp.
3428  */
3429 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
3430 {
3431 	int nid;
3432 
3433 	for (nid = 0; nid < nr_node_ids; nid++) {
3434 		struct node_hstate *nhs = &node_hstates[nid];
3435 		int i;
3436 		for (i = 0; i < HUGE_MAX_HSTATE; i++)
3437 			if (nhs->hstate_kobjs[i] == kobj) {
3438 				if (nidp)
3439 					*nidp = nid;
3440 				return &hstates[i];
3441 			}
3442 	}
3443 
3444 	BUG();
3445 	return NULL;
3446 }
3447 
3448 /*
3449  * Unregister hstate attributes from a single node device.
3450  * No-op if no hstate attributes attached.
3451  */
3452 static void hugetlb_unregister_node(struct node *node)
3453 {
3454 	struct hstate *h;
3455 	struct node_hstate *nhs = &node_hstates[node->dev.id];
3456 
3457 	if (!nhs->hugepages_kobj)
3458 		return;		/* no hstate attributes */
3459 
3460 	for_each_hstate(h) {
3461 		int idx = hstate_index(h);
3462 		if (nhs->hstate_kobjs[idx]) {
3463 			kobject_put(nhs->hstate_kobjs[idx]);
3464 			nhs->hstate_kobjs[idx] = NULL;
3465 		}
3466 	}
3467 
3468 	kobject_put(nhs->hugepages_kobj);
3469 	nhs->hugepages_kobj = NULL;
3470 }
3471 
3472 
3473 /*
3474  * Register hstate attributes for a single node device.
3475  * No-op if attributes already registered.
3476  */
3477 static void hugetlb_register_node(struct node *node)
3478 {
3479 	struct hstate *h;
3480 	struct node_hstate *nhs = &node_hstates[node->dev.id];
3481 	int err;
3482 
3483 	if (nhs->hugepages_kobj)
3484 		return;		/* already allocated */
3485 
3486 	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
3487 							&node->dev.kobj);
3488 	if (!nhs->hugepages_kobj)
3489 		return;
3490 
3491 	for_each_hstate(h) {
3492 		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
3493 						nhs->hstate_kobjs,
3494 						&per_node_hstate_attr_group);
3495 		if (err) {
3496 			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
3497 				h->name, node->dev.id);
3498 			hugetlb_unregister_node(node);
3499 			break;
3500 		}
3501 	}
3502 }
3503 
3504 /*
3505  * hugetlb init time:  register hstate attributes for all registered node
3506  * devices of nodes that have memory.  All on-line nodes should have
3507  * registered their associated device by this time.
3508  */
3509 static void __init hugetlb_register_all_nodes(void)
3510 {
3511 	int nid;
3512 
3513 	for_each_node_state(nid, N_MEMORY) {
3514 		struct node *node = node_devices[nid];
3515 		if (node->dev.id == nid)
3516 			hugetlb_register_node(node);
3517 	}
3518 
3519 	/*
3520 	 * Let the node device driver know we're here so it can
3521 	 * [un]register hstate attributes on node hotplug.
3522 	 */
3523 	register_hugetlbfs_with_node(hugetlb_register_node,
3524 				     hugetlb_unregister_node);
3525 }
3526 #else	/* !CONFIG_NUMA */
3527 
3528 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
3529 {
3530 	BUG();
3531 	if (nidp)
3532 		*nidp = -1;
3533 	return NULL;
3534 }
3535 
3536 static void hugetlb_register_all_nodes(void) { }
3537 
3538 #endif
3539 
3540 static int __init hugetlb_init(void)
3541 {
3542 	int i;
3543 
3544 	BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
3545 			__NR_HPAGEFLAGS);
3546 
3547 	if (!hugepages_supported()) {
3548 		if (hugetlb_max_hstate || default_hstate_max_huge_pages)
3549 			pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
3550 		return 0;
3551 	}
3552 
3553 	/*
3554 	 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
3555 	 * architectures depend on setup being done here.
3556 	 */
3557 	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
3558 	if (!parsed_default_hugepagesz) {
3559 		/*
3560 		 * If we did not parse a default huge page size, set
3561 		 * default_hstate_idx to HPAGE_SIZE hstate. And, if the
3562 		 * number of huge pages for this default size was implicitly
3563 		 * specified, set that here as well.
3564 		 * Note that the implicit setting will overwrite an explicit
3565 		 * setting.  A warning will be printed in this case.
3566 		 */
3567 		default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
3568 		if (default_hstate_max_huge_pages) {
3569 			if (default_hstate.max_huge_pages) {
3570 				char buf[32];
3571 
3572 				string_get_size(huge_page_size(&default_hstate),
3573 					1, STRING_UNITS_2, buf, 32);
3574 				pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
3575 					default_hstate.max_huge_pages, buf);
3576 				pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
3577 					default_hstate_max_huge_pages);
3578 			}
3579 			default_hstate.max_huge_pages =
3580 				default_hstate_max_huge_pages;
3581 		}
3582 	}
3583 
3584 	hugetlb_cma_check();
3585 	hugetlb_init_hstates();
3586 	gather_bootmem_prealloc();
3587 	report_hugepages();
3588 
3589 	hugetlb_sysfs_init();
3590 	hugetlb_register_all_nodes();
3591 	hugetlb_cgroup_file_init();
3592 
3593 #ifdef CONFIG_SMP
3594 	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
3595 #else
3596 	num_fault_mutexes = 1;
3597 #endif
3598 	hugetlb_fault_mutex_table =
3599 		kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
3600 			      GFP_KERNEL);
3601 	BUG_ON(!hugetlb_fault_mutex_table);
3602 
3603 	for (i = 0; i < num_fault_mutexes; i++)
3604 		mutex_init(&hugetlb_fault_mutex_table[i]);
3605 	return 0;
3606 }
3607 subsys_initcall(hugetlb_init);
3608 
3609 /* Overwritten by architectures with more huge page sizes */
3610 bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
3611 {
3612 	return size == HPAGE_SIZE;
3613 }
3614 
3615 void __init hugetlb_add_hstate(unsigned int order)
3616 {
3617 	struct hstate *h;
3618 	unsigned long i;
3619 
3620 	if (size_to_hstate(PAGE_SIZE << order)) {
3621 		return;
3622 	}
3623 	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
3624 	BUG_ON(order == 0);
3625 	h = &hstates[hugetlb_max_hstate++];
3626 	mutex_init(&h->resize_lock);
3627 	h->order = order;
3628 	h->mask = ~(huge_page_size(h) - 1);
3629 	for (i = 0; i < MAX_NUMNODES; ++i)
3630 		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
3631 	INIT_LIST_HEAD(&h->hugepage_activelist);
3632 	h->next_nid_to_alloc = first_memory_node;
3633 	h->next_nid_to_free = first_memory_node;
3634 	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
3635 					huge_page_size(h)/1024);
3636 	hugetlb_vmemmap_init(h);
3637 
3638 	parsed_hstate = h;
3639 }
3640 
3641 /*
3642  * hugepages command line processing
3643  * hugepages normally follows a valid hugepagsz or default_hugepagsz
3644  * specification.  If not, ignore the hugepages value.  hugepages can also
3645  * be the first huge page command line  option in which case it implicitly
3646  * specifies the number of huge pages for the default size.
3647  */
3648 static int __init hugepages_setup(char *s)
3649 {
3650 	unsigned long *mhp;
3651 	static unsigned long *last_mhp;
3652 
3653 	if (!parsed_valid_hugepagesz) {
3654 		pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
3655 		parsed_valid_hugepagesz = true;
3656 		return 0;
3657 	}
3658 
3659 	/*
3660 	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
3661 	 * yet, so this hugepages= parameter goes to the "default hstate".
3662 	 * Otherwise, it goes with the previously parsed hugepagesz or
3663 	 * default_hugepagesz.
3664 	 */
3665 	else if (!hugetlb_max_hstate)
3666 		mhp = &default_hstate_max_huge_pages;
3667 	else
3668 		mhp = &parsed_hstate->max_huge_pages;
3669 
3670 	if (mhp == last_mhp) {
3671 		pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
3672 		return 0;
3673 	}
3674 
3675 	if (sscanf(s, "%lu", mhp) <= 0)
3676 		*mhp = 0;
3677 
3678 	/*
3679 	 * Global state is always initialized later in hugetlb_init.
3680 	 * But we need to allocate gigantic hstates here early to still
3681 	 * use the bootmem allocator.
3682 	 */
3683 	if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate))
3684 		hugetlb_hstate_alloc_pages(parsed_hstate);
3685 
3686 	last_mhp = mhp;
3687 
3688 	return 1;
3689 }
3690 __setup("hugepages=", hugepages_setup);
3691 
3692 /*
3693  * hugepagesz command line processing
3694  * A specific huge page size can only be specified once with hugepagesz.
3695  * hugepagesz is followed by hugepages on the command line.  The global
3696  * variable 'parsed_valid_hugepagesz' is used to determine if prior
3697  * hugepagesz argument was valid.
3698  */
3699 static int __init hugepagesz_setup(char *s)
3700 {
3701 	unsigned long size;
3702 	struct hstate *h;
3703 
3704 	parsed_valid_hugepagesz = false;
3705 	size = (unsigned long)memparse(s, NULL);
3706 
3707 	if (!arch_hugetlb_valid_size(size)) {
3708 		pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
3709 		return 0;
3710 	}
3711 
3712 	h = size_to_hstate(size);
3713 	if (h) {
3714 		/*
3715 		 * hstate for this size already exists.  This is normally
3716 		 * an error, but is allowed if the existing hstate is the
3717 		 * default hstate.  More specifically, it is only allowed if
3718 		 * the number of huge pages for the default hstate was not
3719 		 * previously specified.
3720 		 */
3721 		if (!parsed_default_hugepagesz ||  h != &default_hstate ||
3722 		    default_hstate.max_huge_pages) {
3723 			pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
3724 			return 0;
3725 		}
3726 
3727 		/*
3728 		 * No need to call hugetlb_add_hstate() as hstate already
3729 		 * exists.  But, do set parsed_hstate so that a following
3730 		 * hugepages= parameter will be applied to this hstate.
3731 		 */
3732 		parsed_hstate = h;
3733 		parsed_valid_hugepagesz = true;
3734 		return 1;
3735 	}
3736 
3737 	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
3738 	parsed_valid_hugepagesz = true;
3739 	return 1;
3740 }
3741 __setup("hugepagesz=", hugepagesz_setup);
3742 
3743 /*
3744  * default_hugepagesz command line input
3745  * Only one instance of default_hugepagesz allowed on command line.
3746  */
3747 static int __init default_hugepagesz_setup(char *s)
3748 {
3749 	unsigned long size;
3750 
3751 	parsed_valid_hugepagesz = false;
3752 	if (parsed_default_hugepagesz) {
3753 		pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
3754 		return 0;
3755 	}
3756 
3757 	size = (unsigned long)memparse(s, NULL);
3758 
3759 	if (!arch_hugetlb_valid_size(size)) {
3760 		pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
3761 		return 0;
3762 	}
3763 
3764 	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
3765 	parsed_valid_hugepagesz = true;
3766 	parsed_default_hugepagesz = true;
3767 	default_hstate_idx = hstate_index(size_to_hstate(size));
3768 
3769 	/*
3770 	 * The number of default huge pages (for this size) could have been
3771 	 * specified as the first hugetlb parameter: hugepages=X.  If so,
3772 	 * then default_hstate_max_huge_pages is set.  If the default huge
3773 	 * page size is gigantic (>= MAX_ORDER), then the pages must be
3774 	 * allocated here from bootmem allocator.
3775 	 */
3776 	if (default_hstate_max_huge_pages) {
3777 		default_hstate.max_huge_pages = default_hstate_max_huge_pages;
3778 		if (hstate_is_gigantic(&default_hstate))
3779 			hugetlb_hstate_alloc_pages(&default_hstate);
3780 		default_hstate_max_huge_pages = 0;
3781 	}
3782 
3783 	return 1;
3784 }
3785 __setup("default_hugepagesz=", default_hugepagesz_setup);
3786 
3787 static unsigned int allowed_mems_nr(struct hstate *h)
3788 {
3789 	int node;
3790 	unsigned int nr = 0;
3791 	nodemask_t *mpol_allowed;
3792 	unsigned int *array = h->free_huge_pages_node;
3793 	gfp_t gfp_mask = htlb_alloc_mask(h);
3794 
3795 	mpol_allowed = policy_nodemask_current(gfp_mask);
3796 
3797 	for_each_node_mask(node, cpuset_current_mems_allowed) {
3798 		if (!mpol_allowed || node_isset(node, *mpol_allowed))
3799 			nr += array[node];
3800 	}
3801 
3802 	return nr;
3803 }
3804 
3805 #ifdef CONFIG_SYSCTL
3806 static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write,
3807 					  void *buffer, size_t *length,
3808 					  loff_t *ppos, unsigned long *out)
3809 {
3810 	struct ctl_table dup_table;
3811 
3812 	/*
3813 	 * In order to avoid races with __do_proc_doulongvec_minmax(), we
3814 	 * can duplicate the @table and alter the duplicate of it.
3815 	 */
3816 	dup_table = *table;
3817 	dup_table.data = out;
3818 
3819 	return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
3820 }
3821 
3822 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
3823 			 struct ctl_table *table, int write,
3824 			 void *buffer, size_t *length, loff_t *ppos)
3825 {
3826 	struct hstate *h = &default_hstate;
3827 	unsigned long tmp = h->max_huge_pages;
3828 	int ret;
3829 
3830 	if (!hugepages_supported())
3831 		return -EOPNOTSUPP;
3832 
3833 	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
3834 					     &tmp);
3835 	if (ret)
3836 		goto out;
3837 
3838 	if (write)
3839 		ret = __nr_hugepages_store_common(obey_mempolicy, h,
3840 						  NUMA_NO_NODE, tmp, *length);
3841 out:
3842 	return ret;
3843 }
3844 
3845 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
3846 			  void *buffer, size_t *length, loff_t *ppos)
3847 {
3848 
3849 	return hugetlb_sysctl_handler_common(false, table, write,
3850 							buffer, length, ppos);
3851 }
3852 
3853 #ifdef CONFIG_NUMA
3854 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
3855 			  void *buffer, size_t *length, loff_t *ppos)
3856 {
3857 	return hugetlb_sysctl_handler_common(true, table, write,
3858 							buffer, length, ppos);
3859 }
3860 #endif /* CONFIG_NUMA */
3861 
3862 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
3863 		void *buffer, size_t *length, loff_t *ppos)
3864 {
3865 	struct hstate *h = &default_hstate;
3866 	unsigned long tmp;
3867 	int ret;
3868 
3869 	if (!hugepages_supported())
3870 		return -EOPNOTSUPP;
3871 
3872 	tmp = h->nr_overcommit_huge_pages;
3873 
3874 	if (write && hstate_is_gigantic(h))
3875 		return -EINVAL;
3876 
3877 	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
3878 					     &tmp);
3879 	if (ret)
3880 		goto out;
3881 
3882 	if (write) {
3883 		spin_lock_irq(&hugetlb_lock);
3884 		h->nr_overcommit_huge_pages = tmp;
3885 		spin_unlock_irq(&hugetlb_lock);
3886 	}
3887 out:
3888 	return ret;
3889 }
3890 
3891 #endif /* CONFIG_SYSCTL */
3892 
3893 void hugetlb_report_meminfo(struct seq_file *m)
3894 {
3895 	struct hstate *h;
3896 	unsigned long total = 0;
3897 
3898 	if (!hugepages_supported())
3899 		return;
3900 
3901 	for_each_hstate(h) {
3902 		unsigned long count = h->nr_huge_pages;
3903 
3904 		total += huge_page_size(h) * count;
3905 
3906 		if (h == &default_hstate)
3907 			seq_printf(m,
3908 				   "HugePages_Total:   %5lu\n"
3909 				   "HugePages_Free:    %5lu\n"
3910 				   "HugePages_Rsvd:    %5lu\n"
3911 				   "HugePages_Surp:    %5lu\n"
3912 				   "Hugepagesize:   %8lu kB\n",
3913 				   count,
3914 				   h->free_huge_pages,
3915 				   h->resv_huge_pages,
3916 				   h->surplus_huge_pages,
3917 				   huge_page_size(h) / SZ_1K);
3918 	}
3919 
3920 	seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
3921 }
3922 
3923 int hugetlb_report_node_meminfo(char *buf, int len, int nid)
3924 {
3925 	struct hstate *h = &default_hstate;
3926 
3927 	if (!hugepages_supported())
3928 		return 0;
3929 
3930 	return sysfs_emit_at(buf, len,
3931 			     "Node %d HugePages_Total: %5u\n"
3932 			     "Node %d HugePages_Free:  %5u\n"
3933 			     "Node %d HugePages_Surp:  %5u\n",
3934 			     nid, h->nr_huge_pages_node[nid],
3935 			     nid, h->free_huge_pages_node[nid],
3936 			     nid, h->surplus_huge_pages_node[nid]);
3937 }
3938 
3939 void hugetlb_show_meminfo(void)
3940 {
3941 	struct hstate *h;
3942 	int nid;
3943 
3944 	if (!hugepages_supported())
3945 		return;
3946 
3947 	for_each_node_state(nid, N_MEMORY)
3948 		for_each_hstate(h)
3949 			pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
3950 				nid,
3951 				h->nr_huge_pages_node[nid],
3952 				h->free_huge_pages_node[nid],
3953 				h->surplus_huge_pages_node[nid],
3954 				huge_page_size(h) / SZ_1K);
3955 }
3956 
3957 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
3958 {
3959 	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
3960 		   atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
3961 }
3962 
3963 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
3964 unsigned long hugetlb_total_pages(void)
3965 {
3966 	struct hstate *h;
3967 	unsigned long nr_total_pages = 0;
3968 
3969 	for_each_hstate(h)
3970 		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
3971 	return nr_total_pages;
3972 }
3973 
3974 static int hugetlb_acct_memory(struct hstate *h, long delta)
3975 {
3976 	int ret = -ENOMEM;
3977 
3978 	if (!delta)
3979 		return 0;
3980 
3981 	spin_lock_irq(&hugetlb_lock);
3982 	/*
3983 	 * When cpuset is configured, it breaks the strict hugetlb page
3984 	 * reservation as the accounting is done on a global variable. Such
3985 	 * reservation is completely rubbish in the presence of cpuset because
3986 	 * the reservation is not checked against page availability for the
3987 	 * current cpuset. Application can still potentially OOM'ed by kernel
3988 	 * with lack of free htlb page in cpuset that the task is in.
3989 	 * Attempt to enforce strict accounting with cpuset is almost
3990 	 * impossible (or too ugly) because cpuset is too fluid that
3991 	 * task or memory node can be dynamically moved between cpusets.
3992 	 *
3993 	 * The change of semantics for shared hugetlb mapping with cpuset is
3994 	 * undesirable. However, in order to preserve some of the semantics,
3995 	 * we fall back to check against current free page availability as
3996 	 * a best attempt and hopefully to minimize the impact of changing
3997 	 * semantics that cpuset has.
3998 	 *
3999 	 * Apart from cpuset, we also have memory policy mechanism that
4000 	 * also determines from which node the kernel will allocate memory
4001 	 * in a NUMA system. So similar to cpuset, we also should consider
4002 	 * the memory policy of the current task. Similar to the description
4003 	 * above.
4004 	 */
4005 	if (delta > 0) {
4006 		if (gather_surplus_pages(h, delta) < 0)
4007 			goto out;
4008 
4009 		if (delta > allowed_mems_nr(h)) {
4010 			return_unused_surplus_pages(h, delta);
4011 			goto out;
4012 		}
4013 	}
4014 
4015 	ret = 0;
4016 	if (delta < 0)
4017 		return_unused_surplus_pages(h, (unsigned long) -delta);
4018 
4019 out:
4020 	spin_unlock_irq(&hugetlb_lock);
4021 	return ret;
4022 }
4023 
4024 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
4025 {
4026 	struct resv_map *resv = vma_resv_map(vma);
4027 
4028 	/*
4029 	 * This new VMA should share its siblings reservation map if present.
4030 	 * The VMA will only ever have a valid reservation map pointer where
4031 	 * it is being copied for another still existing VMA.  As that VMA
4032 	 * has a reference to the reservation map it cannot disappear until
4033 	 * after this open call completes.  It is therefore safe to take a
4034 	 * new reference here without additional locking.
4035 	 */
4036 	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4037 		kref_get(&resv->refs);
4038 }
4039 
4040 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
4041 {
4042 	struct hstate *h = hstate_vma(vma);
4043 	struct resv_map *resv = vma_resv_map(vma);
4044 	struct hugepage_subpool *spool = subpool_vma(vma);
4045 	unsigned long reserve, start, end;
4046 	long gbl_reserve;
4047 
4048 	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4049 		return;
4050 
4051 	start = vma_hugecache_offset(h, vma, vma->vm_start);
4052 	end = vma_hugecache_offset(h, vma, vma->vm_end);
4053 
4054 	reserve = (end - start) - region_count(resv, start, end);
4055 	hugetlb_cgroup_uncharge_counter(resv, start, end);
4056 	if (reserve) {
4057 		/*
4058 		 * Decrement reserve counts.  The global reserve count may be
4059 		 * adjusted if the subpool has a minimum size.
4060 		 */
4061 		gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
4062 		hugetlb_acct_memory(h, -gbl_reserve);
4063 	}
4064 
4065 	kref_put(&resv->refs, resv_map_release);
4066 }
4067 
4068 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
4069 {
4070 	if (addr & ~(huge_page_mask(hstate_vma(vma))))
4071 		return -EINVAL;
4072 	return 0;
4073 }
4074 
4075 static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
4076 {
4077 	return huge_page_size(hstate_vma(vma));
4078 }
4079 
4080 /*
4081  * We cannot handle pagefaults against hugetlb pages at all.  They cause
4082  * handle_mm_fault() to try to instantiate regular-sized pages in the
4083  * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
4084  * this far.
4085  */
4086 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
4087 {
4088 	BUG();
4089 	return 0;
4090 }
4091 
4092 /*
4093  * When a new function is introduced to vm_operations_struct and added
4094  * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
4095  * This is because under System V memory model, mappings created via
4096  * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
4097  * their original vm_ops are overwritten with shm_vm_ops.
4098  */
4099 const struct vm_operations_struct hugetlb_vm_ops = {
4100 	.fault = hugetlb_vm_op_fault,
4101 	.open = hugetlb_vm_op_open,
4102 	.close = hugetlb_vm_op_close,
4103 	.may_split = hugetlb_vm_op_split,
4104 	.pagesize = hugetlb_vm_op_pagesize,
4105 };
4106 
4107 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
4108 				int writable)
4109 {
4110 	pte_t entry;
4111 	unsigned int shift = huge_page_shift(hstate_vma(vma));
4112 
4113 	if (writable) {
4114 		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
4115 					 vma->vm_page_prot)));
4116 	} else {
4117 		entry = huge_pte_wrprotect(mk_huge_pte(page,
4118 					   vma->vm_page_prot));
4119 	}
4120 	entry = pte_mkyoung(entry);
4121 	entry = pte_mkhuge(entry);
4122 	entry = arch_make_huge_pte(entry, shift, vma->vm_flags);
4123 
4124 	return entry;
4125 }
4126 
4127 static void set_huge_ptep_writable(struct vm_area_struct *vma,
4128 				   unsigned long address, pte_t *ptep)
4129 {
4130 	pte_t entry;
4131 
4132 	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
4133 	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
4134 		update_mmu_cache(vma, address, ptep);
4135 }
4136 
4137 bool is_hugetlb_entry_migration(pte_t pte)
4138 {
4139 	swp_entry_t swp;
4140 
4141 	if (huge_pte_none(pte) || pte_present(pte))
4142 		return false;
4143 	swp = pte_to_swp_entry(pte);
4144 	if (is_migration_entry(swp))
4145 		return true;
4146 	else
4147 		return false;
4148 }
4149 
4150 static bool is_hugetlb_entry_hwpoisoned(pte_t pte)
4151 {
4152 	swp_entry_t swp;
4153 
4154 	if (huge_pte_none(pte) || pte_present(pte))
4155 		return false;
4156 	swp = pte_to_swp_entry(pte);
4157 	if (is_hwpoison_entry(swp))
4158 		return true;
4159 	else
4160 		return false;
4161 }
4162 
4163 static void
4164 hugetlb_install_page(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
4165 		     struct page *new_page)
4166 {
4167 	__SetPageUptodate(new_page);
4168 	set_huge_pte_at(vma->vm_mm, addr, ptep, make_huge_pte(vma, new_page, 1));
4169 	hugepage_add_new_anon_rmap(new_page, vma, addr);
4170 	hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
4171 	ClearHPageRestoreReserve(new_page);
4172 	SetHPageMigratable(new_page);
4173 }
4174 
4175 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
4176 			    struct vm_area_struct *vma)
4177 {
4178 	pte_t *src_pte, *dst_pte, entry, dst_entry;
4179 	struct page *ptepage;
4180 	unsigned long addr;
4181 	bool cow = is_cow_mapping(vma->vm_flags);
4182 	struct hstate *h = hstate_vma(vma);
4183 	unsigned long sz = huge_page_size(h);
4184 	unsigned long npages = pages_per_huge_page(h);
4185 	struct address_space *mapping = vma->vm_file->f_mapping;
4186 	struct mmu_notifier_range range;
4187 	int ret = 0;
4188 
4189 	if (cow) {
4190 		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src,
4191 					vma->vm_start,
4192 					vma->vm_end);
4193 		mmu_notifier_invalidate_range_start(&range);
4194 	} else {
4195 		/*
4196 		 * For shared mappings i_mmap_rwsem must be held to call
4197 		 * huge_pte_alloc, otherwise the returned ptep could go
4198 		 * away if part of a shared pmd and another thread calls
4199 		 * huge_pmd_unshare.
4200 		 */
4201 		i_mmap_lock_read(mapping);
4202 	}
4203 
4204 	for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
4205 		spinlock_t *src_ptl, *dst_ptl;
4206 		src_pte = huge_pte_offset(src, addr, sz);
4207 		if (!src_pte)
4208 			continue;
4209 		dst_pte = huge_pte_alloc(dst, vma, addr, sz);
4210 		if (!dst_pte) {
4211 			ret = -ENOMEM;
4212 			break;
4213 		}
4214 
4215 		/*
4216 		 * If the pagetables are shared don't copy or take references.
4217 		 * dst_pte == src_pte is the common case of src/dest sharing.
4218 		 *
4219 		 * However, src could have 'unshared' and dst shares with
4220 		 * another vma.  If dst_pte !none, this implies sharing.
4221 		 * Check here before taking page table lock, and once again
4222 		 * after taking the lock below.
4223 		 */
4224 		dst_entry = huge_ptep_get(dst_pte);
4225 		if ((dst_pte == src_pte) || !huge_pte_none(dst_entry))
4226 			continue;
4227 
4228 		dst_ptl = huge_pte_lock(h, dst, dst_pte);
4229 		src_ptl = huge_pte_lockptr(h, src, src_pte);
4230 		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4231 		entry = huge_ptep_get(src_pte);
4232 		dst_entry = huge_ptep_get(dst_pte);
4233 again:
4234 		if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) {
4235 			/*
4236 			 * Skip if src entry none.  Also, skip in the
4237 			 * unlikely case dst entry !none as this implies
4238 			 * sharing with another vma.
4239 			 */
4240 			;
4241 		} else if (unlikely(is_hugetlb_entry_migration(entry) ||
4242 				    is_hugetlb_entry_hwpoisoned(entry))) {
4243 			swp_entry_t swp_entry = pte_to_swp_entry(entry);
4244 
4245 			if (is_writable_migration_entry(swp_entry) && cow) {
4246 				/*
4247 				 * COW mappings require pages in both
4248 				 * parent and child to be set to read.
4249 				 */
4250 				swp_entry = make_readable_migration_entry(
4251 							swp_offset(swp_entry));
4252 				entry = swp_entry_to_pte(swp_entry);
4253 				set_huge_swap_pte_at(src, addr, src_pte,
4254 						     entry, sz);
4255 			}
4256 			set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz);
4257 		} else {
4258 			entry = huge_ptep_get(src_pte);
4259 			ptepage = pte_page(entry);
4260 			get_page(ptepage);
4261 
4262 			/*
4263 			 * This is a rare case where we see pinned hugetlb
4264 			 * pages while they're prone to COW.  We need to do the
4265 			 * COW earlier during fork.
4266 			 *
4267 			 * When pre-allocating the page or copying data, we
4268 			 * need to be without the pgtable locks since we could
4269 			 * sleep during the process.
4270 			 */
4271 			if (unlikely(page_needs_cow_for_dma(vma, ptepage))) {
4272 				pte_t src_pte_old = entry;
4273 				struct page *new;
4274 
4275 				spin_unlock(src_ptl);
4276 				spin_unlock(dst_ptl);
4277 				/* Do not use reserve as it's private owned */
4278 				new = alloc_huge_page(vma, addr, 1);
4279 				if (IS_ERR(new)) {
4280 					put_page(ptepage);
4281 					ret = PTR_ERR(new);
4282 					break;
4283 				}
4284 				copy_user_huge_page(new, ptepage, addr, vma,
4285 						    npages);
4286 				put_page(ptepage);
4287 
4288 				/* Install the new huge page if src pte stable */
4289 				dst_ptl = huge_pte_lock(h, dst, dst_pte);
4290 				src_ptl = huge_pte_lockptr(h, src, src_pte);
4291 				spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4292 				entry = huge_ptep_get(src_pte);
4293 				if (!pte_same(src_pte_old, entry)) {
4294 					restore_reserve_on_error(h, vma, addr,
4295 								new);
4296 					put_page(new);
4297 					/* dst_entry won't change as in child */
4298 					goto again;
4299 				}
4300 				hugetlb_install_page(vma, dst_pte, addr, new);
4301 				spin_unlock(src_ptl);
4302 				spin_unlock(dst_ptl);
4303 				continue;
4304 			}
4305 
4306 			if (cow) {
4307 				/*
4308 				 * No need to notify as we are downgrading page
4309 				 * table protection not changing it to point
4310 				 * to a new page.
4311 				 *
4312 				 * See Documentation/vm/mmu_notifier.rst
4313 				 */
4314 				huge_ptep_set_wrprotect(src, addr, src_pte);
4315 				entry = huge_pte_wrprotect(entry);
4316 			}
4317 
4318 			page_dup_rmap(ptepage, true);
4319 			set_huge_pte_at(dst, addr, dst_pte, entry);
4320 			hugetlb_count_add(npages, dst);
4321 		}
4322 		spin_unlock(src_ptl);
4323 		spin_unlock(dst_ptl);
4324 	}
4325 
4326 	if (cow)
4327 		mmu_notifier_invalidate_range_end(&range);
4328 	else
4329 		i_mmap_unlock_read(mapping);
4330 
4331 	return ret;
4332 }
4333 
4334 void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
4335 			    unsigned long start, unsigned long end,
4336 			    struct page *ref_page)
4337 {
4338 	struct mm_struct *mm = vma->vm_mm;
4339 	unsigned long address;
4340 	pte_t *ptep;
4341 	pte_t pte;
4342 	spinlock_t *ptl;
4343 	struct page *page;
4344 	struct hstate *h = hstate_vma(vma);
4345 	unsigned long sz = huge_page_size(h);
4346 	struct mmu_notifier_range range;
4347 
4348 	WARN_ON(!is_vm_hugetlb_page(vma));
4349 	BUG_ON(start & ~huge_page_mask(h));
4350 	BUG_ON(end & ~huge_page_mask(h));
4351 
4352 	/*
4353 	 * This is a hugetlb vma, all the pte entries should point
4354 	 * to huge page.
4355 	 */
4356 	tlb_change_page_size(tlb, sz);
4357 	tlb_start_vma(tlb, vma);
4358 
4359 	/*
4360 	 * If sharing possible, alert mmu notifiers of worst case.
4361 	 */
4362 	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start,
4363 				end);
4364 	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
4365 	mmu_notifier_invalidate_range_start(&range);
4366 	address = start;
4367 	for (; address < end; address += sz) {
4368 		ptep = huge_pte_offset(mm, address, sz);
4369 		if (!ptep)
4370 			continue;
4371 
4372 		ptl = huge_pte_lock(h, mm, ptep);
4373 		if (huge_pmd_unshare(mm, vma, &address, ptep)) {
4374 			spin_unlock(ptl);
4375 			/*
4376 			 * We just unmapped a page of PMDs by clearing a PUD.
4377 			 * The caller's TLB flush range should cover this area.
4378 			 */
4379 			continue;
4380 		}
4381 
4382 		pte = huge_ptep_get(ptep);
4383 		if (huge_pte_none(pte)) {
4384 			spin_unlock(ptl);
4385 			continue;
4386 		}
4387 
4388 		/*
4389 		 * Migrating hugepage or HWPoisoned hugepage is already
4390 		 * unmapped and its refcount is dropped, so just clear pte here.
4391 		 */
4392 		if (unlikely(!pte_present(pte))) {
4393 			huge_pte_clear(mm, address, ptep, sz);
4394 			spin_unlock(ptl);
4395 			continue;
4396 		}
4397 
4398 		page = pte_page(pte);
4399 		/*
4400 		 * If a reference page is supplied, it is because a specific
4401 		 * page is being unmapped, not a range. Ensure the page we
4402 		 * are about to unmap is the actual page of interest.
4403 		 */
4404 		if (ref_page) {
4405 			if (page != ref_page) {
4406 				spin_unlock(ptl);
4407 				continue;
4408 			}
4409 			/*
4410 			 * Mark the VMA as having unmapped its page so that
4411 			 * future faults in this VMA will fail rather than
4412 			 * looking like data was lost
4413 			 */
4414 			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
4415 		}
4416 
4417 		pte = huge_ptep_get_and_clear(mm, address, ptep);
4418 		tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
4419 		if (huge_pte_dirty(pte))
4420 			set_page_dirty(page);
4421 
4422 		hugetlb_count_sub(pages_per_huge_page(h), mm);
4423 		page_remove_rmap(page, true);
4424 
4425 		spin_unlock(ptl);
4426 		tlb_remove_page_size(tlb, page, huge_page_size(h));
4427 		/*
4428 		 * Bail out after unmapping reference page if supplied
4429 		 */
4430 		if (ref_page)
4431 			break;
4432 	}
4433 	mmu_notifier_invalidate_range_end(&range);
4434 	tlb_end_vma(tlb, vma);
4435 }
4436 
4437 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
4438 			  struct vm_area_struct *vma, unsigned long start,
4439 			  unsigned long end, struct page *ref_page)
4440 {
4441 	__unmap_hugepage_range(tlb, vma, start, end, ref_page);
4442 
4443 	/*
4444 	 * Clear this flag so that x86's huge_pmd_share page_table_shareable
4445 	 * test will fail on a vma being torn down, and not grab a page table
4446 	 * on its way out.  We're lucky that the flag has such an appropriate
4447 	 * name, and can in fact be safely cleared here. We could clear it
4448 	 * before the __unmap_hugepage_range above, but all that's necessary
4449 	 * is to clear it before releasing the i_mmap_rwsem. This works
4450 	 * because in the context this is called, the VMA is about to be
4451 	 * destroyed and the i_mmap_rwsem is held.
4452 	 */
4453 	vma->vm_flags &= ~VM_MAYSHARE;
4454 }
4455 
4456 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
4457 			  unsigned long end, struct page *ref_page)
4458 {
4459 	struct mmu_gather tlb;
4460 
4461 	tlb_gather_mmu(&tlb, vma->vm_mm);
4462 	__unmap_hugepage_range(&tlb, vma, start, end, ref_page);
4463 	tlb_finish_mmu(&tlb);
4464 }
4465 
4466 /*
4467  * This is called when the original mapper is failing to COW a MAP_PRIVATE
4468  * mapping it owns the reserve page for. The intention is to unmap the page
4469  * from other VMAs and let the children be SIGKILLed if they are faulting the
4470  * same region.
4471  */
4472 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
4473 			      struct page *page, unsigned long address)
4474 {
4475 	struct hstate *h = hstate_vma(vma);
4476 	struct vm_area_struct *iter_vma;
4477 	struct address_space *mapping;
4478 	pgoff_t pgoff;
4479 
4480 	/*
4481 	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
4482 	 * from page cache lookup which is in HPAGE_SIZE units.
4483 	 */
4484 	address = address & huge_page_mask(h);
4485 	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
4486 			vma->vm_pgoff;
4487 	mapping = vma->vm_file->f_mapping;
4488 
4489 	/*
4490 	 * Take the mapping lock for the duration of the table walk. As
4491 	 * this mapping should be shared between all the VMAs,
4492 	 * __unmap_hugepage_range() is called as the lock is already held
4493 	 */
4494 	i_mmap_lock_write(mapping);
4495 	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
4496 		/* Do not unmap the current VMA */
4497 		if (iter_vma == vma)
4498 			continue;
4499 
4500 		/*
4501 		 * Shared VMAs have their own reserves and do not affect
4502 		 * MAP_PRIVATE accounting but it is possible that a shared
4503 		 * VMA is using the same page so check and skip such VMAs.
4504 		 */
4505 		if (iter_vma->vm_flags & VM_MAYSHARE)
4506 			continue;
4507 
4508 		/*
4509 		 * Unmap the page from other VMAs without their own reserves.
4510 		 * They get marked to be SIGKILLed if they fault in these
4511 		 * areas. This is because a future no-page fault on this VMA
4512 		 * could insert a zeroed page instead of the data existing
4513 		 * from the time of fork. This would look like data corruption
4514 		 */
4515 		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
4516 			unmap_hugepage_range(iter_vma, address,
4517 					     address + huge_page_size(h), page);
4518 	}
4519 	i_mmap_unlock_write(mapping);
4520 }
4521 
4522 /*
4523  * Hugetlb_cow() should be called with page lock of the original hugepage held.
4524  * Called with hugetlb_instantiation_mutex held and pte_page locked so we
4525  * cannot race with other handlers or page migration.
4526  * Keep the pte_same checks anyway to make transition from the mutex easier.
4527  */
4528 static vm_fault_t hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
4529 		       unsigned long address, pte_t *ptep,
4530 		       struct page *pagecache_page, spinlock_t *ptl)
4531 {
4532 	pte_t pte;
4533 	struct hstate *h = hstate_vma(vma);
4534 	struct page *old_page, *new_page;
4535 	int outside_reserve = 0;
4536 	vm_fault_t ret = 0;
4537 	unsigned long haddr = address & huge_page_mask(h);
4538 	struct mmu_notifier_range range;
4539 
4540 	pte = huge_ptep_get(ptep);
4541 	old_page = pte_page(pte);
4542 
4543 retry_avoidcopy:
4544 	/* If no-one else is actually using this page, avoid the copy
4545 	 * and just make the page writable */
4546 	if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
4547 		page_move_anon_rmap(old_page, vma);
4548 		set_huge_ptep_writable(vma, haddr, ptep);
4549 		return 0;
4550 	}
4551 
4552 	/*
4553 	 * If the process that created a MAP_PRIVATE mapping is about to
4554 	 * perform a COW due to a shared page count, attempt to satisfy
4555 	 * the allocation without using the existing reserves. The pagecache
4556 	 * page is used to determine if the reserve at this address was
4557 	 * consumed or not. If reserves were used, a partial faulted mapping
4558 	 * at the time of fork() could consume its reserves on COW instead
4559 	 * of the full address range.
4560 	 */
4561 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
4562 			old_page != pagecache_page)
4563 		outside_reserve = 1;
4564 
4565 	get_page(old_page);
4566 
4567 	/*
4568 	 * Drop page table lock as buddy allocator may be called. It will
4569 	 * be acquired again before returning to the caller, as expected.
4570 	 */
4571 	spin_unlock(ptl);
4572 	new_page = alloc_huge_page(vma, haddr, outside_reserve);
4573 
4574 	if (IS_ERR(new_page)) {
4575 		/*
4576 		 * If a process owning a MAP_PRIVATE mapping fails to COW,
4577 		 * it is due to references held by a child and an insufficient
4578 		 * huge page pool. To guarantee the original mappers
4579 		 * reliability, unmap the page from child processes. The child
4580 		 * may get SIGKILLed if it later faults.
4581 		 */
4582 		if (outside_reserve) {
4583 			struct address_space *mapping = vma->vm_file->f_mapping;
4584 			pgoff_t idx;
4585 			u32 hash;
4586 
4587 			put_page(old_page);
4588 			BUG_ON(huge_pte_none(pte));
4589 			/*
4590 			 * Drop hugetlb_fault_mutex and i_mmap_rwsem before
4591 			 * unmapping.  unmapping needs to hold i_mmap_rwsem
4592 			 * in write mode.  Dropping i_mmap_rwsem in read mode
4593 			 * here is OK as COW mappings do not interact with
4594 			 * PMD sharing.
4595 			 *
4596 			 * Reacquire both after unmap operation.
4597 			 */
4598 			idx = vma_hugecache_offset(h, vma, haddr);
4599 			hash = hugetlb_fault_mutex_hash(mapping, idx);
4600 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4601 			i_mmap_unlock_read(mapping);
4602 
4603 			unmap_ref_private(mm, vma, old_page, haddr);
4604 
4605 			i_mmap_lock_read(mapping);
4606 			mutex_lock(&hugetlb_fault_mutex_table[hash]);
4607 			spin_lock(ptl);
4608 			ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4609 			if (likely(ptep &&
4610 				   pte_same(huge_ptep_get(ptep), pte)))
4611 				goto retry_avoidcopy;
4612 			/*
4613 			 * race occurs while re-acquiring page table
4614 			 * lock, and our job is done.
4615 			 */
4616 			return 0;
4617 		}
4618 
4619 		ret = vmf_error(PTR_ERR(new_page));
4620 		goto out_release_old;
4621 	}
4622 
4623 	/*
4624 	 * When the original hugepage is shared one, it does not have
4625 	 * anon_vma prepared.
4626 	 */
4627 	if (unlikely(anon_vma_prepare(vma))) {
4628 		ret = VM_FAULT_OOM;
4629 		goto out_release_all;
4630 	}
4631 
4632 	copy_user_huge_page(new_page, old_page, address, vma,
4633 			    pages_per_huge_page(h));
4634 	__SetPageUptodate(new_page);
4635 
4636 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr,
4637 				haddr + huge_page_size(h));
4638 	mmu_notifier_invalidate_range_start(&range);
4639 
4640 	/*
4641 	 * Retake the page table lock to check for racing updates
4642 	 * before the page tables are altered
4643 	 */
4644 	spin_lock(ptl);
4645 	ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4646 	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
4647 		ClearHPageRestoreReserve(new_page);
4648 
4649 		/* Break COW */
4650 		huge_ptep_clear_flush(vma, haddr, ptep);
4651 		mmu_notifier_invalidate_range(mm, range.start, range.end);
4652 		set_huge_pte_at(mm, haddr, ptep,
4653 				make_huge_pte(vma, new_page, 1));
4654 		page_remove_rmap(old_page, true);
4655 		hugepage_add_new_anon_rmap(new_page, vma, haddr);
4656 		SetHPageMigratable(new_page);
4657 		/* Make the old page be freed below */
4658 		new_page = old_page;
4659 	}
4660 	spin_unlock(ptl);
4661 	mmu_notifier_invalidate_range_end(&range);
4662 out_release_all:
4663 	restore_reserve_on_error(h, vma, haddr, new_page);
4664 	put_page(new_page);
4665 out_release_old:
4666 	put_page(old_page);
4667 
4668 	spin_lock(ptl); /* Caller expects lock to be held */
4669 	return ret;
4670 }
4671 
4672 /* Return the pagecache page at a given address within a VMA */
4673 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
4674 			struct vm_area_struct *vma, unsigned long address)
4675 {
4676 	struct address_space *mapping;
4677 	pgoff_t idx;
4678 
4679 	mapping = vma->vm_file->f_mapping;
4680 	idx = vma_hugecache_offset(h, vma, address);
4681 
4682 	return find_lock_page(mapping, idx);
4683 }
4684 
4685 /*
4686  * Return whether there is a pagecache page to back given address within VMA.
4687  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
4688  */
4689 static bool hugetlbfs_pagecache_present(struct hstate *h,
4690 			struct vm_area_struct *vma, unsigned long address)
4691 {
4692 	struct address_space *mapping;
4693 	pgoff_t idx;
4694 	struct page *page;
4695 
4696 	mapping = vma->vm_file->f_mapping;
4697 	idx = vma_hugecache_offset(h, vma, address);
4698 
4699 	page = find_get_page(mapping, idx);
4700 	if (page)
4701 		put_page(page);
4702 	return page != NULL;
4703 }
4704 
4705 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
4706 			   pgoff_t idx)
4707 {
4708 	struct inode *inode = mapping->host;
4709 	struct hstate *h = hstate_inode(inode);
4710 	int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
4711 
4712 	if (err)
4713 		return err;
4714 	ClearHPageRestoreReserve(page);
4715 
4716 	/*
4717 	 * set page dirty so that it will not be removed from cache/file
4718 	 * by non-hugetlbfs specific code paths.
4719 	 */
4720 	set_page_dirty(page);
4721 
4722 	spin_lock(&inode->i_lock);
4723 	inode->i_blocks += blocks_per_huge_page(h);
4724 	spin_unlock(&inode->i_lock);
4725 	return 0;
4726 }
4727 
4728 static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma,
4729 						  struct address_space *mapping,
4730 						  pgoff_t idx,
4731 						  unsigned int flags,
4732 						  unsigned long haddr,
4733 						  unsigned long reason)
4734 {
4735 	vm_fault_t ret;
4736 	u32 hash;
4737 	struct vm_fault vmf = {
4738 		.vma = vma,
4739 		.address = haddr,
4740 		.flags = flags,
4741 
4742 		/*
4743 		 * Hard to debug if it ends up being
4744 		 * used by a callee that assumes
4745 		 * something about the other
4746 		 * uninitialized fields... same as in
4747 		 * memory.c
4748 		 */
4749 	};
4750 
4751 	/*
4752 	 * hugetlb_fault_mutex and i_mmap_rwsem must be
4753 	 * dropped before handling userfault.  Reacquire
4754 	 * after handling fault to make calling code simpler.
4755 	 */
4756 	hash = hugetlb_fault_mutex_hash(mapping, idx);
4757 	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
4758 	i_mmap_unlock_read(mapping);
4759 	ret = handle_userfault(&vmf, reason);
4760 	i_mmap_lock_read(mapping);
4761 	mutex_lock(&hugetlb_fault_mutex_table[hash]);
4762 
4763 	return ret;
4764 }
4765 
4766 static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
4767 			struct vm_area_struct *vma,
4768 			struct address_space *mapping, pgoff_t idx,
4769 			unsigned long address, pte_t *ptep, unsigned int flags)
4770 {
4771 	struct hstate *h = hstate_vma(vma);
4772 	vm_fault_t ret = VM_FAULT_SIGBUS;
4773 	int anon_rmap = 0;
4774 	unsigned long size;
4775 	struct page *page;
4776 	pte_t new_pte;
4777 	spinlock_t *ptl;
4778 	unsigned long haddr = address & huge_page_mask(h);
4779 	bool new_page = false;
4780 
4781 	/*
4782 	 * Currently, we are forced to kill the process in the event the
4783 	 * original mapper has unmapped pages from the child due to a failed
4784 	 * COW. Warn that such a situation has occurred as it may not be obvious
4785 	 */
4786 	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
4787 		pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
4788 			   current->pid);
4789 		return ret;
4790 	}
4791 
4792 	/*
4793 	 * We can not race with truncation due to holding i_mmap_rwsem.
4794 	 * i_size is modified when holding i_mmap_rwsem, so check here
4795 	 * once for faults beyond end of file.
4796 	 */
4797 	size = i_size_read(mapping->host) >> huge_page_shift(h);
4798 	if (idx >= size)
4799 		goto out;
4800 
4801 retry:
4802 	page = find_lock_page(mapping, idx);
4803 	if (!page) {
4804 		/* Check for page in userfault range */
4805 		if (userfaultfd_missing(vma)) {
4806 			ret = hugetlb_handle_userfault(vma, mapping, idx,
4807 						       flags, haddr,
4808 						       VM_UFFD_MISSING);
4809 			goto out;
4810 		}
4811 
4812 		page = alloc_huge_page(vma, haddr, 0);
4813 		if (IS_ERR(page)) {
4814 			/*
4815 			 * Returning error will result in faulting task being
4816 			 * sent SIGBUS.  The hugetlb fault mutex prevents two
4817 			 * tasks from racing to fault in the same page which
4818 			 * could result in false unable to allocate errors.
4819 			 * Page migration does not take the fault mutex, but
4820 			 * does a clear then write of pte's under page table
4821 			 * lock.  Page fault code could race with migration,
4822 			 * notice the clear pte and try to allocate a page
4823 			 * here.  Before returning error, get ptl and make
4824 			 * sure there really is no pte entry.
4825 			 */
4826 			ptl = huge_pte_lock(h, mm, ptep);
4827 			ret = 0;
4828 			if (huge_pte_none(huge_ptep_get(ptep)))
4829 				ret = vmf_error(PTR_ERR(page));
4830 			spin_unlock(ptl);
4831 			goto out;
4832 		}
4833 		clear_huge_page(page, address, pages_per_huge_page(h));
4834 		__SetPageUptodate(page);
4835 		new_page = true;
4836 
4837 		if (vma->vm_flags & VM_MAYSHARE) {
4838 			int err = huge_add_to_page_cache(page, mapping, idx);
4839 			if (err) {
4840 				put_page(page);
4841 				if (err == -EEXIST)
4842 					goto retry;
4843 				goto out;
4844 			}
4845 		} else {
4846 			lock_page(page);
4847 			if (unlikely(anon_vma_prepare(vma))) {
4848 				ret = VM_FAULT_OOM;
4849 				goto backout_unlocked;
4850 			}
4851 			anon_rmap = 1;
4852 		}
4853 	} else {
4854 		/*
4855 		 * If memory error occurs between mmap() and fault, some process
4856 		 * don't have hwpoisoned swap entry for errored virtual address.
4857 		 * So we need to block hugepage fault by PG_hwpoison bit check.
4858 		 */
4859 		if (unlikely(PageHWPoison(page))) {
4860 			ret = VM_FAULT_HWPOISON_LARGE |
4861 				VM_FAULT_SET_HINDEX(hstate_index(h));
4862 			goto backout_unlocked;
4863 		}
4864 
4865 		/* Check for page in userfault range. */
4866 		if (userfaultfd_minor(vma)) {
4867 			unlock_page(page);
4868 			put_page(page);
4869 			ret = hugetlb_handle_userfault(vma, mapping, idx,
4870 						       flags, haddr,
4871 						       VM_UFFD_MINOR);
4872 			goto out;
4873 		}
4874 	}
4875 
4876 	/*
4877 	 * If we are going to COW a private mapping later, we examine the
4878 	 * pending reservations for this page now. This will ensure that
4879 	 * any allocations necessary to record that reservation occur outside
4880 	 * the spinlock.
4881 	 */
4882 	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
4883 		if (vma_needs_reservation(h, vma, haddr) < 0) {
4884 			ret = VM_FAULT_OOM;
4885 			goto backout_unlocked;
4886 		}
4887 		/* Just decrements count, does not deallocate */
4888 		vma_end_reservation(h, vma, haddr);
4889 	}
4890 
4891 	ptl = huge_pte_lock(h, mm, ptep);
4892 	ret = 0;
4893 	if (!huge_pte_none(huge_ptep_get(ptep)))
4894 		goto backout;
4895 
4896 	if (anon_rmap) {
4897 		ClearHPageRestoreReserve(page);
4898 		hugepage_add_new_anon_rmap(page, vma, haddr);
4899 	} else
4900 		page_dup_rmap(page, true);
4901 	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
4902 				&& (vma->vm_flags & VM_SHARED)));
4903 	set_huge_pte_at(mm, haddr, ptep, new_pte);
4904 
4905 	hugetlb_count_add(pages_per_huge_page(h), mm);
4906 	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
4907 		/* Optimization, do the COW without a second fault */
4908 		ret = hugetlb_cow(mm, vma, address, ptep, page, ptl);
4909 	}
4910 
4911 	spin_unlock(ptl);
4912 
4913 	/*
4914 	 * Only set HPageMigratable in newly allocated pages.  Existing pages
4915 	 * found in the pagecache may not have HPageMigratableset if they have
4916 	 * been isolated for migration.
4917 	 */
4918 	if (new_page)
4919 		SetHPageMigratable(page);
4920 
4921 	unlock_page(page);
4922 out:
4923 	return ret;
4924 
4925 backout:
4926 	spin_unlock(ptl);
4927 backout_unlocked:
4928 	unlock_page(page);
4929 	restore_reserve_on_error(h, vma, haddr, page);
4930 	put_page(page);
4931 	goto out;
4932 }
4933 
4934 #ifdef CONFIG_SMP
4935 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
4936 {
4937 	unsigned long key[2];
4938 	u32 hash;
4939 
4940 	key[0] = (unsigned long) mapping;
4941 	key[1] = idx;
4942 
4943 	hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
4944 
4945 	return hash & (num_fault_mutexes - 1);
4946 }
4947 #else
4948 /*
4949  * For uniprocessor systems we always use a single mutex, so just
4950  * return 0 and avoid the hashing overhead.
4951  */
4952 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
4953 {
4954 	return 0;
4955 }
4956 #endif
4957 
4958 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
4959 			unsigned long address, unsigned int flags)
4960 {
4961 	pte_t *ptep, entry;
4962 	spinlock_t *ptl;
4963 	vm_fault_t ret;
4964 	u32 hash;
4965 	pgoff_t idx;
4966 	struct page *page = NULL;
4967 	struct page *pagecache_page = NULL;
4968 	struct hstate *h = hstate_vma(vma);
4969 	struct address_space *mapping;
4970 	int need_wait_lock = 0;
4971 	unsigned long haddr = address & huge_page_mask(h);
4972 
4973 	ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
4974 	if (ptep) {
4975 		/*
4976 		 * Since we hold no locks, ptep could be stale.  That is
4977 		 * OK as we are only making decisions based on content and
4978 		 * not actually modifying content here.
4979 		 */
4980 		entry = huge_ptep_get(ptep);
4981 		if (unlikely(is_hugetlb_entry_migration(entry))) {
4982 			migration_entry_wait_huge(vma, mm, ptep);
4983 			return 0;
4984 		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
4985 			return VM_FAULT_HWPOISON_LARGE |
4986 				VM_FAULT_SET_HINDEX(hstate_index(h));
4987 	}
4988 
4989 	/*
4990 	 * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold
4991 	 * until finished with ptep.  This serves two purposes:
4992 	 * 1) It prevents huge_pmd_unshare from being called elsewhere
4993 	 *    and making the ptep no longer valid.
4994 	 * 2) It synchronizes us with i_size modifications during truncation.
4995 	 *
4996 	 * ptep could have already be assigned via huge_pte_offset.  That
4997 	 * is OK, as huge_pte_alloc will return the same value unless
4998 	 * something has changed.
4999 	 */
5000 	mapping = vma->vm_file->f_mapping;
5001 	i_mmap_lock_read(mapping);
5002 	ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
5003 	if (!ptep) {
5004 		i_mmap_unlock_read(mapping);
5005 		return VM_FAULT_OOM;
5006 	}
5007 
5008 	/*
5009 	 * Serialize hugepage allocation and instantiation, so that we don't
5010 	 * get spurious allocation failures if two CPUs race to instantiate
5011 	 * the same page in the page cache.
5012 	 */
5013 	idx = vma_hugecache_offset(h, vma, haddr);
5014 	hash = hugetlb_fault_mutex_hash(mapping, idx);
5015 	mutex_lock(&hugetlb_fault_mutex_table[hash]);
5016 
5017 	entry = huge_ptep_get(ptep);
5018 	if (huge_pte_none(entry)) {
5019 		ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep, flags);
5020 		goto out_mutex;
5021 	}
5022 
5023 	ret = 0;
5024 
5025 	/*
5026 	 * entry could be a migration/hwpoison entry at this point, so this
5027 	 * check prevents the kernel from going below assuming that we have
5028 	 * an active hugepage in pagecache. This goto expects the 2nd page
5029 	 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will
5030 	 * properly handle it.
5031 	 */
5032 	if (!pte_present(entry))
5033 		goto out_mutex;
5034 
5035 	/*
5036 	 * If we are going to COW the mapping later, we examine the pending
5037 	 * reservations for this page now. This will ensure that any
5038 	 * allocations necessary to record that reservation occur outside the
5039 	 * spinlock. For private mappings, we also lookup the pagecache
5040 	 * page now as it is used to determine if a reservation has been
5041 	 * consumed.
5042 	 */
5043 	if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
5044 		if (vma_needs_reservation(h, vma, haddr) < 0) {
5045 			ret = VM_FAULT_OOM;
5046 			goto out_mutex;
5047 		}
5048 		/* Just decrements count, does not deallocate */
5049 		vma_end_reservation(h, vma, haddr);
5050 
5051 		if (!(vma->vm_flags & VM_MAYSHARE))
5052 			pagecache_page = hugetlbfs_pagecache_page(h,
5053 								vma, haddr);
5054 	}
5055 
5056 	ptl = huge_pte_lock(h, mm, ptep);
5057 
5058 	/* Check for a racing update before calling hugetlb_cow */
5059 	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
5060 		goto out_ptl;
5061 
5062 	/*
5063 	 * hugetlb_cow() requires page locks of pte_page(entry) and
5064 	 * pagecache_page, so here we need take the former one
5065 	 * when page != pagecache_page or !pagecache_page.
5066 	 */
5067 	page = pte_page(entry);
5068 	if (page != pagecache_page)
5069 		if (!trylock_page(page)) {
5070 			need_wait_lock = 1;
5071 			goto out_ptl;
5072 		}
5073 
5074 	get_page(page);
5075 
5076 	if (flags & FAULT_FLAG_WRITE) {
5077 		if (!huge_pte_write(entry)) {
5078 			ret = hugetlb_cow(mm, vma, address, ptep,
5079 					  pagecache_page, ptl);
5080 			goto out_put_page;
5081 		}
5082 		entry = huge_pte_mkdirty(entry);
5083 	}
5084 	entry = pte_mkyoung(entry);
5085 	if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
5086 						flags & FAULT_FLAG_WRITE))
5087 		update_mmu_cache(vma, haddr, ptep);
5088 out_put_page:
5089 	if (page != pagecache_page)
5090 		unlock_page(page);
5091 	put_page(page);
5092 out_ptl:
5093 	spin_unlock(ptl);
5094 
5095 	if (pagecache_page) {
5096 		unlock_page(pagecache_page);
5097 		put_page(pagecache_page);
5098 	}
5099 out_mutex:
5100 	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5101 	i_mmap_unlock_read(mapping);
5102 	/*
5103 	 * Generally it's safe to hold refcount during waiting page lock. But
5104 	 * here we just wait to defer the next page fault to avoid busy loop and
5105 	 * the page is not used after unlocked before returning from the current
5106 	 * page fault. So we are safe from accessing freed page, even if we wait
5107 	 * here without taking refcount.
5108 	 */
5109 	if (need_wait_lock)
5110 		wait_on_page_locked(page);
5111 	return ret;
5112 }
5113 
5114 #ifdef CONFIG_USERFAULTFD
5115 /*
5116  * Used by userfaultfd UFFDIO_COPY.  Based on mcopy_atomic_pte with
5117  * modifications for huge pages.
5118  */
5119 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
5120 			    pte_t *dst_pte,
5121 			    struct vm_area_struct *dst_vma,
5122 			    unsigned long dst_addr,
5123 			    unsigned long src_addr,
5124 			    enum mcopy_atomic_mode mode,
5125 			    struct page **pagep)
5126 {
5127 	bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE);
5128 	struct hstate *h = hstate_vma(dst_vma);
5129 	struct address_space *mapping = dst_vma->vm_file->f_mapping;
5130 	pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
5131 	unsigned long size;
5132 	int vm_shared = dst_vma->vm_flags & VM_SHARED;
5133 	pte_t _dst_pte;
5134 	spinlock_t *ptl;
5135 	int ret = -ENOMEM;
5136 	struct page *page;
5137 	int writable;
5138 
5139 	if (is_continue) {
5140 		ret = -EFAULT;
5141 		page = find_lock_page(mapping, idx);
5142 		if (!page)
5143 			goto out;
5144 	} else if (!*pagep) {
5145 		/* If a page already exists, then it's UFFDIO_COPY for
5146 		 * a non-missing case. Return -EEXIST.
5147 		 */
5148 		if (vm_shared &&
5149 		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
5150 			ret = -EEXIST;
5151 			goto out;
5152 		}
5153 
5154 		page = alloc_huge_page(dst_vma, dst_addr, 0);
5155 		if (IS_ERR(page)) {
5156 			ret = -ENOMEM;
5157 			goto out;
5158 		}
5159 
5160 		ret = copy_huge_page_from_user(page,
5161 						(const void __user *) src_addr,
5162 						pages_per_huge_page(h), false);
5163 
5164 		/* fallback to copy_from_user outside mmap_lock */
5165 		if (unlikely(ret)) {
5166 			ret = -ENOENT;
5167 			/* Free the allocated page which may have
5168 			 * consumed a reservation.
5169 			 */
5170 			restore_reserve_on_error(h, dst_vma, dst_addr, page);
5171 			put_page(page);
5172 
5173 			/* Allocate a temporary page to hold the copied
5174 			 * contents.
5175 			 */
5176 			page = alloc_huge_page_vma(h, dst_vma, dst_addr);
5177 			if (!page) {
5178 				ret = -ENOMEM;
5179 				goto out;
5180 			}
5181 			*pagep = page;
5182 			/* Set the outparam pagep and return to the caller to
5183 			 * copy the contents outside the lock. Don't free the
5184 			 * page.
5185 			 */
5186 			goto out;
5187 		}
5188 	} else {
5189 		if (vm_shared &&
5190 		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
5191 			put_page(*pagep);
5192 			ret = -EEXIST;
5193 			*pagep = NULL;
5194 			goto out;
5195 		}
5196 
5197 		page = alloc_huge_page(dst_vma, dst_addr, 0);
5198 		if (IS_ERR(page)) {
5199 			ret = -ENOMEM;
5200 			*pagep = NULL;
5201 			goto out;
5202 		}
5203 		copy_huge_page(page, *pagep);
5204 		put_page(*pagep);
5205 		*pagep = NULL;
5206 	}
5207 
5208 	/*
5209 	 * The memory barrier inside __SetPageUptodate makes sure that
5210 	 * preceding stores to the page contents become visible before
5211 	 * the set_pte_at() write.
5212 	 */
5213 	__SetPageUptodate(page);
5214 
5215 	/* Add shared, newly allocated pages to the page cache. */
5216 	if (vm_shared && !is_continue) {
5217 		size = i_size_read(mapping->host) >> huge_page_shift(h);
5218 		ret = -EFAULT;
5219 		if (idx >= size)
5220 			goto out_release_nounlock;
5221 
5222 		/*
5223 		 * Serialization between remove_inode_hugepages() and
5224 		 * huge_add_to_page_cache() below happens through the
5225 		 * hugetlb_fault_mutex_table that here must be hold by
5226 		 * the caller.
5227 		 */
5228 		ret = huge_add_to_page_cache(page, mapping, idx);
5229 		if (ret)
5230 			goto out_release_nounlock;
5231 	}
5232 
5233 	ptl = huge_pte_lockptr(h, dst_mm, dst_pte);
5234 	spin_lock(ptl);
5235 
5236 	/*
5237 	 * Recheck the i_size after holding PT lock to make sure not
5238 	 * to leave any page mapped (as page_mapped()) beyond the end
5239 	 * of the i_size (remove_inode_hugepages() is strict about
5240 	 * enforcing that). If we bail out here, we'll also leave a
5241 	 * page in the radix tree in the vm_shared case beyond the end
5242 	 * of the i_size, but remove_inode_hugepages() will take care
5243 	 * of it as soon as we drop the hugetlb_fault_mutex_table.
5244 	 */
5245 	size = i_size_read(mapping->host) >> huge_page_shift(h);
5246 	ret = -EFAULT;
5247 	if (idx >= size)
5248 		goto out_release_unlock;
5249 
5250 	ret = -EEXIST;
5251 	if (!huge_pte_none(huge_ptep_get(dst_pte)))
5252 		goto out_release_unlock;
5253 
5254 	if (vm_shared) {
5255 		page_dup_rmap(page, true);
5256 	} else {
5257 		ClearHPageRestoreReserve(page);
5258 		hugepage_add_new_anon_rmap(page, dst_vma, dst_addr);
5259 	}
5260 
5261 	/* For CONTINUE on a non-shared VMA, don't set VM_WRITE for CoW. */
5262 	if (is_continue && !vm_shared)
5263 		writable = 0;
5264 	else
5265 		writable = dst_vma->vm_flags & VM_WRITE;
5266 
5267 	_dst_pte = make_huge_pte(dst_vma, page, writable);
5268 	if (writable)
5269 		_dst_pte = huge_pte_mkdirty(_dst_pte);
5270 	_dst_pte = pte_mkyoung(_dst_pte);
5271 
5272 	set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
5273 
5274 	(void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte,
5275 					dst_vma->vm_flags & VM_WRITE);
5276 	hugetlb_count_add(pages_per_huge_page(h), dst_mm);
5277 
5278 	/* No need to invalidate - it was non-present before */
5279 	update_mmu_cache(dst_vma, dst_addr, dst_pte);
5280 
5281 	spin_unlock(ptl);
5282 	if (!is_continue)
5283 		SetHPageMigratable(page);
5284 	if (vm_shared || is_continue)
5285 		unlock_page(page);
5286 	ret = 0;
5287 out:
5288 	return ret;
5289 out_release_unlock:
5290 	spin_unlock(ptl);
5291 	if (vm_shared || is_continue)
5292 		unlock_page(page);
5293 out_release_nounlock:
5294 	restore_reserve_on_error(h, dst_vma, dst_addr, page);
5295 	put_page(page);
5296 	goto out;
5297 }
5298 #endif /* CONFIG_USERFAULTFD */
5299 
5300 static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma,
5301 				 int refs, struct page **pages,
5302 				 struct vm_area_struct **vmas)
5303 {
5304 	int nr;
5305 
5306 	for (nr = 0; nr < refs; nr++) {
5307 		if (likely(pages))
5308 			pages[nr] = mem_map_offset(page, nr);
5309 		if (vmas)
5310 			vmas[nr] = vma;
5311 	}
5312 }
5313 
5314 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
5315 			 struct page **pages, struct vm_area_struct **vmas,
5316 			 unsigned long *position, unsigned long *nr_pages,
5317 			 long i, unsigned int flags, int *locked)
5318 {
5319 	unsigned long pfn_offset;
5320 	unsigned long vaddr = *position;
5321 	unsigned long remainder = *nr_pages;
5322 	struct hstate *h = hstate_vma(vma);
5323 	int err = -EFAULT, refs;
5324 
5325 	while (vaddr < vma->vm_end && remainder) {
5326 		pte_t *pte;
5327 		spinlock_t *ptl = NULL;
5328 		int absent;
5329 		struct page *page;
5330 
5331 		/*
5332 		 * If we have a pending SIGKILL, don't keep faulting pages and
5333 		 * potentially allocating memory.
5334 		 */
5335 		if (fatal_signal_pending(current)) {
5336 			remainder = 0;
5337 			break;
5338 		}
5339 
5340 		/*
5341 		 * Some archs (sparc64, sh*) have multiple pte_ts to
5342 		 * each hugepage.  We have to make sure we get the
5343 		 * first, for the page indexing below to work.
5344 		 *
5345 		 * Note that page table lock is not held when pte is null.
5346 		 */
5347 		pte = huge_pte_offset(mm, vaddr & huge_page_mask(h),
5348 				      huge_page_size(h));
5349 		if (pte)
5350 			ptl = huge_pte_lock(h, mm, pte);
5351 		absent = !pte || huge_pte_none(huge_ptep_get(pte));
5352 
5353 		/*
5354 		 * When coredumping, it suits get_dump_page if we just return
5355 		 * an error where there's an empty slot with no huge pagecache
5356 		 * to back it.  This way, we avoid allocating a hugepage, and
5357 		 * the sparse dumpfile avoids allocating disk blocks, but its
5358 		 * huge holes still show up with zeroes where they need to be.
5359 		 */
5360 		if (absent && (flags & FOLL_DUMP) &&
5361 		    !hugetlbfs_pagecache_present(h, vma, vaddr)) {
5362 			if (pte)
5363 				spin_unlock(ptl);
5364 			remainder = 0;
5365 			break;
5366 		}
5367 
5368 		/*
5369 		 * We need call hugetlb_fault for both hugepages under migration
5370 		 * (in which case hugetlb_fault waits for the migration,) and
5371 		 * hwpoisoned hugepages (in which case we need to prevent the
5372 		 * caller from accessing to them.) In order to do this, we use
5373 		 * here is_swap_pte instead of is_hugetlb_entry_migration and
5374 		 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
5375 		 * both cases, and because we can't follow correct pages
5376 		 * directly from any kind of swap entries.
5377 		 */
5378 		if (absent || is_swap_pte(huge_ptep_get(pte)) ||
5379 		    ((flags & FOLL_WRITE) &&
5380 		      !huge_pte_write(huge_ptep_get(pte)))) {
5381 			vm_fault_t ret;
5382 			unsigned int fault_flags = 0;
5383 
5384 			if (pte)
5385 				spin_unlock(ptl);
5386 			if (flags & FOLL_WRITE)
5387 				fault_flags |= FAULT_FLAG_WRITE;
5388 			if (locked)
5389 				fault_flags |= FAULT_FLAG_ALLOW_RETRY |
5390 					FAULT_FLAG_KILLABLE;
5391 			if (flags & FOLL_NOWAIT)
5392 				fault_flags |= FAULT_FLAG_ALLOW_RETRY |
5393 					FAULT_FLAG_RETRY_NOWAIT;
5394 			if (flags & FOLL_TRIED) {
5395 				/*
5396 				 * Note: FAULT_FLAG_ALLOW_RETRY and
5397 				 * FAULT_FLAG_TRIED can co-exist
5398 				 */
5399 				fault_flags |= FAULT_FLAG_TRIED;
5400 			}
5401 			ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
5402 			if (ret & VM_FAULT_ERROR) {
5403 				err = vm_fault_to_errno(ret, flags);
5404 				remainder = 0;
5405 				break;
5406 			}
5407 			if (ret & VM_FAULT_RETRY) {
5408 				if (locked &&
5409 				    !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
5410 					*locked = 0;
5411 				*nr_pages = 0;
5412 				/*
5413 				 * VM_FAULT_RETRY must not return an
5414 				 * error, it will return zero
5415 				 * instead.
5416 				 *
5417 				 * No need to update "position" as the
5418 				 * caller will not check it after
5419 				 * *nr_pages is set to 0.
5420 				 */
5421 				return i;
5422 			}
5423 			continue;
5424 		}
5425 
5426 		pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
5427 		page = pte_page(huge_ptep_get(pte));
5428 
5429 		/*
5430 		 * If subpage information not requested, update counters
5431 		 * and skip the same_page loop below.
5432 		 */
5433 		if (!pages && !vmas && !pfn_offset &&
5434 		    (vaddr + huge_page_size(h) < vma->vm_end) &&
5435 		    (remainder >= pages_per_huge_page(h))) {
5436 			vaddr += huge_page_size(h);
5437 			remainder -= pages_per_huge_page(h);
5438 			i += pages_per_huge_page(h);
5439 			spin_unlock(ptl);
5440 			continue;
5441 		}
5442 
5443 		refs = min3(pages_per_huge_page(h) - pfn_offset,
5444 			    (vma->vm_end - vaddr) >> PAGE_SHIFT, remainder);
5445 
5446 		if (pages || vmas)
5447 			record_subpages_vmas(mem_map_offset(page, pfn_offset),
5448 					     vma, refs,
5449 					     likely(pages) ? pages + i : NULL,
5450 					     vmas ? vmas + i : NULL);
5451 
5452 		if (pages) {
5453 			/*
5454 			 * try_grab_compound_head() should always succeed here,
5455 			 * because: a) we hold the ptl lock, and b) we've just
5456 			 * checked that the huge page is present in the page
5457 			 * tables. If the huge page is present, then the tail
5458 			 * pages must also be present. The ptl prevents the
5459 			 * head page and tail pages from being rearranged in
5460 			 * any way. So this page must be available at this
5461 			 * point, unless the page refcount overflowed:
5462 			 */
5463 			if (WARN_ON_ONCE(!try_grab_compound_head(pages[i],
5464 								 refs,
5465 								 flags))) {
5466 				spin_unlock(ptl);
5467 				remainder = 0;
5468 				err = -ENOMEM;
5469 				break;
5470 			}
5471 		}
5472 
5473 		vaddr += (refs << PAGE_SHIFT);
5474 		remainder -= refs;
5475 		i += refs;
5476 
5477 		spin_unlock(ptl);
5478 	}
5479 	*nr_pages = remainder;
5480 	/*
5481 	 * setting position is actually required only if remainder is
5482 	 * not zero but it's faster not to add a "if (remainder)"
5483 	 * branch.
5484 	 */
5485 	*position = vaddr;
5486 
5487 	return i ? i : err;
5488 }
5489 
5490 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
5491 		unsigned long address, unsigned long end, pgprot_t newprot)
5492 {
5493 	struct mm_struct *mm = vma->vm_mm;
5494 	unsigned long start = address;
5495 	pte_t *ptep;
5496 	pte_t pte;
5497 	struct hstate *h = hstate_vma(vma);
5498 	unsigned long pages = 0;
5499 	bool shared_pmd = false;
5500 	struct mmu_notifier_range range;
5501 
5502 	/*
5503 	 * In the case of shared PMDs, the area to flush could be beyond
5504 	 * start/end.  Set range.start/range.end to cover the maximum possible
5505 	 * range if PMD sharing is possible.
5506 	 */
5507 	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
5508 				0, vma, mm, start, end);
5509 	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5510 
5511 	BUG_ON(address >= end);
5512 	flush_cache_range(vma, range.start, range.end);
5513 
5514 	mmu_notifier_invalidate_range_start(&range);
5515 	i_mmap_lock_write(vma->vm_file->f_mapping);
5516 	for (; address < end; address += huge_page_size(h)) {
5517 		spinlock_t *ptl;
5518 		ptep = huge_pte_offset(mm, address, huge_page_size(h));
5519 		if (!ptep)
5520 			continue;
5521 		ptl = huge_pte_lock(h, mm, ptep);
5522 		if (huge_pmd_unshare(mm, vma, &address, ptep)) {
5523 			pages++;
5524 			spin_unlock(ptl);
5525 			shared_pmd = true;
5526 			continue;
5527 		}
5528 		pte = huge_ptep_get(ptep);
5529 		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
5530 			spin_unlock(ptl);
5531 			continue;
5532 		}
5533 		if (unlikely(is_hugetlb_entry_migration(pte))) {
5534 			swp_entry_t entry = pte_to_swp_entry(pte);
5535 
5536 			if (is_writable_migration_entry(entry)) {
5537 				pte_t newpte;
5538 
5539 				entry = make_readable_migration_entry(
5540 							swp_offset(entry));
5541 				newpte = swp_entry_to_pte(entry);
5542 				set_huge_swap_pte_at(mm, address, ptep,
5543 						     newpte, huge_page_size(h));
5544 				pages++;
5545 			}
5546 			spin_unlock(ptl);
5547 			continue;
5548 		}
5549 		if (!huge_pte_none(pte)) {
5550 			pte_t old_pte;
5551 			unsigned int shift = huge_page_shift(hstate_vma(vma));
5552 
5553 			old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
5554 			pte = pte_mkhuge(huge_pte_modify(old_pte, newprot));
5555 			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
5556 			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
5557 			pages++;
5558 		}
5559 		spin_unlock(ptl);
5560 	}
5561 	/*
5562 	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
5563 	 * may have cleared our pud entry and done put_page on the page table:
5564 	 * once we release i_mmap_rwsem, another task can do the final put_page
5565 	 * and that page table be reused and filled with junk.  If we actually
5566 	 * did unshare a page of pmds, flush the range corresponding to the pud.
5567 	 */
5568 	if (shared_pmd)
5569 		flush_hugetlb_tlb_range(vma, range.start, range.end);
5570 	else
5571 		flush_hugetlb_tlb_range(vma, start, end);
5572 	/*
5573 	 * No need to call mmu_notifier_invalidate_range() we are downgrading
5574 	 * page table protection not changing it to point to a new page.
5575 	 *
5576 	 * See Documentation/vm/mmu_notifier.rst
5577 	 */
5578 	i_mmap_unlock_write(vma->vm_file->f_mapping);
5579 	mmu_notifier_invalidate_range_end(&range);
5580 
5581 	return pages << h->order;
5582 }
5583 
5584 /* Return true if reservation was successful, false otherwise.  */
5585 bool hugetlb_reserve_pages(struct inode *inode,
5586 					long from, long to,
5587 					struct vm_area_struct *vma,
5588 					vm_flags_t vm_flags)
5589 {
5590 	long chg, add = -1;
5591 	struct hstate *h = hstate_inode(inode);
5592 	struct hugepage_subpool *spool = subpool_inode(inode);
5593 	struct resv_map *resv_map;
5594 	struct hugetlb_cgroup *h_cg = NULL;
5595 	long gbl_reserve, regions_needed = 0;
5596 
5597 	/* This should never happen */
5598 	if (from > to) {
5599 		VM_WARN(1, "%s called with a negative range\n", __func__);
5600 		return false;
5601 	}
5602 
5603 	/*
5604 	 * Only apply hugepage reservation if asked. At fault time, an
5605 	 * attempt will be made for VM_NORESERVE to allocate a page
5606 	 * without using reserves
5607 	 */
5608 	if (vm_flags & VM_NORESERVE)
5609 		return true;
5610 
5611 	/*
5612 	 * Shared mappings base their reservation on the number of pages that
5613 	 * are already allocated on behalf of the file. Private mappings need
5614 	 * to reserve the full area even if read-only as mprotect() may be
5615 	 * called to make the mapping read-write. Assume !vma is a shm mapping
5616 	 */
5617 	if (!vma || vma->vm_flags & VM_MAYSHARE) {
5618 		/*
5619 		 * resv_map can not be NULL as hugetlb_reserve_pages is only
5620 		 * called for inodes for which resv_maps were created (see
5621 		 * hugetlbfs_get_inode).
5622 		 */
5623 		resv_map = inode_resv_map(inode);
5624 
5625 		chg = region_chg(resv_map, from, to, &regions_needed);
5626 
5627 	} else {
5628 		/* Private mapping. */
5629 		resv_map = resv_map_alloc();
5630 		if (!resv_map)
5631 			return false;
5632 
5633 		chg = to - from;
5634 
5635 		set_vma_resv_map(vma, resv_map);
5636 		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
5637 	}
5638 
5639 	if (chg < 0)
5640 		goto out_err;
5641 
5642 	if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
5643 				chg * pages_per_huge_page(h), &h_cg) < 0)
5644 		goto out_err;
5645 
5646 	if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) {
5647 		/* For private mappings, the hugetlb_cgroup uncharge info hangs
5648 		 * of the resv_map.
5649 		 */
5650 		resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
5651 	}
5652 
5653 	/*
5654 	 * There must be enough pages in the subpool for the mapping. If
5655 	 * the subpool has a minimum size, there may be some global
5656 	 * reservations already in place (gbl_reserve).
5657 	 */
5658 	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
5659 	if (gbl_reserve < 0)
5660 		goto out_uncharge_cgroup;
5661 
5662 	/*
5663 	 * Check enough hugepages are available for the reservation.
5664 	 * Hand the pages back to the subpool if there are not
5665 	 */
5666 	if (hugetlb_acct_memory(h, gbl_reserve) < 0)
5667 		goto out_put_pages;
5668 
5669 	/*
5670 	 * Account for the reservations made. Shared mappings record regions
5671 	 * that have reservations as they are shared by multiple VMAs.
5672 	 * When the last VMA disappears, the region map says how much
5673 	 * the reservation was and the page cache tells how much of
5674 	 * the reservation was consumed. Private mappings are per-VMA and
5675 	 * only the consumed reservations are tracked. When the VMA
5676 	 * disappears, the original reservation is the VMA size and the
5677 	 * consumed reservations are stored in the map. Hence, nothing
5678 	 * else has to be done for private mappings here
5679 	 */
5680 	if (!vma || vma->vm_flags & VM_MAYSHARE) {
5681 		add = region_add(resv_map, from, to, regions_needed, h, h_cg);
5682 
5683 		if (unlikely(add < 0)) {
5684 			hugetlb_acct_memory(h, -gbl_reserve);
5685 			goto out_put_pages;
5686 		} else if (unlikely(chg > add)) {
5687 			/*
5688 			 * pages in this range were added to the reserve
5689 			 * map between region_chg and region_add.  This
5690 			 * indicates a race with alloc_huge_page.  Adjust
5691 			 * the subpool and reserve counts modified above
5692 			 * based on the difference.
5693 			 */
5694 			long rsv_adjust;
5695 
5696 			/*
5697 			 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
5698 			 * reference to h_cg->css. See comment below for detail.
5699 			 */
5700 			hugetlb_cgroup_uncharge_cgroup_rsvd(
5701 				hstate_index(h),
5702 				(chg - add) * pages_per_huge_page(h), h_cg);
5703 
5704 			rsv_adjust = hugepage_subpool_put_pages(spool,
5705 								chg - add);
5706 			hugetlb_acct_memory(h, -rsv_adjust);
5707 		} else if (h_cg) {
5708 			/*
5709 			 * The file_regions will hold their own reference to
5710 			 * h_cg->css. So we should release the reference held
5711 			 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
5712 			 * done.
5713 			 */
5714 			hugetlb_cgroup_put_rsvd_cgroup(h_cg);
5715 		}
5716 	}
5717 	return true;
5718 
5719 out_put_pages:
5720 	/* put back original number of pages, chg */
5721 	(void)hugepage_subpool_put_pages(spool, chg);
5722 out_uncharge_cgroup:
5723 	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
5724 					    chg * pages_per_huge_page(h), h_cg);
5725 out_err:
5726 	if (!vma || vma->vm_flags & VM_MAYSHARE)
5727 		/* Only call region_abort if the region_chg succeeded but the
5728 		 * region_add failed or didn't run.
5729 		 */
5730 		if (chg >= 0 && add < 0)
5731 			region_abort(resv_map, from, to, regions_needed);
5732 	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
5733 		kref_put(&resv_map->refs, resv_map_release);
5734 	return false;
5735 }
5736 
5737 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
5738 								long freed)
5739 {
5740 	struct hstate *h = hstate_inode(inode);
5741 	struct resv_map *resv_map = inode_resv_map(inode);
5742 	long chg = 0;
5743 	struct hugepage_subpool *spool = subpool_inode(inode);
5744 	long gbl_reserve;
5745 
5746 	/*
5747 	 * Since this routine can be called in the evict inode path for all
5748 	 * hugetlbfs inodes, resv_map could be NULL.
5749 	 */
5750 	if (resv_map) {
5751 		chg = region_del(resv_map, start, end);
5752 		/*
5753 		 * region_del() can fail in the rare case where a region
5754 		 * must be split and another region descriptor can not be
5755 		 * allocated.  If end == LONG_MAX, it will not fail.
5756 		 */
5757 		if (chg < 0)
5758 			return chg;
5759 	}
5760 
5761 	spin_lock(&inode->i_lock);
5762 	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
5763 	spin_unlock(&inode->i_lock);
5764 
5765 	/*
5766 	 * If the subpool has a minimum size, the number of global
5767 	 * reservations to be released may be adjusted.
5768 	 *
5769 	 * Note that !resv_map implies freed == 0. So (chg - freed)
5770 	 * won't go negative.
5771 	 */
5772 	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
5773 	hugetlb_acct_memory(h, -gbl_reserve);
5774 
5775 	return 0;
5776 }
5777 
5778 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
5779 static unsigned long page_table_shareable(struct vm_area_struct *svma,
5780 				struct vm_area_struct *vma,
5781 				unsigned long addr, pgoff_t idx)
5782 {
5783 	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
5784 				svma->vm_start;
5785 	unsigned long sbase = saddr & PUD_MASK;
5786 	unsigned long s_end = sbase + PUD_SIZE;
5787 
5788 	/* Allow segments to share if only one is marked locked */
5789 	unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
5790 	unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
5791 
5792 	/*
5793 	 * match the virtual addresses, permission and the alignment of the
5794 	 * page table page.
5795 	 */
5796 	if (pmd_index(addr) != pmd_index(saddr) ||
5797 	    vm_flags != svm_flags ||
5798 	    !range_in_vma(svma, sbase, s_end))
5799 		return 0;
5800 
5801 	return saddr;
5802 }
5803 
5804 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
5805 {
5806 	unsigned long base = addr & PUD_MASK;
5807 	unsigned long end = base + PUD_SIZE;
5808 
5809 	/*
5810 	 * check on proper vm_flags and page table alignment
5811 	 */
5812 	if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
5813 		return true;
5814 	return false;
5815 }
5816 
5817 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
5818 {
5819 #ifdef CONFIG_USERFAULTFD
5820 	if (uffd_disable_huge_pmd_share(vma))
5821 		return false;
5822 #endif
5823 	return vma_shareable(vma, addr);
5824 }
5825 
5826 /*
5827  * Determine if start,end range within vma could be mapped by shared pmd.
5828  * If yes, adjust start and end to cover range associated with possible
5829  * shared pmd mappings.
5830  */
5831 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
5832 				unsigned long *start, unsigned long *end)
5833 {
5834 	unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
5835 		v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
5836 
5837 	/*
5838 	 * vma needs to span at least one aligned PUD size, and the range
5839 	 * must be at least partially within in.
5840 	 */
5841 	if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
5842 		(*end <= v_start) || (*start >= v_end))
5843 		return;
5844 
5845 	/* Extend the range to be PUD aligned for a worst case scenario */
5846 	if (*start > v_start)
5847 		*start = ALIGN_DOWN(*start, PUD_SIZE);
5848 
5849 	if (*end < v_end)
5850 		*end = ALIGN(*end, PUD_SIZE);
5851 }
5852 
5853 /*
5854  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
5855  * and returns the corresponding pte. While this is not necessary for the
5856  * !shared pmd case because we can allocate the pmd later as well, it makes the
5857  * code much cleaner.
5858  *
5859  * This routine must be called with i_mmap_rwsem held in at least read mode if
5860  * sharing is possible.  For hugetlbfs, this prevents removal of any page
5861  * table entries associated with the address space.  This is important as we
5862  * are setting up sharing based on existing page table entries (mappings).
5863  *
5864  * NOTE: This routine is only called from huge_pte_alloc.  Some callers of
5865  * huge_pte_alloc know that sharing is not possible and do not take
5866  * i_mmap_rwsem as a performance optimization.  This is handled by the
5867  * if !vma_shareable check at the beginning of the routine. i_mmap_rwsem is
5868  * only required for subsequent processing.
5869  */
5870 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
5871 		      unsigned long addr, pud_t *pud)
5872 {
5873 	struct address_space *mapping = vma->vm_file->f_mapping;
5874 	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
5875 			vma->vm_pgoff;
5876 	struct vm_area_struct *svma;
5877 	unsigned long saddr;
5878 	pte_t *spte = NULL;
5879 	pte_t *pte;
5880 	spinlock_t *ptl;
5881 
5882 	i_mmap_assert_locked(mapping);
5883 	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
5884 		if (svma == vma)
5885 			continue;
5886 
5887 		saddr = page_table_shareable(svma, vma, addr, idx);
5888 		if (saddr) {
5889 			spte = huge_pte_offset(svma->vm_mm, saddr,
5890 					       vma_mmu_pagesize(svma));
5891 			if (spte) {
5892 				get_page(virt_to_page(spte));
5893 				break;
5894 			}
5895 		}
5896 	}
5897 
5898 	if (!spte)
5899 		goto out;
5900 
5901 	ptl = huge_pte_lock(hstate_vma(vma), mm, spte);
5902 	if (pud_none(*pud)) {
5903 		pud_populate(mm, pud,
5904 				(pmd_t *)((unsigned long)spte & PAGE_MASK));
5905 		mm_inc_nr_pmds(mm);
5906 	} else {
5907 		put_page(virt_to_page(spte));
5908 	}
5909 	spin_unlock(ptl);
5910 out:
5911 	pte = (pte_t *)pmd_alloc(mm, pud, addr);
5912 	return pte;
5913 }
5914 
5915 /*
5916  * unmap huge page backed by shared pte.
5917  *
5918  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
5919  * indicated by page_count > 1, unmap is achieved by clearing pud and
5920  * decrementing the ref count. If count == 1, the pte page is not shared.
5921  *
5922  * Called with page table lock held and i_mmap_rwsem held in write mode.
5923  *
5924  * returns: 1 successfully unmapped a shared pte page
5925  *	    0 the underlying pte page is not shared, or it is the last user
5926  */
5927 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
5928 					unsigned long *addr, pte_t *ptep)
5929 {
5930 	pgd_t *pgd = pgd_offset(mm, *addr);
5931 	p4d_t *p4d = p4d_offset(pgd, *addr);
5932 	pud_t *pud = pud_offset(p4d, *addr);
5933 
5934 	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
5935 	BUG_ON(page_count(virt_to_page(ptep)) == 0);
5936 	if (page_count(virt_to_page(ptep)) == 1)
5937 		return 0;
5938 
5939 	pud_clear(pud);
5940 	put_page(virt_to_page(ptep));
5941 	mm_dec_nr_pmds(mm);
5942 	*addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;
5943 	return 1;
5944 }
5945 
5946 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
5947 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
5948 		      unsigned long addr, pud_t *pud)
5949 {
5950 	return NULL;
5951 }
5952 
5953 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
5954 				unsigned long *addr, pte_t *ptep)
5955 {
5956 	return 0;
5957 }
5958 
5959 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
5960 				unsigned long *start, unsigned long *end)
5961 {
5962 }
5963 
5964 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
5965 {
5966 	return false;
5967 }
5968 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
5969 
5970 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
5971 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
5972 			unsigned long addr, unsigned long sz)
5973 {
5974 	pgd_t *pgd;
5975 	p4d_t *p4d;
5976 	pud_t *pud;
5977 	pte_t *pte = NULL;
5978 
5979 	pgd = pgd_offset(mm, addr);
5980 	p4d = p4d_alloc(mm, pgd, addr);
5981 	if (!p4d)
5982 		return NULL;
5983 	pud = pud_alloc(mm, p4d, addr);
5984 	if (pud) {
5985 		if (sz == PUD_SIZE) {
5986 			pte = (pte_t *)pud;
5987 		} else {
5988 			BUG_ON(sz != PMD_SIZE);
5989 			if (want_pmd_share(vma, addr) && pud_none(*pud))
5990 				pte = huge_pmd_share(mm, vma, addr, pud);
5991 			else
5992 				pte = (pte_t *)pmd_alloc(mm, pud, addr);
5993 		}
5994 	}
5995 	BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
5996 
5997 	return pte;
5998 }
5999 
6000 /*
6001  * huge_pte_offset() - Walk the page table to resolve the hugepage
6002  * entry at address @addr
6003  *
6004  * Return: Pointer to page table entry (PUD or PMD) for
6005  * address @addr, or NULL if a !p*d_present() entry is encountered and the
6006  * size @sz doesn't match the hugepage size at this level of the page
6007  * table.
6008  */
6009 pte_t *huge_pte_offset(struct mm_struct *mm,
6010 		       unsigned long addr, unsigned long sz)
6011 {
6012 	pgd_t *pgd;
6013 	p4d_t *p4d;
6014 	pud_t *pud;
6015 	pmd_t *pmd;
6016 
6017 	pgd = pgd_offset(mm, addr);
6018 	if (!pgd_present(*pgd))
6019 		return NULL;
6020 	p4d = p4d_offset(pgd, addr);
6021 	if (!p4d_present(*p4d))
6022 		return NULL;
6023 
6024 	pud = pud_offset(p4d, addr);
6025 	if (sz == PUD_SIZE)
6026 		/* must be pud huge, non-present or none */
6027 		return (pte_t *)pud;
6028 	if (!pud_present(*pud))
6029 		return NULL;
6030 	/* must have a valid entry and size to go further */
6031 
6032 	pmd = pmd_offset(pud, addr);
6033 	/* must be pmd huge, non-present or none */
6034 	return (pte_t *)pmd;
6035 }
6036 
6037 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
6038 
6039 /*
6040  * These functions are overwritable if your architecture needs its own
6041  * behavior.
6042  */
6043 struct page * __weak
6044 follow_huge_addr(struct mm_struct *mm, unsigned long address,
6045 			      int write)
6046 {
6047 	return ERR_PTR(-EINVAL);
6048 }
6049 
6050 struct page * __weak
6051 follow_huge_pd(struct vm_area_struct *vma,
6052 	       unsigned long address, hugepd_t hpd, int flags, int pdshift)
6053 {
6054 	WARN(1, "hugepd follow called with no support for hugepage directory format\n");
6055 	return NULL;
6056 }
6057 
6058 struct page * __weak
6059 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
6060 		pmd_t *pmd, int flags)
6061 {
6062 	struct page *page = NULL;
6063 	spinlock_t *ptl;
6064 	pte_t pte;
6065 
6066 	/* FOLL_GET and FOLL_PIN are mutually exclusive. */
6067 	if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
6068 			 (FOLL_PIN | FOLL_GET)))
6069 		return NULL;
6070 
6071 retry:
6072 	ptl = pmd_lockptr(mm, pmd);
6073 	spin_lock(ptl);
6074 	/*
6075 	 * make sure that the address range covered by this pmd is not
6076 	 * unmapped from other threads.
6077 	 */
6078 	if (!pmd_huge(*pmd))
6079 		goto out;
6080 	pte = huge_ptep_get((pte_t *)pmd);
6081 	if (pte_present(pte)) {
6082 		page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
6083 		/*
6084 		 * try_grab_page() should always succeed here, because: a) we
6085 		 * hold the pmd (ptl) lock, and b) we've just checked that the
6086 		 * huge pmd (head) page is present in the page tables. The ptl
6087 		 * prevents the head page and tail pages from being rearranged
6088 		 * in any way. So this page must be available at this point,
6089 		 * unless the page refcount overflowed:
6090 		 */
6091 		if (WARN_ON_ONCE(!try_grab_page(page, flags))) {
6092 			page = NULL;
6093 			goto out;
6094 		}
6095 	} else {
6096 		if (is_hugetlb_entry_migration(pte)) {
6097 			spin_unlock(ptl);
6098 			__migration_entry_wait(mm, (pte_t *)pmd, ptl);
6099 			goto retry;
6100 		}
6101 		/*
6102 		 * hwpoisoned entry is treated as no_page_table in
6103 		 * follow_page_mask().
6104 		 */
6105 	}
6106 out:
6107 	spin_unlock(ptl);
6108 	return page;
6109 }
6110 
6111 struct page * __weak
6112 follow_huge_pud(struct mm_struct *mm, unsigned long address,
6113 		pud_t *pud, int flags)
6114 {
6115 	if (flags & (FOLL_GET | FOLL_PIN))
6116 		return NULL;
6117 
6118 	return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
6119 }
6120 
6121 struct page * __weak
6122 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags)
6123 {
6124 	if (flags & (FOLL_GET | FOLL_PIN))
6125 		return NULL;
6126 
6127 	return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT);
6128 }
6129 
6130 bool isolate_huge_page(struct page *page, struct list_head *list)
6131 {
6132 	bool ret = true;
6133 
6134 	spin_lock_irq(&hugetlb_lock);
6135 	if (!PageHeadHuge(page) ||
6136 	    !HPageMigratable(page) ||
6137 	    !get_page_unless_zero(page)) {
6138 		ret = false;
6139 		goto unlock;
6140 	}
6141 	ClearHPageMigratable(page);
6142 	list_move_tail(&page->lru, list);
6143 unlock:
6144 	spin_unlock_irq(&hugetlb_lock);
6145 	return ret;
6146 }
6147 
6148 int get_hwpoison_huge_page(struct page *page, bool *hugetlb)
6149 {
6150 	int ret = 0;
6151 
6152 	*hugetlb = false;
6153 	spin_lock_irq(&hugetlb_lock);
6154 	if (PageHeadHuge(page)) {
6155 		*hugetlb = true;
6156 		if (HPageFreed(page) || HPageMigratable(page))
6157 			ret = get_page_unless_zero(page);
6158 		else
6159 			ret = -EBUSY;
6160 	}
6161 	spin_unlock_irq(&hugetlb_lock);
6162 	return ret;
6163 }
6164 
6165 void putback_active_hugepage(struct page *page)
6166 {
6167 	spin_lock_irq(&hugetlb_lock);
6168 	SetHPageMigratable(page);
6169 	list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
6170 	spin_unlock_irq(&hugetlb_lock);
6171 	put_page(page);
6172 }
6173 
6174 void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason)
6175 {
6176 	struct hstate *h = page_hstate(oldpage);
6177 
6178 	hugetlb_cgroup_migrate(oldpage, newpage);
6179 	set_page_owner_migrate_reason(newpage, reason);
6180 
6181 	/*
6182 	 * transfer temporary state of the new huge page. This is
6183 	 * reverse to other transitions because the newpage is going to
6184 	 * be final while the old one will be freed so it takes over
6185 	 * the temporary status.
6186 	 *
6187 	 * Also note that we have to transfer the per-node surplus state
6188 	 * here as well otherwise the global surplus count will not match
6189 	 * the per-node's.
6190 	 */
6191 	if (HPageTemporary(newpage)) {
6192 		int old_nid = page_to_nid(oldpage);
6193 		int new_nid = page_to_nid(newpage);
6194 
6195 		SetHPageTemporary(oldpage);
6196 		ClearHPageTemporary(newpage);
6197 
6198 		/*
6199 		 * There is no need to transfer the per-node surplus state
6200 		 * when we do not cross the node.
6201 		 */
6202 		if (new_nid == old_nid)
6203 			return;
6204 		spin_lock_irq(&hugetlb_lock);
6205 		if (h->surplus_huge_pages_node[old_nid]) {
6206 			h->surplus_huge_pages_node[old_nid]--;
6207 			h->surplus_huge_pages_node[new_nid]++;
6208 		}
6209 		spin_unlock_irq(&hugetlb_lock);
6210 	}
6211 }
6212 
6213 /*
6214  * This function will unconditionally remove all the shared pmd pgtable entries
6215  * within the specific vma for a hugetlbfs memory range.
6216  */
6217 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
6218 {
6219 	struct hstate *h = hstate_vma(vma);
6220 	unsigned long sz = huge_page_size(h);
6221 	struct mm_struct *mm = vma->vm_mm;
6222 	struct mmu_notifier_range range;
6223 	unsigned long address, start, end;
6224 	spinlock_t *ptl;
6225 	pte_t *ptep;
6226 
6227 	if (!(vma->vm_flags & VM_MAYSHARE))
6228 		return;
6229 
6230 	start = ALIGN(vma->vm_start, PUD_SIZE);
6231 	end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
6232 
6233 	if (start >= end)
6234 		return;
6235 
6236 	/*
6237 	 * No need to call adjust_range_if_pmd_sharing_possible(), because
6238 	 * we have already done the PUD_SIZE alignment.
6239 	 */
6240 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
6241 				start, end);
6242 	mmu_notifier_invalidate_range_start(&range);
6243 	i_mmap_lock_write(vma->vm_file->f_mapping);
6244 	for (address = start; address < end; address += PUD_SIZE) {
6245 		unsigned long tmp = address;
6246 
6247 		ptep = huge_pte_offset(mm, address, sz);
6248 		if (!ptep)
6249 			continue;
6250 		ptl = huge_pte_lock(h, mm, ptep);
6251 		/* We don't want 'address' to be changed */
6252 		huge_pmd_unshare(mm, vma, &tmp, ptep);
6253 		spin_unlock(ptl);
6254 	}
6255 	flush_hugetlb_tlb_range(vma, start, end);
6256 	i_mmap_unlock_write(vma->vm_file->f_mapping);
6257 	/*
6258 	 * No need to call mmu_notifier_invalidate_range(), see
6259 	 * Documentation/vm/mmu_notifier.rst.
6260 	 */
6261 	mmu_notifier_invalidate_range_end(&range);
6262 }
6263 
6264 #ifdef CONFIG_CMA
6265 static bool cma_reserve_called __initdata;
6266 
6267 static int __init cmdline_parse_hugetlb_cma(char *p)
6268 {
6269 	hugetlb_cma_size = memparse(p, &p);
6270 	return 0;
6271 }
6272 
6273 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
6274 
6275 void __init hugetlb_cma_reserve(int order)
6276 {
6277 	unsigned long size, reserved, per_node;
6278 	int nid;
6279 
6280 	cma_reserve_called = true;
6281 
6282 	if (!hugetlb_cma_size)
6283 		return;
6284 
6285 	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
6286 		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
6287 			(PAGE_SIZE << order) / SZ_1M);
6288 		return;
6289 	}
6290 
6291 	/*
6292 	 * If 3 GB area is requested on a machine with 4 numa nodes,
6293 	 * let's allocate 1 GB on first three nodes and ignore the last one.
6294 	 */
6295 	per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
6296 	pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
6297 		hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
6298 
6299 	reserved = 0;
6300 	for_each_node_state(nid, N_ONLINE) {
6301 		int res;
6302 		char name[CMA_MAX_NAME];
6303 
6304 		size = min(per_node, hugetlb_cma_size - reserved);
6305 		size = round_up(size, PAGE_SIZE << order);
6306 
6307 		snprintf(name, sizeof(name), "hugetlb%d", nid);
6308 		res = cma_declare_contiguous_nid(0, size, 0, PAGE_SIZE << order,
6309 						 0, false, name,
6310 						 &hugetlb_cma[nid], nid);
6311 		if (res) {
6312 			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
6313 				res, nid);
6314 			continue;
6315 		}
6316 
6317 		reserved += size;
6318 		pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
6319 			size / SZ_1M, nid);
6320 
6321 		if (reserved >= hugetlb_cma_size)
6322 			break;
6323 	}
6324 }
6325 
6326 void __init hugetlb_cma_check(void)
6327 {
6328 	if (!hugetlb_cma_size || cma_reserve_called)
6329 		return;
6330 
6331 	pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
6332 }
6333 
6334 #endif /* CONFIG_CMA */
6335