xref: /openbmc/linux/mm/hugetlb.c (revision 2dd6532e)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Generic hugetlb support.
4  * (C) Nadia Yvette Chambers, April 2004
5  */
6 #include <linux/list.h>
7 #include <linux/init.h>
8 #include <linux/mm.h>
9 #include <linux/seq_file.h>
10 #include <linux/sysctl.h>
11 #include <linux/highmem.h>
12 #include <linux/mmu_notifier.h>
13 #include <linux/nodemask.h>
14 #include <linux/pagemap.h>
15 #include <linux/mempolicy.h>
16 #include <linux/compiler.h>
17 #include <linux/cpuset.h>
18 #include <linux/mutex.h>
19 #include <linux/memblock.h>
20 #include <linux/sysfs.h>
21 #include <linux/slab.h>
22 #include <linux/sched/mm.h>
23 #include <linux/mmdebug.h>
24 #include <linux/sched/signal.h>
25 #include <linux/rmap.h>
26 #include <linux/string_helpers.h>
27 #include <linux/swap.h>
28 #include <linux/swapops.h>
29 #include <linux/jhash.h>
30 #include <linux/numa.h>
31 #include <linux/llist.h>
32 #include <linux/cma.h>
33 #include <linux/migrate.h>
34 #include <linux/nospec.h>
35 #include <linux/delayacct.h>
36 
37 #include <asm/page.h>
38 #include <asm/pgalloc.h>
39 #include <asm/tlb.h>
40 
41 #include <linux/io.h>
42 #include <linux/hugetlb.h>
43 #include <linux/hugetlb_cgroup.h>
44 #include <linux/node.h>
45 #include <linux/page_owner.h>
46 #include "internal.h"
47 #include "hugetlb_vmemmap.h"
48 
49 int hugetlb_max_hstate __read_mostly;
50 unsigned int default_hstate_idx;
51 struct hstate hstates[HUGE_MAX_HSTATE];
52 
53 #ifdef CONFIG_CMA
54 static struct cma *hugetlb_cma[MAX_NUMNODES];
55 static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
56 static bool hugetlb_cma_page(struct page *page, unsigned int order)
57 {
58 	return cma_pages_valid(hugetlb_cma[page_to_nid(page)], page,
59 				1 << order);
60 }
61 #else
62 static bool hugetlb_cma_page(struct page *page, unsigned int order)
63 {
64 	return false;
65 }
66 #endif
67 static unsigned long hugetlb_cma_size __initdata;
68 
69 /*
70  * Minimum page order among possible hugepage sizes, set to a proper value
71  * at boot time.
72  */
73 static unsigned int minimum_order __read_mostly = UINT_MAX;
74 
75 __initdata LIST_HEAD(huge_boot_pages);
76 
77 /* for command line parsing */
78 static struct hstate * __initdata parsed_hstate;
79 static unsigned long __initdata default_hstate_max_huge_pages;
80 static bool __initdata parsed_valid_hugepagesz = true;
81 static bool __initdata parsed_default_hugepagesz;
82 static unsigned int default_hugepages_in_node[MAX_NUMNODES] __initdata;
83 
84 /*
85  * Protects updates to hugepage_freelists, hugepage_activelist, nr_huge_pages,
86  * free_huge_pages, and surplus_huge_pages.
87  */
88 DEFINE_SPINLOCK(hugetlb_lock);
89 
90 /*
91  * Serializes faults on the same logical page.  This is used to
92  * prevent spurious OOMs when the hugepage pool is fully utilized.
93  */
94 static int num_fault_mutexes;
95 struct mutex *hugetlb_fault_mutex_table ____cacheline_aligned_in_smp;
96 
97 /* Forward declaration */
98 static int hugetlb_acct_memory(struct hstate *h, long delta);
99 
100 static inline bool subpool_is_free(struct hugepage_subpool *spool)
101 {
102 	if (spool->count)
103 		return false;
104 	if (spool->max_hpages != -1)
105 		return spool->used_hpages == 0;
106 	if (spool->min_hpages != -1)
107 		return spool->rsv_hpages == spool->min_hpages;
108 
109 	return true;
110 }
111 
112 static inline void unlock_or_release_subpool(struct hugepage_subpool *spool,
113 						unsigned long irq_flags)
114 {
115 	spin_unlock_irqrestore(&spool->lock, irq_flags);
116 
117 	/* If no pages are used, and no other handles to the subpool
118 	 * remain, give up any reservations based on minimum size and
119 	 * free the subpool */
120 	if (subpool_is_free(spool)) {
121 		if (spool->min_hpages != -1)
122 			hugetlb_acct_memory(spool->hstate,
123 						-spool->min_hpages);
124 		kfree(spool);
125 	}
126 }
127 
128 struct hugepage_subpool *hugepage_new_subpool(struct hstate *h, long max_hpages,
129 						long min_hpages)
130 {
131 	struct hugepage_subpool *spool;
132 
133 	spool = kzalloc(sizeof(*spool), GFP_KERNEL);
134 	if (!spool)
135 		return NULL;
136 
137 	spin_lock_init(&spool->lock);
138 	spool->count = 1;
139 	spool->max_hpages = max_hpages;
140 	spool->hstate = h;
141 	spool->min_hpages = min_hpages;
142 
143 	if (min_hpages != -1 && hugetlb_acct_memory(h, min_hpages)) {
144 		kfree(spool);
145 		return NULL;
146 	}
147 	spool->rsv_hpages = min_hpages;
148 
149 	return spool;
150 }
151 
152 void hugepage_put_subpool(struct hugepage_subpool *spool)
153 {
154 	unsigned long flags;
155 
156 	spin_lock_irqsave(&spool->lock, flags);
157 	BUG_ON(!spool->count);
158 	spool->count--;
159 	unlock_or_release_subpool(spool, flags);
160 }
161 
162 /*
163  * Subpool accounting for allocating and reserving pages.
164  * Return -ENOMEM if there are not enough resources to satisfy the
165  * request.  Otherwise, return the number of pages by which the
166  * global pools must be adjusted (upward).  The returned value may
167  * only be different than the passed value (delta) in the case where
168  * a subpool minimum size must be maintained.
169  */
170 static long hugepage_subpool_get_pages(struct hugepage_subpool *spool,
171 				      long delta)
172 {
173 	long ret = delta;
174 
175 	if (!spool)
176 		return ret;
177 
178 	spin_lock_irq(&spool->lock);
179 
180 	if (spool->max_hpages != -1) {		/* maximum size accounting */
181 		if ((spool->used_hpages + delta) <= spool->max_hpages)
182 			spool->used_hpages += delta;
183 		else {
184 			ret = -ENOMEM;
185 			goto unlock_ret;
186 		}
187 	}
188 
189 	/* minimum size accounting */
190 	if (spool->min_hpages != -1 && spool->rsv_hpages) {
191 		if (delta > spool->rsv_hpages) {
192 			/*
193 			 * Asking for more reserves than those already taken on
194 			 * behalf of subpool.  Return difference.
195 			 */
196 			ret = delta - spool->rsv_hpages;
197 			spool->rsv_hpages = 0;
198 		} else {
199 			ret = 0;	/* reserves already accounted for */
200 			spool->rsv_hpages -= delta;
201 		}
202 	}
203 
204 unlock_ret:
205 	spin_unlock_irq(&spool->lock);
206 	return ret;
207 }
208 
209 /*
210  * Subpool accounting for freeing and unreserving pages.
211  * Return the number of global page reservations that must be dropped.
212  * The return value may only be different than the passed value (delta)
213  * in the case where a subpool minimum size must be maintained.
214  */
215 static long hugepage_subpool_put_pages(struct hugepage_subpool *spool,
216 				       long delta)
217 {
218 	long ret = delta;
219 	unsigned long flags;
220 
221 	if (!spool)
222 		return delta;
223 
224 	spin_lock_irqsave(&spool->lock, flags);
225 
226 	if (spool->max_hpages != -1)		/* maximum size accounting */
227 		spool->used_hpages -= delta;
228 
229 	 /* minimum size accounting */
230 	if (spool->min_hpages != -1 && spool->used_hpages < spool->min_hpages) {
231 		if (spool->rsv_hpages + delta <= spool->min_hpages)
232 			ret = 0;
233 		else
234 			ret = spool->rsv_hpages + delta - spool->min_hpages;
235 
236 		spool->rsv_hpages += delta;
237 		if (spool->rsv_hpages > spool->min_hpages)
238 			spool->rsv_hpages = spool->min_hpages;
239 	}
240 
241 	/*
242 	 * If hugetlbfs_put_super couldn't free spool due to an outstanding
243 	 * quota reference, free it now.
244 	 */
245 	unlock_or_release_subpool(spool, flags);
246 
247 	return ret;
248 }
249 
250 static inline struct hugepage_subpool *subpool_inode(struct inode *inode)
251 {
252 	return HUGETLBFS_SB(inode->i_sb)->spool;
253 }
254 
255 static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
256 {
257 	return subpool_inode(file_inode(vma->vm_file));
258 }
259 
260 /* Helper that removes a struct file_region from the resv_map cache and returns
261  * it for use.
262  */
263 static struct file_region *
264 get_file_region_entry_from_cache(struct resv_map *resv, long from, long to)
265 {
266 	struct file_region *nrg = NULL;
267 
268 	VM_BUG_ON(resv->region_cache_count <= 0);
269 
270 	resv->region_cache_count--;
271 	nrg = list_first_entry(&resv->region_cache, struct file_region, link);
272 	list_del(&nrg->link);
273 
274 	nrg->from = from;
275 	nrg->to = to;
276 
277 	return nrg;
278 }
279 
280 static void copy_hugetlb_cgroup_uncharge_info(struct file_region *nrg,
281 					      struct file_region *rg)
282 {
283 #ifdef CONFIG_CGROUP_HUGETLB
284 	nrg->reservation_counter = rg->reservation_counter;
285 	nrg->css = rg->css;
286 	if (rg->css)
287 		css_get(rg->css);
288 #endif
289 }
290 
291 /* Helper that records hugetlb_cgroup uncharge info. */
292 static void record_hugetlb_cgroup_uncharge_info(struct hugetlb_cgroup *h_cg,
293 						struct hstate *h,
294 						struct resv_map *resv,
295 						struct file_region *nrg)
296 {
297 #ifdef CONFIG_CGROUP_HUGETLB
298 	if (h_cg) {
299 		nrg->reservation_counter =
300 			&h_cg->rsvd_hugepage[hstate_index(h)];
301 		nrg->css = &h_cg->css;
302 		/*
303 		 * The caller will hold exactly one h_cg->css reference for the
304 		 * whole contiguous reservation region. But this area might be
305 		 * scattered when there are already some file_regions reside in
306 		 * it. As a result, many file_regions may share only one css
307 		 * reference. In order to ensure that one file_region must hold
308 		 * exactly one h_cg->css reference, we should do css_get for
309 		 * each file_region and leave the reference held by caller
310 		 * untouched.
311 		 */
312 		css_get(&h_cg->css);
313 		if (!resv->pages_per_hpage)
314 			resv->pages_per_hpage = pages_per_huge_page(h);
315 		/* pages_per_hpage should be the same for all entries in
316 		 * a resv_map.
317 		 */
318 		VM_BUG_ON(resv->pages_per_hpage != pages_per_huge_page(h));
319 	} else {
320 		nrg->reservation_counter = NULL;
321 		nrg->css = NULL;
322 	}
323 #endif
324 }
325 
326 static void put_uncharge_info(struct file_region *rg)
327 {
328 #ifdef CONFIG_CGROUP_HUGETLB
329 	if (rg->css)
330 		css_put(rg->css);
331 #endif
332 }
333 
334 static bool has_same_uncharge_info(struct file_region *rg,
335 				   struct file_region *org)
336 {
337 #ifdef CONFIG_CGROUP_HUGETLB
338 	return rg->reservation_counter == org->reservation_counter &&
339 	       rg->css == org->css;
340 
341 #else
342 	return true;
343 #endif
344 }
345 
346 static void coalesce_file_region(struct resv_map *resv, struct file_region *rg)
347 {
348 	struct file_region *nrg = NULL, *prg = NULL;
349 
350 	prg = list_prev_entry(rg, link);
351 	if (&prg->link != &resv->regions && prg->to == rg->from &&
352 	    has_same_uncharge_info(prg, rg)) {
353 		prg->to = rg->to;
354 
355 		list_del(&rg->link);
356 		put_uncharge_info(rg);
357 		kfree(rg);
358 
359 		rg = prg;
360 	}
361 
362 	nrg = list_next_entry(rg, link);
363 	if (&nrg->link != &resv->regions && nrg->from == rg->to &&
364 	    has_same_uncharge_info(nrg, rg)) {
365 		nrg->from = rg->from;
366 
367 		list_del(&rg->link);
368 		put_uncharge_info(rg);
369 		kfree(rg);
370 	}
371 }
372 
373 static inline long
374 hugetlb_resv_map_add(struct resv_map *map, struct list_head *rg, long from,
375 		     long to, struct hstate *h, struct hugetlb_cgroup *cg,
376 		     long *regions_needed)
377 {
378 	struct file_region *nrg;
379 
380 	if (!regions_needed) {
381 		nrg = get_file_region_entry_from_cache(map, from, to);
382 		record_hugetlb_cgroup_uncharge_info(cg, h, map, nrg);
383 		list_add(&nrg->link, rg);
384 		coalesce_file_region(map, nrg);
385 	} else
386 		*regions_needed += 1;
387 
388 	return to - from;
389 }
390 
391 /*
392  * Must be called with resv->lock held.
393  *
394  * Calling this with regions_needed != NULL will count the number of pages
395  * to be added but will not modify the linked list. And regions_needed will
396  * indicate the number of file_regions needed in the cache to carry out to add
397  * the regions for this range.
398  */
399 static long add_reservation_in_range(struct resv_map *resv, long f, long t,
400 				     struct hugetlb_cgroup *h_cg,
401 				     struct hstate *h, long *regions_needed)
402 {
403 	long add = 0;
404 	struct list_head *head = &resv->regions;
405 	long last_accounted_offset = f;
406 	struct file_region *iter, *trg = NULL;
407 	struct list_head *rg = NULL;
408 
409 	if (regions_needed)
410 		*regions_needed = 0;
411 
412 	/* In this loop, we essentially handle an entry for the range
413 	 * [last_accounted_offset, iter->from), at every iteration, with some
414 	 * bounds checking.
415 	 */
416 	list_for_each_entry_safe(iter, trg, head, link) {
417 		/* Skip irrelevant regions that start before our range. */
418 		if (iter->from < f) {
419 			/* If this region ends after the last accounted offset,
420 			 * then we need to update last_accounted_offset.
421 			 */
422 			if (iter->to > last_accounted_offset)
423 				last_accounted_offset = iter->to;
424 			continue;
425 		}
426 
427 		/* When we find a region that starts beyond our range, we've
428 		 * finished.
429 		 */
430 		if (iter->from >= t) {
431 			rg = iter->link.prev;
432 			break;
433 		}
434 
435 		/* Add an entry for last_accounted_offset -> iter->from, and
436 		 * update last_accounted_offset.
437 		 */
438 		if (iter->from > last_accounted_offset)
439 			add += hugetlb_resv_map_add(resv, iter->link.prev,
440 						    last_accounted_offset,
441 						    iter->from, h, h_cg,
442 						    regions_needed);
443 
444 		last_accounted_offset = iter->to;
445 	}
446 
447 	/* Handle the case where our range extends beyond
448 	 * last_accounted_offset.
449 	 */
450 	if (!rg)
451 		rg = head->prev;
452 	if (last_accounted_offset < t)
453 		add += hugetlb_resv_map_add(resv, rg, last_accounted_offset,
454 					    t, h, h_cg, regions_needed);
455 
456 	return add;
457 }
458 
459 /* Must be called with resv->lock acquired. Will drop lock to allocate entries.
460  */
461 static int allocate_file_region_entries(struct resv_map *resv,
462 					int regions_needed)
463 	__must_hold(&resv->lock)
464 {
465 	struct list_head allocated_regions;
466 	int to_allocate = 0, i = 0;
467 	struct file_region *trg = NULL, *rg = NULL;
468 
469 	VM_BUG_ON(regions_needed < 0);
470 
471 	INIT_LIST_HEAD(&allocated_regions);
472 
473 	/*
474 	 * Check for sufficient descriptors in the cache to accommodate
475 	 * the number of in progress add operations plus regions_needed.
476 	 *
477 	 * This is a while loop because when we drop the lock, some other call
478 	 * to region_add or region_del may have consumed some region_entries,
479 	 * so we keep looping here until we finally have enough entries for
480 	 * (adds_in_progress + regions_needed).
481 	 */
482 	while (resv->region_cache_count <
483 	       (resv->adds_in_progress + regions_needed)) {
484 		to_allocate = resv->adds_in_progress + regions_needed -
485 			      resv->region_cache_count;
486 
487 		/* At this point, we should have enough entries in the cache
488 		 * for all the existing adds_in_progress. We should only be
489 		 * needing to allocate for regions_needed.
490 		 */
491 		VM_BUG_ON(resv->region_cache_count < resv->adds_in_progress);
492 
493 		spin_unlock(&resv->lock);
494 		for (i = 0; i < to_allocate; i++) {
495 			trg = kmalloc(sizeof(*trg), GFP_KERNEL);
496 			if (!trg)
497 				goto out_of_memory;
498 			list_add(&trg->link, &allocated_regions);
499 		}
500 
501 		spin_lock(&resv->lock);
502 
503 		list_splice(&allocated_regions, &resv->region_cache);
504 		resv->region_cache_count += to_allocate;
505 	}
506 
507 	return 0;
508 
509 out_of_memory:
510 	list_for_each_entry_safe(rg, trg, &allocated_regions, link) {
511 		list_del(&rg->link);
512 		kfree(rg);
513 	}
514 	return -ENOMEM;
515 }
516 
517 /*
518  * Add the huge page range represented by [f, t) to the reserve
519  * map.  Regions will be taken from the cache to fill in this range.
520  * Sufficient regions should exist in the cache due to the previous
521  * call to region_chg with the same range, but in some cases the cache will not
522  * have sufficient entries due to races with other code doing region_add or
523  * region_del.  The extra needed entries will be allocated.
524  *
525  * regions_needed is the out value provided by a previous call to region_chg.
526  *
527  * Return the number of new huge pages added to the map.  This number is greater
528  * than or equal to zero.  If file_region entries needed to be allocated for
529  * this operation and we were not able to allocate, it returns -ENOMEM.
530  * region_add of regions of length 1 never allocate file_regions and cannot
531  * fail; region_chg will always allocate at least 1 entry and a region_add for
532  * 1 page will only require at most 1 entry.
533  */
534 static long region_add(struct resv_map *resv, long f, long t,
535 		       long in_regions_needed, struct hstate *h,
536 		       struct hugetlb_cgroup *h_cg)
537 {
538 	long add = 0, actual_regions_needed = 0;
539 
540 	spin_lock(&resv->lock);
541 retry:
542 
543 	/* Count how many regions are actually needed to execute this add. */
544 	add_reservation_in_range(resv, f, t, NULL, NULL,
545 				 &actual_regions_needed);
546 
547 	/*
548 	 * Check for sufficient descriptors in the cache to accommodate
549 	 * this add operation. Note that actual_regions_needed may be greater
550 	 * than in_regions_needed, as the resv_map may have been modified since
551 	 * the region_chg call. In this case, we need to make sure that we
552 	 * allocate extra entries, such that we have enough for all the
553 	 * existing adds_in_progress, plus the excess needed for this
554 	 * operation.
555 	 */
556 	if (actual_regions_needed > in_regions_needed &&
557 	    resv->region_cache_count <
558 		    resv->adds_in_progress +
559 			    (actual_regions_needed - in_regions_needed)) {
560 		/* region_add operation of range 1 should never need to
561 		 * allocate file_region entries.
562 		 */
563 		VM_BUG_ON(t - f <= 1);
564 
565 		if (allocate_file_region_entries(
566 			    resv, actual_regions_needed - in_regions_needed)) {
567 			return -ENOMEM;
568 		}
569 
570 		goto retry;
571 	}
572 
573 	add = add_reservation_in_range(resv, f, t, h_cg, h, NULL);
574 
575 	resv->adds_in_progress -= in_regions_needed;
576 
577 	spin_unlock(&resv->lock);
578 	return add;
579 }
580 
581 /*
582  * Examine the existing reserve map and determine how many
583  * huge pages in the specified range [f, t) are NOT currently
584  * represented.  This routine is called before a subsequent
585  * call to region_add that will actually modify the reserve
586  * map to add the specified range [f, t).  region_chg does
587  * not change the number of huge pages represented by the
588  * map.  A number of new file_region structures is added to the cache as a
589  * placeholder, for the subsequent region_add call to use. At least 1
590  * file_region structure is added.
591  *
592  * out_regions_needed is the number of regions added to the
593  * resv->adds_in_progress.  This value needs to be provided to a follow up call
594  * to region_add or region_abort for proper accounting.
595  *
596  * Returns the number of huge pages that need to be added to the existing
597  * reservation map for the range [f, t).  This number is greater or equal to
598  * zero.  -ENOMEM is returned if a new file_region structure or cache entry
599  * is needed and can not be allocated.
600  */
601 static long region_chg(struct resv_map *resv, long f, long t,
602 		       long *out_regions_needed)
603 {
604 	long chg = 0;
605 
606 	spin_lock(&resv->lock);
607 
608 	/* Count how many hugepages in this range are NOT represented. */
609 	chg = add_reservation_in_range(resv, f, t, NULL, NULL,
610 				       out_regions_needed);
611 
612 	if (*out_regions_needed == 0)
613 		*out_regions_needed = 1;
614 
615 	if (allocate_file_region_entries(resv, *out_regions_needed))
616 		return -ENOMEM;
617 
618 	resv->adds_in_progress += *out_regions_needed;
619 
620 	spin_unlock(&resv->lock);
621 	return chg;
622 }
623 
624 /*
625  * Abort the in progress add operation.  The adds_in_progress field
626  * of the resv_map keeps track of the operations in progress between
627  * calls to region_chg and region_add.  Operations are sometimes
628  * aborted after the call to region_chg.  In such cases, region_abort
629  * is called to decrement the adds_in_progress counter. regions_needed
630  * is the value returned by the region_chg call, it is used to decrement
631  * the adds_in_progress counter.
632  *
633  * NOTE: The range arguments [f, t) are not needed or used in this
634  * routine.  They are kept to make reading the calling code easier as
635  * arguments will match the associated region_chg call.
636  */
637 static void region_abort(struct resv_map *resv, long f, long t,
638 			 long regions_needed)
639 {
640 	spin_lock(&resv->lock);
641 	VM_BUG_ON(!resv->region_cache_count);
642 	resv->adds_in_progress -= regions_needed;
643 	spin_unlock(&resv->lock);
644 }
645 
646 /*
647  * Delete the specified range [f, t) from the reserve map.  If the
648  * t parameter is LONG_MAX, this indicates that ALL regions after f
649  * should be deleted.  Locate the regions which intersect [f, t)
650  * and either trim, delete or split the existing regions.
651  *
652  * Returns the number of huge pages deleted from the reserve map.
653  * In the normal case, the return value is zero or more.  In the
654  * case where a region must be split, a new region descriptor must
655  * be allocated.  If the allocation fails, -ENOMEM will be returned.
656  * NOTE: If the parameter t == LONG_MAX, then we will never split
657  * a region and possibly return -ENOMEM.  Callers specifying
658  * t == LONG_MAX do not need to check for -ENOMEM error.
659  */
660 static long region_del(struct resv_map *resv, long f, long t)
661 {
662 	struct list_head *head = &resv->regions;
663 	struct file_region *rg, *trg;
664 	struct file_region *nrg = NULL;
665 	long del = 0;
666 
667 retry:
668 	spin_lock(&resv->lock);
669 	list_for_each_entry_safe(rg, trg, head, link) {
670 		/*
671 		 * Skip regions before the range to be deleted.  file_region
672 		 * ranges are normally of the form [from, to).  However, there
673 		 * may be a "placeholder" entry in the map which is of the form
674 		 * (from, to) with from == to.  Check for placeholder entries
675 		 * at the beginning of the range to be deleted.
676 		 */
677 		if (rg->to <= f && (rg->to != rg->from || rg->to != f))
678 			continue;
679 
680 		if (rg->from >= t)
681 			break;
682 
683 		if (f > rg->from && t < rg->to) { /* Must split region */
684 			/*
685 			 * Check for an entry in the cache before dropping
686 			 * lock and attempting allocation.
687 			 */
688 			if (!nrg &&
689 			    resv->region_cache_count > resv->adds_in_progress) {
690 				nrg = list_first_entry(&resv->region_cache,
691 							struct file_region,
692 							link);
693 				list_del(&nrg->link);
694 				resv->region_cache_count--;
695 			}
696 
697 			if (!nrg) {
698 				spin_unlock(&resv->lock);
699 				nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
700 				if (!nrg)
701 					return -ENOMEM;
702 				goto retry;
703 			}
704 
705 			del += t - f;
706 			hugetlb_cgroup_uncharge_file_region(
707 				resv, rg, t - f, false);
708 
709 			/* New entry for end of split region */
710 			nrg->from = t;
711 			nrg->to = rg->to;
712 
713 			copy_hugetlb_cgroup_uncharge_info(nrg, rg);
714 
715 			INIT_LIST_HEAD(&nrg->link);
716 
717 			/* Original entry is trimmed */
718 			rg->to = f;
719 
720 			list_add(&nrg->link, &rg->link);
721 			nrg = NULL;
722 			break;
723 		}
724 
725 		if (f <= rg->from && t >= rg->to) { /* Remove entire region */
726 			del += rg->to - rg->from;
727 			hugetlb_cgroup_uncharge_file_region(resv, rg,
728 							    rg->to - rg->from, true);
729 			list_del(&rg->link);
730 			kfree(rg);
731 			continue;
732 		}
733 
734 		if (f <= rg->from) {	/* Trim beginning of region */
735 			hugetlb_cgroup_uncharge_file_region(resv, rg,
736 							    t - rg->from, false);
737 
738 			del += t - rg->from;
739 			rg->from = t;
740 		} else {		/* Trim end of region */
741 			hugetlb_cgroup_uncharge_file_region(resv, rg,
742 							    rg->to - f, false);
743 
744 			del += rg->to - f;
745 			rg->to = f;
746 		}
747 	}
748 
749 	spin_unlock(&resv->lock);
750 	kfree(nrg);
751 	return del;
752 }
753 
754 /*
755  * A rare out of memory error was encountered which prevented removal of
756  * the reserve map region for a page.  The huge page itself was free'ed
757  * and removed from the page cache.  This routine will adjust the subpool
758  * usage count, and the global reserve count if needed.  By incrementing
759  * these counts, the reserve map entry which could not be deleted will
760  * appear as a "reserved" entry instead of simply dangling with incorrect
761  * counts.
762  */
763 void hugetlb_fix_reserve_counts(struct inode *inode)
764 {
765 	struct hugepage_subpool *spool = subpool_inode(inode);
766 	long rsv_adjust;
767 	bool reserved = false;
768 
769 	rsv_adjust = hugepage_subpool_get_pages(spool, 1);
770 	if (rsv_adjust > 0) {
771 		struct hstate *h = hstate_inode(inode);
772 
773 		if (!hugetlb_acct_memory(h, 1))
774 			reserved = true;
775 	} else if (!rsv_adjust) {
776 		reserved = true;
777 	}
778 
779 	if (!reserved)
780 		pr_warn("hugetlb: Huge Page Reserved count may go negative.\n");
781 }
782 
783 /*
784  * Count and return the number of huge pages in the reserve map
785  * that intersect with the range [f, t).
786  */
787 static long region_count(struct resv_map *resv, long f, long t)
788 {
789 	struct list_head *head = &resv->regions;
790 	struct file_region *rg;
791 	long chg = 0;
792 
793 	spin_lock(&resv->lock);
794 	/* Locate each segment we overlap with, and count that overlap. */
795 	list_for_each_entry(rg, head, link) {
796 		long seg_from;
797 		long seg_to;
798 
799 		if (rg->to <= f)
800 			continue;
801 		if (rg->from >= t)
802 			break;
803 
804 		seg_from = max(rg->from, f);
805 		seg_to = min(rg->to, t);
806 
807 		chg += seg_to - seg_from;
808 	}
809 	spin_unlock(&resv->lock);
810 
811 	return chg;
812 }
813 
814 /*
815  * Convert the address within this vma to the page offset within
816  * the mapping, in pagecache page units; huge pages here.
817  */
818 static pgoff_t vma_hugecache_offset(struct hstate *h,
819 			struct vm_area_struct *vma, unsigned long address)
820 {
821 	return ((address - vma->vm_start) >> huge_page_shift(h)) +
822 			(vma->vm_pgoff >> huge_page_order(h));
823 }
824 
825 pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
826 				     unsigned long address)
827 {
828 	return vma_hugecache_offset(hstate_vma(vma), vma, address);
829 }
830 EXPORT_SYMBOL_GPL(linear_hugepage_index);
831 
832 /*
833  * Return the size of the pages allocated when backing a VMA. In the majority
834  * cases this will be same size as used by the page table entries.
835  */
836 unsigned long vma_kernel_pagesize(struct vm_area_struct *vma)
837 {
838 	if (vma->vm_ops && vma->vm_ops->pagesize)
839 		return vma->vm_ops->pagesize(vma);
840 	return PAGE_SIZE;
841 }
842 EXPORT_SYMBOL_GPL(vma_kernel_pagesize);
843 
844 /*
845  * Return the page size being used by the MMU to back a VMA. In the majority
846  * of cases, the page size used by the kernel matches the MMU size. On
847  * architectures where it differs, an architecture-specific 'strong'
848  * version of this symbol is required.
849  */
850 __weak unsigned long vma_mmu_pagesize(struct vm_area_struct *vma)
851 {
852 	return vma_kernel_pagesize(vma);
853 }
854 
855 /*
856  * Flags for MAP_PRIVATE reservations.  These are stored in the bottom
857  * bits of the reservation map pointer, which are always clear due to
858  * alignment.
859  */
860 #define HPAGE_RESV_OWNER    (1UL << 0)
861 #define HPAGE_RESV_UNMAPPED (1UL << 1)
862 #define HPAGE_RESV_MASK (HPAGE_RESV_OWNER | HPAGE_RESV_UNMAPPED)
863 
864 /*
865  * These helpers are used to track how many pages are reserved for
866  * faults in a MAP_PRIVATE mapping. Only the process that called mmap()
867  * is guaranteed to have their future faults succeed.
868  *
869  * With the exception of reset_vma_resv_huge_pages() which is called at fork(),
870  * the reserve counters are updated with the hugetlb_lock held. It is safe
871  * to reset the VMA at fork() time as it is not in use yet and there is no
872  * chance of the global counters getting corrupted as a result of the values.
873  *
874  * The private mapping reservation is represented in a subtly different
875  * manner to a shared mapping.  A shared mapping has a region map associated
876  * with the underlying file, this region map represents the backing file
877  * pages which have ever had a reservation assigned which this persists even
878  * after the page is instantiated.  A private mapping has a region map
879  * associated with the original mmap which is attached to all VMAs which
880  * reference it, this region map represents those offsets which have consumed
881  * reservation ie. where pages have been instantiated.
882  */
883 static unsigned long get_vma_private_data(struct vm_area_struct *vma)
884 {
885 	return (unsigned long)vma->vm_private_data;
886 }
887 
888 static void set_vma_private_data(struct vm_area_struct *vma,
889 							unsigned long value)
890 {
891 	vma->vm_private_data = (void *)value;
892 }
893 
894 static void
895 resv_map_set_hugetlb_cgroup_uncharge_info(struct resv_map *resv_map,
896 					  struct hugetlb_cgroup *h_cg,
897 					  struct hstate *h)
898 {
899 #ifdef CONFIG_CGROUP_HUGETLB
900 	if (!h_cg || !h) {
901 		resv_map->reservation_counter = NULL;
902 		resv_map->pages_per_hpage = 0;
903 		resv_map->css = NULL;
904 	} else {
905 		resv_map->reservation_counter =
906 			&h_cg->rsvd_hugepage[hstate_index(h)];
907 		resv_map->pages_per_hpage = pages_per_huge_page(h);
908 		resv_map->css = &h_cg->css;
909 	}
910 #endif
911 }
912 
913 struct resv_map *resv_map_alloc(void)
914 {
915 	struct resv_map *resv_map = kmalloc(sizeof(*resv_map), GFP_KERNEL);
916 	struct file_region *rg = kmalloc(sizeof(*rg), GFP_KERNEL);
917 
918 	if (!resv_map || !rg) {
919 		kfree(resv_map);
920 		kfree(rg);
921 		return NULL;
922 	}
923 
924 	kref_init(&resv_map->refs);
925 	spin_lock_init(&resv_map->lock);
926 	INIT_LIST_HEAD(&resv_map->regions);
927 
928 	resv_map->adds_in_progress = 0;
929 	/*
930 	 * Initialize these to 0. On shared mappings, 0's here indicate these
931 	 * fields don't do cgroup accounting. On private mappings, these will be
932 	 * re-initialized to the proper values, to indicate that hugetlb cgroup
933 	 * reservations are to be un-charged from here.
934 	 */
935 	resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, NULL, NULL);
936 
937 	INIT_LIST_HEAD(&resv_map->region_cache);
938 	list_add(&rg->link, &resv_map->region_cache);
939 	resv_map->region_cache_count = 1;
940 
941 	return resv_map;
942 }
943 
944 void resv_map_release(struct kref *ref)
945 {
946 	struct resv_map *resv_map = container_of(ref, struct resv_map, refs);
947 	struct list_head *head = &resv_map->region_cache;
948 	struct file_region *rg, *trg;
949 
950 	/* Clear out any active regions before we release the map. */
951 	region_del(resv_map, 0, LONG_MAX);
952 
953 	/* ... and any entries left in the cache */
954 	list_for_each_entry_safe(rg, trg, head, link) {
955 		list_del(&rg->link);
956 		kfree(rg);
957 	}
958 
959 	VM_BUG_ON(resv_map->adds_in_progress);
960 
961 	kfree(resv_map);
962 }
963 
964 static inline struct resv_map *inode_resv_map(struct inode *inode)
965 {
966 	/*
967 	 * At inode evict time, i_mapping may not point to the original
968 	 * address space within the inode.  This original address space
969 	 * contains the pointer to the resv_map.  So, always use the
970 	 * address space embedded within the inode.
971 	 * The VERY common case is inode->mapping == &inode->i_data but,
972 	 * this may not be true for device special inodes.
973 	 */
974 	return (struct resv_map *)(&inode->i_data)->private_data;
975 }
976 
977 static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
978 {
979 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
980 	if (vma->vm_flags & VM_MAYSHARE) {
981 		struct address_space *mapping = vma->vm_file->f_mapping;
982 		struct inode *inode = mapping->host;
983 
984 		return inode_resv_map(inode);
985 
986 	} else {
987 		return (struct resv_map *)(get_vma_private_data(vma) &
988 							~HPAGE_RESV_MASK);
989 	}
990 }
991 
992 static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
993 {
994 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
995 	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
996 
997 	set_vma_private_data(vma, (get_vma_private_data(vma) &
998 				HPAGE_RESV_MASK) | (unsigned long)map);
999 }
1000 
1001 static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
1002 {
1003 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1004 	VM_BUG_ON_VMA(vma->vm_flags & VM_MAYSHARE, vma);
1005 
1006 	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
1007 }
1008 
1009 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
1010 {
1011 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1012 
1013 	return (get_vma_private_data(vma) & flag) != 0;
1014 }
1015 
1016 /* Reset counters to 0 and clear all HPAGE_RESV_* flags */
1017 void reset_vma_resv_huge_pages(struct vm_area_struct *vma)
1018 {
1019 	VM_BUG_ON_VMA(!is_vm_hugetlb_page(vma), vma);
1020 	if (!(vma->vm_flags & VM_MAYSHARE))
1021 		vma->vm_private_data = (void *)0;
1022 }
1023 
1024 /*
1025  * Reset and decrement one ref on hugepage private reservation.
1026  * Called with mm->mmap_sem writer semaphore held.
1027  * This function should be only used by move_vma() and operate on
1028  * same sized vma. It should never come here with last ref on the
1029  * reservation.
1030  */
1031 void clear_vma_resv_huge_pages(struct vm_area_struct *vma)
1032 {
1033 	/*
1034 	 * Clear the old hugetlb private page reservation.
1035 	 * It has already been transferred to new_vma.
1036 	 *
1037 	 * During a mremap() operation of a hugetlb vma we call move_vma()
1038 	 * which copies vma into new_vma and unmaps vma. After the copy
1039 	 * operation both new_vma and vma share a reference to the resv_map
1040 	 * struct, and at that point vma is about to be unmapped. We don't
1041 	 * want to return the reservation to the pool at unmap of vma because
1042 	 * the reservation still lives on in new_vma, so simply decrement the
1043 	 * ref here and remove the resv_map reference from this vma.
1044 	 */
1045 	struct resv_map *reservations = vma_resv_map(vma);
1046 
1047 	if (reservations && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1048 		resv_map_put_hugetlb_cgroup_uncharge_info(reservations);
1049 		kref_put(&reservations->refs, resv_map_release);
1050 	}
1051 
1052 	reset_vma_resv_huge_pages(vma);
1053 }
1054 
1055 /* Returns true if the VMA has associated reserve pages */
1056 static bool vma_has_reserves(struct vm_area_struct *vma, long chg)
1057 {
1058 	if (vma->vm_flags & VM_NORESERVE) {
1059 		/*
1060 		 * This address is already reserved by other process(chg == 0),
1061 		 * so, we should decrement reserved count. Without decrementing,
1062 		 * reserve count remains after releasing inode, because this
1063 		 * allocated page will go into page cache and is regarded as
1064 		 * coming from reserved pool in releasing step.  Currently, we
1065 		 * don't have any other solution to deal with this situation
1066 		 * properly, so add work-around here.
1067 		 */
1068 		if (vma->vm_flags & VM_MAYSHARE && chg == 0)
1069 			return true;
1070 		else
1071 			return false;
1072 	}
1073 
1074 	/* Shared mappings always use reserves */
1075 	if (vma->vm_flags & VM_MAYSHARE) {
1076 		/*
1077 		 * We know VM_NORESERVE is not set.  Therefore, there SHOULD
1078 		 * be a region map for all pages.  The only situation where
1079 		 * there is no region map is if a hole was punched via
1080 		 * fallocate.  In this case, there really are no reserves to
1081 		 * use.  This situation is indicated if chg != 0.
1082 		 */
1083 		if (chg)
1084 			return false;
1085 		else
1086 			return true;
1087 	}
1088 
1089 	/*
1090 	 * Only the process that called mmap() has reserves for
1091 	 * private mappings.
1092 	 */
1093 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
1094 		/*
1095 		 * Like the shared case above, a hole punch or truncate
1096 		 * could have been performed on the private mapping.
1097 		 * Examine the value of chg to determine if reserves
1098 		 * actually exist or were previously consumed.
1099 		 * Very Subtle - The value of chg comes from a previous
1100 		 * call to vma_needs_reserves().  The reserve map for
1101 		 * private mappings has different (opposite) semantics
1102 		 * than that of shared mappings.  vma_needs_reserves()
1103 		 * has already taken this difference in semantics into
1104 		 * account.  Therefore, the meaning of chg is the same
1105 		 * as in the shared case above.  Code could easily be
1106 		 * combined, but keeping it separate draws attention to
1107 		 * subtle differences.
1108 		 */
1109 		if (chg)
1110 			return false;
1111 		else
1112 			return true;
1113 	}
1114 
1115 	return false;
1116 }
1117 
1118 static void enqueue_huge_page(struct hstate *h, struct page *page)
1119 {
1120 	int nid = page_to_nid(page);
1121 
1122 	lockdep_assert_held(&hugetlb_lock);
1123 	VM_BUG_ON_PAGE(page_count(page), page);
1124 
1125 	list_move(&page->lru, &h->hugepage_freelists[nid]);
1126 	h->free_huge_pages++;
1127 	h->free_huge_pages_node[nid]++;
1128 	SetHPageFreed(page);
1129 }
1130 
1131 static struct page *dequeue_huge_page_node_exact(struct hstate *h, int nid)
1132 {
1133 	struct page *page;
1134 	bool pin = !!(current->flags & PF_MEMALLOC_PIN);
1135 
1136 	lockdep_assert_held(&hugetlb_lock);
1137 	list_for_each_entry(page, &h->hugepage_freelists[nid], lru) {
1138 		if (pin && !is_pinnable_page(page))
1139 			continue;
1140 
1141 		if (PageHWPoison(page))
1142 			continue;
1143 
1144 		list_move(&page->lru, &h->hugepage_activelist);
1145 		set_page_refcounted(page);
1146 		ClearHPageFreed(page);
1147 		h->free_huge_pages--;
1148 		h->free_huge_pages_node[nid]--;
1149 		return page;
1150 	}
1151 
1152 	return NULL;
1153 }
1154 
1155 static struct page *dequeue_huge_page_nodemask(struct hstate *h, gfp_t gfp_mask, int nid,
1156 		nodemask_t *nmask)
1157 {
1158 	unsigned int cpuset_mems_cookie;
1159 	struct zonelist *zonelist;
1160 	struct zone *zone;
1161 	struct zoneref *z;
1162 	int node = NUMA_NO_NODE;
1163 
1164 	zonelist = node_zonelist(nid, gfp_mask);
1165 
1166 retry_cpuset:
1167 	cpuset_mems_cookie = read_mems_allowed_begin();
1168 	for_each_zone_zonelist_nodemask(zone, z, zonelist, gfp_zone(gfp_mask), nmask) {
1169 		struct page *page;
1170 
1171 		if (!cpuset_zone_allowed(zone, gfp_mask))
1172 			continue;
1173 		/*
1174 		 * no need to ask again on the same node. Pool is node rather than
1175 		 * zone aware
1176 		 */
1177 		if (zone_to_nid(zone) == node)
1178 			continue;
1179 		node = zone_to_nid(zone);
1180 
1181 		page = dequeue_huge_page_node_exact(h, node);
1182 		if (page)
1183 			return page;
1184 	}
1185 	if (unlikely(read_mems_allowed_retry(cpuset_mems_cookie)))
1186 		goto retry_cpuset;
1187 
1188 	return NULL;
1189 }
1190 
1191 static struct page *dequeue_huge_page_vma(struct hstate *h,
1192 				struct vm_area_struct *vma,
1193 				unsigned long address, int avoid_reserve,
1194 				long chg)
1195 {
1196 	struct page *page = NULL;
1197 	struct mempolicy *mpol;
1198 	gfp_t gfp_mask;
1199 	nodemask_t *nodemask;
1200 	int nid;
1201 
1202 	/*
1203 	 * A child process with MAP_PRIVATE mappings created by their parent
1204 	 * have no page reserves. This check ensures that reservations are
1205 	 * not "stolen". The child may still get SIGKILLed
1206 	 */
1207 	if (!vma_has_reserves(vma, chg) &&
1208 			h->free_huge_pages - h->resv_huge_pages == 0)
1209 		goto err;
1210 
1211 	/* If reserves cannot be used, ensure enough pages are in the pool */
1212 	if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0)
1213 		goto err;
1214 
1215 	gfp_mask = htlb_alloc_mask(h);
1216 	nid = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
1217 
1218 	if (mpol_is_preferred_many(mpol)) {
1219 		page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
1220 
1221 		/* Fallback to all nodes if page==NULL */
1222 		nodemask = NULL;
1223 	}
1224 
1225 	if (!page)
1226 		page = dequeue_huge_page_nodemask(h, gfp_mask, nid, nodemask);
1227 
1228 	if (page && !avoid_reserve && vma_has_reserves(vma, chg)) {
1229 		SetHPageRestoreReserve(page);
1230 		h->resv_huge_pages--;
1231 	}
1232 
1233 	mpol_cond_put(mpol);
1234 	return page;
1235 
1236 err:
1237 	return NULL;
1238 }
1239 
1240 /*
1241  * common helper functions for hstate_next_node_to_{alloc|free}.
1242  * We may have allocated or freed a huge page based on a different
1243  * nodes_allowed previously, so h->next_node_to_{alloc|free} might
1244  * be outside of *nodes_allowed.  Ensure that we use an allowed
1245  * node for alloc or free.
1246  */
1247 static int next_node_allowed(int nid, nodemask_t *nodes_allowed)
1248 {
1249 	nid = next_node_in(nid, *nodes_allowed);
1250 	VM_BUG_ON(nid >= MAX_NUMNODES);
1251 
1252 	return nid;
1253 }
1254 
1255 static int get_valid_node_allowed(int nid, nodemask_t *nodes_allowed)
1256 {
1257 	if (!node_isset(nid, *nodes_allowed))
1258 		nid = next_node_allowed(nid, nodes_allowed);
1259 	return nid;
1260 }
1261 
1262 /*
1263  * returns the previously saved node ["this node"] from which to
1264  * allocate a persistent huge page for the pool and advance the
1265  * next node from which to allocate, handling wrap at end of node
1266  * mask.
1267  */
1268 static int hstate_next_node_to_alloc(struct hstate *h,
1269 					nodemask_t *nodes_allowed)
1270 {
1271 	int nid;
1272 
1273 	VM_BUG_ON(!nodes_allowed);
1274 
1275 	nid = get_valid_node_allowed(h->next_nid_to_alloc, nodes_allowed);
1276 	h->next_nid_to_alloc = next_node_allowed(nid, nodes_allowed);
1277 
1278 	return nid;
1279 }
1280 
1281 /*
1282  * helper for remove_pool_huge_page() - return the previously saved
1283  * node ["this node"] from which to free a huge page.  Advance the
1284  * next node id whether or not we find a free huge page to free so
1285  * that the next attempt to free addresses the next node.
1286  */
1287 static int hstate_next_node_to_free(struct hstate *h, nodemask_t *nodes_allowed)
1288 {
1289 	int nid;
1290 
1291 	VM_BUG_ON(!nodes_allowed);
1292 
1293 	nid = get_valid_node_allowed(h->next_nid_to_free, nodes_allowed);
1294 	h->next_nid_to_free = next_node_allowed(nid, nodes_allowed);
1295 
1296 	return nid;
1297 }
1298 
1299 #define for_each_node_mask_to_alloc(hs, nr_nodes, node, mask)		\
1300 	for (nr_nodes = nodes_weight(*mask);				\
1301 		nr_nodes > 0 &&						\
1302 		((node = hstate_next_node_to_alloc(hs, mask)) || 1);	\
1303 		nr_nodes--)
1304 
1305 #define for_each_node_mask_to_free(hs, nr_nodes, node, mask)		\
1306 	for (nr_nodes = nodes_weight(*mask);				\
1307 		nr_nodes > 0 &&						\
1308 		((node = hstate_next_node_to_free(hs, mask)) || 1);	\
1309 		nr_nodes--)
1310 
1311 /* used to demote non-gigantic_huge pages as well */
1312 static void __destroy_compound_gigantic_page(struct page *page,
1313 					unsigned int order, bool demote)
1314 {
1315 	int i;
1316 	int nr_pages = 1 << order;
1317 	struct page *p = page + 1;
1318 
1319 	atomic_set(compound_mapcount_ptr(page), 0);
1320 	atomic_set(compound_pincount_ptr(page), 0);
1321 
1322 	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1323 		p->mapping = NULL;
1324 		clear_compound_head(p);
1325 		if (!demote)
1326 			set_page_refcounted(p);
1327 	}
1328 
1329 	set_compound_order(page, 0);
1330 #ifdef CONFIG_64BIT
1331 	page[1].compound_nr = 0;
1332 #endif
1333 	__ClearPageHead(page);
1334 }
1335 
1336 static void destroy_compound_hugetlb_page_for_demote(struct page *page,
1337 					unsigned int order)
1338 {
1339 	__destroy_compound_gigantic_page(page, order, true);
1340 }
1341 
1342 #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE
1343 static void destroy_compound_gigantic_page(struct page *page,
1344 					unsigned int order)
1345 {
1346 	__destroy_compound_gigantic_page(page, order, false);
1347 }
1348 
1349 static void free_gigantic_page(struct page *page, unsigned int order)
1350 {
1351 	/*
1352 	 * If the page isn't allocated using the cma allocator,
1353 	 * cma_release() returns false.
1354 	 */
1355 #ifdef CONFIG_CMA
1356 	if (cma_release(hugetlb_cma[page_to_nid(page)], page, 1 << order))
1357 		return;
1358 #endif
1359 
1360 	free_contig_range(page_to_pfn(page), 1 << order);
1361 }
1362 
1363 #ifdef CONFIG_CONTIG_ALLOC
1364 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1365 		int nid, nodemask_t *nodemask)
1366 {
1367 	unsigned long nr_pages = pages_per_huge_page(h);
1368 	if (nid == NUMA_NO_NODE)
1369 		nid = numa_mem_id();
1370 
1371 #ifdef CONFIG_CMA
1372 	{
1373 		struct page *page;
1374 		int node;
1375 
1376 		if (hugetlb_cma[nid]) {
1377 			page = cma_alloc(hugetlb_cma[nid], nr_pages,
1378 					huge_page_order(h), true);
1379 			if (page)
1380 				return page;
1381 		}
1382 
1383 		if (!(gfp_mask & __GFP_THISNODE)) {
1384 			for_each_node_mask(node, *nodemask) {
1385 				if (node == nid || !hugetlb_cma[node])
1386 					continue;
1387 
1388 				page = cma_alloc(hugetlb_cma[node], nr_pages,
1389 						huge_page_order(h), true);
1390 				if (page)
1391 					return page;
1392 			}
1393 		}
1394 	}
1395 #endif
1396 
1397 	return alloc_contig_pages(nr_pages, gfp_mask, nid, nodemask);
1398 }
1399 
1400 #else /* !CONFIG_CONTIG_ALLOC */
1401 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1402 					int nid, nodemask_t *nodemask)
1403 {
1404 	return NULL;
1405 }
1406 #endif /* CONFIG_CONTIG_ALLOC */
1407 
1408 #else /* !CONFIG_ARCH_HAS_GIGANTIC_PAGE */
1409 static struct page *alloc_gigantic_page(struct hstate *h, gfp_t gfp_mask,
1410 					int nid, nodemask_t *nodemask)
1411 {
1412 	return NULL;
1413 }
1414 static inline void free_gigantic_page(struct page *page, unsigned int order) { }
1415 static inline void destroy_compound_gigantic_page(struct page *page,
1416 						unsigned int order) { }
1417 #endif
1418 
1419 /*
1420  * Remove hugetlb page from lists, and update dtor so that page appears
1421  * as just a compound page.
1422  *
1423  * A reference is held on the page, except in the case of demote.
1424  *
1425  * Must be called with hugetlb lock held.
1426  */
1427 static void __remove_hugetlb_page(struct hstate *h, struct page *page,
1428 							bool adjust_surplus,
1429 							bool demote)
1430 {
1431 	int nid = page_to_nid(page);
1432 
1433 	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page(page), page);
1434 	VM_BUG_ON_PAGE(hugetlb_cgroup_from_page_rsvd(page), page);
1435 
1436 	lockdep_assert_held(&hugetlb_lock);
1437 	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1438 		return;
1439 
1440 	list_del(&page->lru);
1441 
1442 	if (HPageFreed(page)) {
1443 		h->free_huge_pages--;
1444 		h->free_huge_pages_node[nid]--;
1445 	}
1446 	if (adjust_surplus) {
1447 		h->surplus_huge_pages--;
1448 		h->surplus_huge_pages_node[nid]--;
1449 	}
1450 
1451 	/*
1452 	 * Very subtle
1453 	 *
1454 	 * For non-gigantic pages set the destructor to the normal compound
1455 	 * page dtor.  This is needed in case someone takes an additional
1456 	 * temporary ref to the page, and freeing is delayed until they drop
1457 	 * their reference.
1458 	 *
1459 	 * For gigantic pages set the destructor to the null dtor.  This
1460 	 * destructor will never be called.  Before freeing the gigantic
1461 	 * page destroy_compound_gigantic_page will turn the compound page
1462 	 * into a simple group of pages.  After this the destructor does not
1463 	 * apply.
1464 	 *
1465 	 * This handles the case where more than one ref is held when and
1466 	 * after update_and_free_page is called.
1467 	 *
1468 	 * In the case of demote we do not ref count the page as it will soon
1469 	 * be turned into a page of smaller size.
1470 	 */
1471 	if (!demote)
1472 		set_page_refcounted(page);
1473 	if (hstate_is_gigantic(h))
1474 		set_compound_page_dtor(page, NULL_COMPOUND_DTOR);
1475 	else
1476 		set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
1477 
1478 	h->nr_huge_pages--;
1479 	h->nr_huge_pages_node[nid]--;
1480 }
1481 
1482 static void remove_hugetlb_page(struct hstate *h, struct page *page,
1483 							bool adjust_surplus)
1484 {
1485 	__remove_hugetlb_page(h, page, adjust_surplus, false);
1486 }
1487 
1488 static void remove_hugetlb_page_for_demote(struct hstate *h, struct page *page,
1489 							bool adjust_surplus)
1490 {
1491 	__remove_hugetlb_page(h, page, adjust_surplus, true);
1492 }
1493 
1494 static void add_hugetlb_page(struct hstate *h, struct page *page,
1495 			     bool adjust_surplus)
1496 {
1497 	int zeroed;
1498 	int nid = page_to_nid(page);
1499 
1500 	VM_BUG_ON_PAGE(!HPageVmemmapOptimized(page), page);
1501 
1502 	lockdep_assert_held(&hugetlb_lock);
1503 
1504 	INIT_LIST_HEAD(&page->lru);
1505 	h->nr_huge_pages++;
1506 	h->nr_huge_pages_node[nid]++;
1507 
1508 	if (adjust_surplus) {
1509 		h->surplus_huge_pages++;
1510 		h->surplus_huge_pages_node[nid]++;
1511 	}
1512 
1513 	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1514 	set_page_private(page, 0);
1515 	SetHPageVmemmapOptimized(page);
1516 
1517 	/*
1518 	 * This page is about to be managed by the hugetlb allocator and
1519 	 * should have no users.  Drop our reference, and check for others
1520 	 * just in case.
1521 	 */
1522 	zeroed = put_page_testzero(page);
1523 	if (!zeroed)
1524 		/*
1525 		 * It is VERY unlikely soneone else has taken a ref on
1526 		 * the page.  In this case, we simply return as the
1527 		 * hugetlb destructor (free_huge_page) will be called
1528 		 * when this other ref is dropped.
1529 		 */
1530 		return;
1531 
1532 	arch_clear_hugepage_flags(page);
1533 	enqueue_huge_page(h, page);
1534 }
1535 
1536 static void __update_and_free_page(struct hstate *h, struct page *page)
1537 {
1538 	int i;
1539 	struct page *subpage = page;
1540 
1541 	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
1542 		return;
1543 
1544 	if (hugetlb_vmemmap_alloc(h, page)) {
1545 		spin_lock_irq(&hugetlb_lock);
1546 		/*
1547 		 * If we cannot allocate vmemmap pages, just refuse to free the
1548 		 * page and put the page back on the hugetlb free list and treat
1549 		 * as a surplus page.
1550 		 */
1551 		add_hugetlb_page(h, page, true);
1552 		spin_unlock_irq(&hugetlb_lock);
1553 		return;
1554 	}
1555 
1556 	for (i = 0; i < pages_per_huge_page(h);
1557 	     i++, subpage = mem_map_next(subpage, page, i)) {
1558 		subpage->flags &= ~(1 << PG_locked | 1 << PG_error |
1559 				1 << PG_referenced | 1 << PG_dirty |
1560 				1 << PG_active | 1 << PG_private |
1561 				1 << PG_writeback);
1562 	}
1563 
1564 	/*
1565 	 * Non-gigantic pages demoted from CMA allocated gigantic pages
1566 	 * need to be given back to CMA in free_gigantic_page.
1567 	 */
1568 	if (hstate_is_gigantic(h) ||
1569 	    hugetlb_cma_page(page, huge_page_order(h))) {
1570 		destroy_compound_gigantic_page(page, huge_page_order(h));
1571 		free_gigantic_page(page, huge_page_order(h));
1572 	} else {
1573 		__free_pages(page, huge_page_order(h));
1574 	}
1575 }
1576 
1577 /*
1578  * As update_and_free_page() can be called under any context, so we cannot
1579  * use GFP_KERNEL to allocate vmemmap pages. However, we can defer the
1580  * actual freeing in a workqueue to prevent from using GFP_ATOMIC to allocate
1581  * the vmemmap pages.
1582  *
1583  * free_hpage_workfn() locklessly retrieves the linked list of pages to be
1584  * freed and frees them one-by-one. As the page->mapping pointer is going
1585  * to be cleared in free_hpage_workfn() anyway, it is reused as the llist_node
1586  * structure of a lockless linked list of huge pages to be freed.
1587  */
1588 static LLIST_HEAD(hpage_freelist);
1589 
1590 static void free_hpage_workfn(struct work_struct *work)
1591 {
1592 	struct llist_node *node;
1593 
1594 	node = llist_del_all(&hpage_freelist);
1595 
1596 	while (node) {
1597 		struct page *page;
1598 		struct hstate *h;
1599 
1600 		page = container_of((struct address_space **)node,
1601 				     struct page, mapping);
1602 		node = node->next;
1603 		page->mapping = NULL;
1604 		/*
1605 		 * The VM_BUG_ON_PAGE(!PageHuge(page), page) in page_hstate()
1606 		 * is going to trigger because a previous call to
1607 		 * remove_hugetlb_page() will set_compound_page_dtor(page,
1608 		 * NULL_COMPOUND_DTOR), so do not use page_hstate() directly.
1609 		 */
1610 		h = size_to_hstate(page_size(page));
1611 
1612 		__update_and_free_page(h, page);
1613 
1614 		cond_resched();
1615 	}
1616 }
1617 static DECLARE_WORK(free_hpage_work, free_hpage_workfn);
1618 
1619 static inline void flush_free_hpage_work(struct hstate *h)
1620 {
1621 	if (hugetlb_optimize_vmemmap_pages(h))
1622 		flush_work(&free_hpage_work);
1623 }
1624 
1625 static void update_and_free_page(struct hstate *h, struct page *page,
1626 				 bool atomic)
1627 {
1628 	if (!HPageVmemmapOptimized(page) || !atomic) {
1629 		__update_and_free_page(h, page);
1630 		return;
1631 	}
1632 
1633 	/*
1634 	 * Defer freeing to avoid using GFP_ATOMIC to allocate vmemmap pages.
1635 	 *
1636 	 * Only call schedule_work() if hpage_freelist is previously
1637 	 * empty. Otherwise, schedule_work() had been called but the workfn
1638 	 * hasn't retrieved the list yet.
1639 	 */
1640 	if (llist_add((struct llist_node *)&page->mapping, &hpage_freelist))
1641 		schedule_work(&free_hpage_work);
1642 }
1643 
1644 static void update_and_free_pages_bulk(struct hstate *h, struct list_head *list)
1645 {
1646 	struct page *page, *t_page;
1647 
1648 	list_for_each_entry_safe(page, t_page, list, lru) {
1649 		update_and_free_page(h, page, false);
1650 		cond_resched();
1651 	}
1652 }
1653 
1654 struct hstate *size_to_hstate(unsigned long size)
1655 {
1656 	struct hstate *h;
1657 
1658 	for_each_hstate(h) {
1659 		if (huge_page_size(h) == size)
1660 			return h;
1661 	}
1662 	return NULL;
1663 }
1664 
1665 void free_huge_page(struct page *page)
1666 {
1667 	/*
1668 	 * Can't pass hstate in here because it is called from the
1669 	 * compound page destructor.
1670 	 */
1671 	struct hstate *h = page_hstate(page);
1672 	int nid = page_to_nid(page);
1673 	struct hugepage_subpool *spool = hugetlb_page_subpool(page);
1674 	bool restore_reserve;
1675 	unsigned long flags;
1676 
1677 	VM_BUG_ON_PAGE(page_count(page), page);
1678 	VM_BUG_ON_PAGE(page_mapcount(page), page);
1679 
1680 	hugetlb_set_page_subpool(page, NULL);
1681 	if (PageAnon(page))
1682 		__ClearPageAnonExclusive(page);
1683 	page->mapping = NULL;
1684 	restore_reserve = HPageRestoreReserve(page);
1685 	ClearHPageRestoreReserve(page);
1686 
1687 	/*
1688 	 * If HPageRestoreReserve was set on page, page allocation consumed a
1689 	 * reservation.  If the page was associated with a subpool, there
1690 	 * would have been a page reserved in the subpool before allocation
1691 	 * via hugepage_subpool_get_pages().  Since we are 'restoring' the
1692 	 * reservation, do not call hugepage_subpool_put_pages() as this will
1693 	 * remove the reserved page from the subpool.
1694 	 */
1695 	if (!restore_reserve) {
1696 		/*
1697 		 * A return code of zero implies that the subpool will be
1698 		 * under its minimum size if the reservation is not restored
1699 		 * after page is free.  Therefore, force restore_reserve
1700 		 * operation.
1701 		 */
1702 		if (hugepage_subpool_put_pages(spool, 1) == 0)
1703 			restore_reserve = true;
1704 	}
1705 
1706 	spin_lock_irqsave(&hugetlb_lock, flags);
1707 	ClearHPageMigratable(page);
1708 	hugetlb_cgroup_uncharge_page(hstate_index(h),
1709 				     pages_per_huge_page(h), page);
1710 	hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h),
1711 					  pages_per_huge_page(h), page);
1712 	if (restore_reserve)
1713 		h->resv_huge_pages++;
1714 
1715 	if (HPageTemporary(page)) {
1716 		remove_hugetlb_page(h, page, false);
1717 		spin_unlock_irqrestore(&hugetlb_lock, flags);
1718 		update_and_free_page(h, page, true);
1719 	} else if (h->surplus_huge_pages_node[nid]) {
1720 		/* remove the page from active list */
1721 		remove_hugetlb_page(h, page, true);
1722 		spin_unlock_irqrestore(&hugetlb_lock, flags);
1723 		update_and_free_page(h, page, true);
1724 	} else {
1725 		arch_clear_hugepage_flags(page);
1726 		enqueue_huge_page(h, page);
1727 		spin_unlock_irqrestore(&hugetlb_lock, flags);
1728 	}
1729 }
1730 
1731 /*
1732  * Must be called with the hugetlb lock held
1733  */
1734 static void __prep_account_new_huge_page(struct hstate *h, int nid)
1735 {
1736 	lockdep_assert_held(&hugetlb_lock);
1737 	h->nr_huge_pages++;
1738 	h->nr_huge_pages_node[nid]++;
1739 }
1740 
1741 static void __prep_new_huge_page(struct hstate *h, struct page *page)
1742 {
1743 	hugetlb_vmemmap_free(h, page);
1744 	INIT_LIST_HEAD(&page->lru);
1745 	set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
1746 	hugetlb_set_page_subpool(page, NULL);
1747 	set_hugetlb_cgroup(page, NULL);
1748 	set_hugetlb_cgroup_rsvd(page, NULL);
1749 }
1750 
1751 static void prep_new_huge_page(struct hstate *h, struct page *page, int nid)
1752 {
1753 	__prep_new_huge_page(h, page);
1754 	spin_lock_irq(&hugetlb_lock);
1755 	__prep_account_new_huge_page(h, nid);
1756 	spin_unlock_irq(&hugetlb_lock);
1757 }
1758 
1759 static bool __prep_compound_gigantic_page(struct page *page, unsigned int order,
1760 								bool demote)
1761 {
1762 	int i, j;
1763 	int nr_pages = 1 << order;
1764 	struct page *p = page + 1;
1765 
1766 	/* we rely on prep_new_huge_page to set the destructor */
1767 	set_compound_order(page, order);
1768 	__ClearPageReserved(page);
1769 	__SetPageHead(page);
1770 	for (i = 1; i < nr_pages; i++, p = mem_map_next(p, page, i)) {
1771 		/*
1772 		 * For gigantic hugepages allocated through bootmem at
1773 		 * boot, it's safer to be consistent with the not-gigantic
1774 		 * hugepages and clear the PG_reserved bit from all tail pages
1775 		 * too.  Otherwise drivers using get_user_pages() to access tail
1776 		 * pages may get the reference counting wrong if they see
1777 		 * PG_reserved set on a tail page (despite the head page not
1778 		 * having PG_reserved set).  Enforcing this consistency between
1779 		 * head and tail pages allows drivers to optimize away a check
1780 		 * on the head page when they need know if put_page() is needed
1781 		 * after get_user_pages().
1782 		 */
1783 		__ClearPageReserved(p);
1784 		/*
1785 		 * Subtle and very unlikely
1786 		 *
1787 		 * Gigantic 'page allocators' such as memblock or cma will
1788 		 * return a set of pages with each page ref counted.  We need
1789 		 * to turn this set of pages into a compound page with tail
1790 		 * page ref counts set to zero.  Code such as speculative page
1791 		 * cache adding could take a ref on a 'to be' tail page.
1792 		 * We need to respect any increased ref count, and only set
1793 		 * the ref count to zero if count is currently 1.  If count
1794 		 * is not 1, we return an error.  An error return indicates
1795 		 * the set of pages can not be converted to a gigantic page.
1796 		 * The caller who allocated the pages should then discard the
1797 		 * pages using the appropriate free interface.
1798 		 *
1799 		 * In the case of demote, the ref count will be zero.
1800 		 */
1801 		if (!demote) {
1802 			if (!page_ref_freeze(p, 1)) {
1803 				pr_warn("HugeTLB page can not be used due to unexpected inflated ref count\n");
1804 				goto out_error;
1805 			}
1806 		} else {
1807 			VM_BUG_ON_PAGE(page_count(p), p);
1808 		}
1809 		set_compound_head(p, page);
1810 	}
1811 	atomic_set(compound_mapcount_ptr(page), -1);
1812 	atomic_set(compound_pincount_ptr(page), 0);
1813 	return true;
1814 
1815 out_error:
1816 	/* undo tail page modifications made above */
1817 	p = page + 1;
1818 	for (j = 1; j < i; j++, p = mem_map_next(p, page, j)) {
1819 		clear_compound_head(p);
1820 		set_page_refcounted(p);
1821 	}
1822 	/* need to clear PG_reserved on remaining tail pages  */
1823 	for (; j < nr_pages; j++, p = mem_map_next(p, page, j))
1824 		__ClearPageReserved(p);
1825 	set_compound_order(page, 0);
1826 #ifdef CONFIG_64BIT
1827 	page[1].compound_nr = 0;
1828 #endif
1829 	__ClearPageHead(page);
1830 	return false;
1831 }
1832 
1833 static bool prep_compound_gigantic_page(struct page *page, unsigned int order)
1834 {
1835 	return __prep_compound_gigantic_page(page, order, false);
1836 }
1837 
1838 static bool prep_compound_gigantic_page_for_demote(struct page *page,
1839 							unsigned int order)
1840 {
1841 	return __prep_compound_gigantic_page(page, order, true);
1842 }
1843 
1844 /*
1845  * PageHuge() only returns true for hugetlbfs pages, but not for normal or
1846  * transparent huge pages.  See the PageTransHuge() documentation for more
1847  * details.
1848  */
1849 int PageHuge(struct page *page)
1850 {
1851 	if (!PageCompound(page))
1852 		return 0;
1853 
1854 	page = compound_head(page);
1855 	return page[1].compound_dtor == HUGETLB_PAGE_DTOR;
1856 }
1857 EXPORT_SYMBOL_GPL(PageHuge);
1858 
1859 /*
1860  * PageHeadHuge() only returns true for hugetlbfs head page, but not for
1861  * normal or transparent huge pages.
1862  */
1863 int PageHeadHuge(struct page *page_head)
1864 {
1865 	if (!PageHead(page_head))
1866 		return 0;
1867 
1868 	return page_head[1].compound_dtor == HUGETLB_PAGE_DTOR;
1869 }
1870 EXPORT_SYMBOL_GPL(PageHeadHuge);
1871 
1872 /*
1873  * Find and lock address space (mapping) in write mode.
1874  *
1875  * Upon entry, the page is locked which means that page_mapping() is
1876  * stable.  Due to locking order, we can only trylock_write.  If we can
1877  * not get the lock, simply return NULL to caller.
1878  */
1879 struct address_space *hugetlb_page_mapping_lock_write(struct page *hpage)
1880 {
1881 	struct address_space *mapping = page_mapping(hpage);
1882 
1883 	if (!mapping)
1884 		return mapping;
1885 
1886 	if (i_mmap_trylock_write(mapping))
1887 		return mapping;
1888 
1889 	return NULL;
1890 }
1891 
1892 pgoff_t hugetlb_basepage_index(struct page *page)
1893 {
1894 	struct page *page_head = compound_head(page);
1895 	pgoff_t index = page_index(page_head);
1896 	unsigned long compound_idx;
1897 
1898 	if (compound_order(page_head) >= MAX_ORDER)
1899 		compound_idx = page_to_pfn(page) - page_to_pfn(page_head);
1900 	else
1901 		compound_idx = page - page_head;
1902 
1903 	return (index << compound_order(page_head)) + compound_idx;
1904 }
1905 
1906 static struct page *alloc_buddy_huge_page(struct hstate *h,
1907 		gfp_t gfp_mask, int nid, nodemask_t *nmask,
1908 		nodemask_t *node_alloc_noretry)
1909 {
1910 	int order = huge_page_order(h);
1911 	struct page *page;
1912 	bool alloc_try_hard = true;
1913 
1914 	/*
1915 	 * By default we always try hard to allocate the page with
1916 	 * __GFP_RETRY_MAYFAIL flag.  However, if we are allocating pages in
1917 	 * a loop (to adjust global huge page counts) and previous allocation
1918 	 * failed, do not continue to try hard on the same node.  Use the
1919 	 * node_alloc_noretry bitmap to manage this state information.
1920 	 */
1921 	if (node_alloc_noretry && node_isset(nid, *node_alloc_noretry))
1922 		alloc_try_hard = false;
1923 	gfp_mask |= __GFP_COMP|__GFP_NOWARN;
1924 	if (alloc_try_hard)
1925 		gfp_mask |= __GFP_RETRY_MAYFAIL;
1926 	if (nid == NUMA_NO_NODE)
1927 		nid = numa_mem_id();
1928 	page = __alloc_pages(gfp_mask, order, nid, nmask);
1929 	if (page)
1930 		__count_vm_event(HTLB_BUDDY_PGALLOC);
1931 	else
1932 		__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
1933 
1934 	/*
1935 	 * If we did not specify __GFP_RETRY_MAYFAIL, but still got a page this
1936 	 * indicates an overall state change.  Clear bit so that we resume
1937 	 * normal 'try hard' allocations.
1938 	 */
1939 	if (node_alloc_noretry && page && !alloc_try_hard)
1940 		node_clear(nid, *node_alloc_noretry);
1941 
1942 	/*
1943 	 * If we tried hard to get a page but failed, set bit so that
1944 	 * subsequent attempts will not try as hard until there is an
1945 	 * overall state change.
1946 	 */
1947 	if (node_alloc_noretry && !page && alloc_try_hard)
1948 		node_set(nid, *node_alloc_noretry);
1949 
1950 	return page;
1951 }
1952 
1953 /*
1954  * Common helper to allocate a fresh hugetlb page. All specific allocators
1955  * should use this function to get new hugetlb pages
1956  */
1957 static struct page *alloc_fresh_huge_page(struct hstate *h,
1958 		gfp_t gfp_mask, int nid, nodemask_t *nmask,
1959 		nodemask_t *node_alloc_noretry)
1960 {
1961 	struct page *page;
1962 	bool retry = false;
1963 
1964 retry:
1965 	if (hstate_is_gigantic(h))
1966 		page = alloc_gigantic_page(h, gfp_mask, nid, nmask);
1967 	else
1968 		page = alloc_buddy_huge_page(h, gfp_mask,
1969 				nid, nmask, node_alloc_noretry);
1970 	if (!page)
1971 		return NULL;
1972 
1973 	if (hstate_is_gigantic(h)) {
1974 		if (!prep_compound_gigantic_page(page, huge_page_order(h))) {
1975 			/*
1976 			 * Rare failure to convert pages to compound page.
1977 			 * Free pages and try again - ONCE!
1978 			 */
1979 			free_gigantic_page(page, huge_page_order(h));
1980 			if (!retry) {
1981 				retry = true;
1982 				goto retry;
1983 			}
1984 			return NULL;
1985 		}
1986 	}
1987 	prep_new_huge_page(h, page, page_to_nid(page));
1988 
1989 	return page;
1990 }
1991 
1992 /*
1993  * Allocates a fresh page to the hugetlb allocator pool in the node interleaved
1994  * manner.
1995  */
1996 static int alloc_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed,
1997 				nodemask_t *node_alloc_noretry)
1998 {
1999 	struct page *page;
2000 	int nr_nodes, node;
2001 	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2002 
2003 	for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
2004 		page = alloc_fresh_huge_page(h, gfp_mask, node, nodes_allowed,
2005 						node_alloc_noretry);
2006 		if (page)
2007 			break;
2008 	}
2009 
2010 	if (!page)
2011 		return 0;
2012 
2013 	put_page(page); /* free it into the hugepage allocator */
2014 
2015 	return 1;
2016 }
2017 
2018 /*
2019  * Remove huge page from pool from next node to free.  Attempt to keep
2020  * persistent huge pages more or less balanced over allowed nodes.
2021  * This routine only 'removes' the hugetlb page.  The caller must make
2022  * an additional call to free the page to low level allocators.
2023  * Called with hugetlb_lock locked.
2024  */
2025 static struct page *remove_pool_huge_page(struct hstate *h,
2026 						nodemask_t *nodes_allowed,
2027 						 bool acct_surplus)
2028 {
2029 	int nr_nodes, node;
2030 	struct page *page = NULL;
2031 
2032 	lockdep_assert_held(&hugetlb_lock);
2033 	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
2034 		/*
2035 		 * If we're returning unused surplus pages, only examine
2036 		 * nodes with surplus pages.
2037 		 */
2038 		if ((!acct_surplus || h->surplus_huge_pages_node[node]) &&
2039 		    !list_empty(&h->hugepage_freelists[node])) {
2040 			page = list_entry(h->hugepage_freelists[node].next,
2041 					  struct page, lru);
2042 			remove_hugetlb_page(h, page, acct_surplus);
2043 			break;
2044 		}
2045 	}
2046 
2047 	return page;
2048 }
2049 
2050 /*
2051  * Dissolve a given free hugepage into free buddy pages. This function does
2052  * nothing for in-use hugepages and non-hugepages.
2053  * This function returns values like below:
2054  *
2055  *  -ENOMEM: failed to allocate vmemmap pages to free the freed hugepages
2056  *           when the system is under memory pressure and the feature of
2057  *           freeing unused vmemmap pages associated with each hugetlb page
2058  *           is enabled.
2059  *  -EBUSY:  failed to dissolved free hugepages or the hugepage is in-use
2060  *           (allocated or reserved.)
2061  *       0:  successfully dissolved free hugepages or the page is not a
2062  *           hugepage (considered as already dissolved)
2063  */
2064 int dissolve_free_huge_page(struct page *page)
2065 {
2066 	int rc = -EBUSY;
2067 
2068 retry:
2069 	/* Not to disrupt normal path by vainly holding hugetlb_lock */
2070 	if (!PageHuge(page))
2071 		return 0;
2072 
2073 	spin_lock_irq(&hugetlb_lock);
2074 	if (!PageHuge(page)) {
2075 		rc = 0;
2076 		goto out;
2077 	}
2078 
2079 	if (!page_count(page)) {
2080 		struct page *head = compound_head(page);
2081 		struct hstate *h = page_hstate(head);
2082 		if (h->free_huge_pages - h->resv_huge_pages == 0)
2083 			goto out;
2084 
2085 		/*
2086 		 * We should make sure that the page is already on the free list
2087 		 * when it is dissolved.
2088 		 */
2089 		if (unlikely(!HPageFreed(head))) {
2090 			spin_unlock_irq(&hugetlb_lock);
2091 			cond_resched();
2092 
2093 			/*
2094 			 * Theoretically, we should return -EBUSY when we
2095 			 * encounter this race. In fact, we have a chance
2096 			 * to successfully dissolve the page if we do a
2097 			 * retry. Because the race window is quite small.
2098 			 * If we seize this opportunity, it is an optimization
2099 			 * for increasing the success rate of dissolving page.
2100 			 */
2101 			goto retry;
2102 		}
2103 
2104 		remove_hugetlb_page(h, head, false);
2105 		h->max_huge_pages--;
2106 		spin_unlock_irq(&hugetlb_lock);
2107 
2108 		/*
2109 		 * Normally update_and_free_page will allocate required vmemmmap
2110 		 * before freeing the page.  update_and_free_page will fail to
2111 		 * free the page if it can not allocate required vmemmap.  We
2112 		 * need to adjust max_huge_pages if the page is not freed.
2113 		 * Attempt to allocate vmemmmap here so that we can take
2114 		 * appropriate action on failure.
2115 		 */
2116 		rc = hugetlb_vmemmap_alloc(h, head);
2117 		if (!rc) {
2118 			/*
2119 			 * Move PageHWPoison flag from head page to the raw
2120 			 * error page, which makes any subpages rather than
2121 			 * the error page reusable.
2122 			 */
2123 			if (PageHWPoison(head) && page != head) {
2124 				SetPageHWPoison(page);
2125 				ClearPageHWPoison(head);
2126 			}
2127 			update_and_free_page(h, head, false);
2128 		} else {
2129 			spin_lock_irq(&hugetlb_lock);
2130 			add_hugetlb_page(h, head, false);
2131 			h->max_huge_pages++;
2132 			spin_unlock_irq(&hugetlb_lock);
2133 		}
2134 
2135 		return rc;
2136 	}
2137 out:
2138 	spin_unlock_irq(&hugetlb_lock);
2139 	return rc;
2140 }
2141 
2142 /*
2143  * Dissolve free hugepages in a given pfn range. Used by memory hotplug to
2144  * make specified memory blocks removable from the system.
2145  * Note that this will dissolve a free gigantic hugepage completely, if any
2146  * part of it lies within the given range.
2147  * Also note that if dissolve_free_huge_page() returns with an error, all
2148  * free hugepages that were dissolved before that error are lost.
2149  */
2150 int dissolve_free_huge_pages(unsigned long start_pfn, unsigned long end_pfn)
2151 {
2152 	unsigned long pfn;
2153 	struct page *page;
2154 	int rc = 0;
2155 
2156 	if (!hugepages_supported())
2157 		return rc;
2158 
2159 	for (pfn = start_pfn; pfn < end_pfn; pfn += 1 << minimum_order) {
2160 		page = pfn_to_page(pfn);
2161 		rc = dissolve_free_huge_page(page);
2162 		if (rc)
2163 			break;
2164 	}
2165 
2166 	return rc;
2167 }
2168 
2169 /*
2170  * Allocates a fresh surplus page from the page allocator.
2171  */
2172 static struct page *alloc_surplus_huge_page(struct hstate *h, gfp_t gfp_mask,
2173 		int nid, nodemask_t *nmask, bool zero_ref)
2174 {
2175 	struct page *page = NULL;
2176 	bool retry = false;
2177 
2178 	if (hstate_is_gigantic(h))
2179 		return NULL;
2180 
2181 	spin_lock_irq(&hugetlb_lock);
2182 	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages)
2183 		goto out_unlock;
2184 	spin_unlock_irq(&hugetlb_lock);
2185 
2186 retry:
2187 	page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
2188 	if (!page)
2189 		return NULL;
2190 
2191 	spin_lock_irq(&hugetlb_lock);
2192 	/*
2193 	 * We could have raced with the pool size change.
2194 	 * Double check that and simply deallocate the new page
2195 	 * if we would end up overcommiting the surpluses. Abuse
2196 	 * temporary page to workaround the nasty free_huge_page
2197 	 * codeflow
2198 	 */
2199 	if (h->surplus_huge_pages >= h->nr_overcommit_huge_pages) {
2200 		SetHPageTemporary(page);
2201 		spin_unlock_irq(&hugetlb_lock);
2202 		put_page(page);
2203 		return NULL;
2204 	}
2205 
2206 	if (zero_ref) {
2207 		/*
2208 		 * Caller requires a page with zero ref count.
2209 		 * We will drop ref count here.  If someone else is holding
2210 		 * a ref, the page will be freed when they drop it.  Abuse
2211 		 * temporary page flag to accomplish this.
2212 		 */
2213 		SetHPageTemporary(page);
2214 		if (!put_page_testzero(page)) {
2215 			/*
2216 			 * Unexpected inflated ref count on freshly allocated
2217 			 * huge.  Retry once.
2218 			 */
2219 			pr_info("HugeTLB unexpected inflated ref count on freshly allocated page\n");
2220 			spin_unlock_irq(&hugetlb_lock);
2221 			if (retry)
2222 				return NULL;
2223 
2224 			retry = true;
2225 			goto retry;
2226 		}
2227 		ClearHPageTemporary(page);
2228 	}
2229 
2230 	h->surplus_huge_pages++;
2231 	h->surplus_huge_pages_node[page_to_nid(page)]++;
2232 
2233 out_unlock:
2234 	spin_unlock_irq(&hugetlb_lock);
2235 
2236 	return page;
2237 }
2238 
2239 static struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
2240 				     int nid, nodemask_t *nmask)
2241 {
2242 	struct page *page;
2243 
2244 	if (hstate_is_gigantic(h))
2245 		return NULL;
2246 
2247 	page = alloc_fresh_huge_page(h, gfp_mask, nid, nmask, NULL);
2248 	if (!page)
2249 		return NULL;
2250 
2251 	/*
2252 	 * We do not account these pages as surplus because they are only
2253 	 * temporary and will be released properly on the last reference
2254 	 */
2255 	SetHPageTemporary(page);
2256 
2257 	return page;
2258 }
2259 
2260 /*
2261  * Use the VMA's mpolicy to allocate a huge page from the buddy.
2262  */
2263 static
2264 struct page *alloc_buddy_huge_page_with_mpol(struct hstate *h,
2265 		struct vm_area_struct *vma, unsigned long addr)
2266 {
2267 	struct page *page = NULL;
2268 	struct mempolicy *mpol;
2269 	gfp_t gfp_mask = htlb_alloc_mask(h);
2270 	int nid;
2271 	nodemask_t *nodemask;
2272 
2273 	nid = huge_node(vma, addr, gfp_mask, &mpol, &nodemask);
2274 	if (mpol_is_preferred_many(mpol)) {
2275 		gfp_t gfp = gfp_mask | __GFP_NOWARN;
2276 
2277 		gfp &=  ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2278 		page = alloc_surplus_huge_page(h, gfp, nid, nodemask, false);
2279 
2280 		/* Fallback to all nodes if page==NULL */
2281 		nodemask = NULL;
2282 	}
2283 
2284 	if (!page)
2285 		page = alloc_surplus_huge_page(h, gfp_mask, nid, nodemask, false);
2286 	mpol_cond_put(mpol);
2287 	return page;
2288 }
2289 
2290 /* page migration callback function */
2291 struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
2292 		nodemask_t *nmask, gfp_t gfp_mask)
2293 {
2294 	spin_lock_irq(&hugetlb_lock);
2295 	if (h->free_huge_pages - h->resv_huge_pages > 0) {
2296 		struct page *page;
2297 
2298 		page = dequeue_huge_page_nodemask(h, gfp_mask, preferred_nid, nmask);
2299 		if (page) {
2300 			spin_unlock_irq(&hugetlb_lock);
2301 			return page;
2302 		}
2303 	}
2304 	spin_unlock_irq(&hugetlb_lock);
2305 
2306 	return alloc_migrate_huge_page(h, gfp_mask, preferred_nid, nmask);
2307 }
2308 
2309 /* mempolicy aware migration callback */
2310 struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
2311 		unsigned long address)
2312 {
2313 	struct mempolicy *mpol;
2314 	nodemask_t *nodemask;
2315 	struct page *page;
2316 	gfp_t gfp_mask;
2317 	int node;
2318 
2319 	gfp_mask = htlb_alloc_mask(h);
2320 	node = huge_node(vma, address, gfp_mask, &mpol, &nodemask);
2321 	page = alloc_huge_page_nodemask(h, node, nodemask, gfp_mask);
2322 	mpol_cond_put(mpol);
2323 
2324 	return page;
2325 }
2326 
2327 /*
2328  * Increase the hugetlb pool such that it can accommodate a reservation
2329  * of size 'delta'.
2330  */
2331 static int gather_surplus_pages(struct hstate *h, long delta)
2332 	__must_hold(&hugetlb_lock)
2333 {
2334 	struct list_head surplus_list;
2335 	struct page *page, *tmp;
2336 	int ret;
2337 	long i;
2338 	long needed, allocated;
2339 	bool alloc_ok = true;
2340 
2341 	lockdep_assert_held(&hugetlb_lock);
2342 	needed = (h->resv_huge_pages + delta) - h->free_huge_pages;
2343 	if (needed <= 0) {
2344 		h->resv_huge_pages += delta;
2345 		return 0;
2346 	}
2347 
2348 	allocated = 0;
2349 	INIT_LIST_HEAD(&surplus_list);
2350 
2351 	ret = -ENOMEM;
2352 retry:
2353 	spin_unlock_irq(&hugetlb_lock);
2354 	for (i = 0; i < needed; i++) {
2355 		page = alloc_surplus_huge_page(h, htlb_alloc_mask(h),
2356 				NUMA_NO_NODE, NULL, true);
2357 		if (!page) {
2358 			alloc_ok = false;
2359 			break;
2360 		}
2361 		list_add(&page->lru, &surplus_list);
2362 		cond_resched();
2363 	}
2364 	allocated += i;
2365 
2366 	/*
2367 	 * After retaking hugetlb_lock, we need to recalculate 'needed'
2368 	 * because either resv_huge_pages or free_huge_pages may have changed.
2369 	 */
2370 	spin_lock_irq(&hugetlb_lock);
2371 	needed = (h->resv_huge_pages + delta) -
2372 			(h->free_huge_pages + allocated);
2373 	if (needed > 0) {
2374 		if (alloc_ok)
2375 			goto retry;
2376 		/*
2377 		 * We were not able to allocate enough pages to
2378 		 * satisfy the entire reservation so we free what
2379 		 * we've allocated so far.
2380 		 */
2381 		goto free;
2382 	}
2383 	/*
2384 	 * The surplus_list now contains _at_least_ the number of extra pages
2385 	 * needed to accommodate the reservation.  Add the appropriate number
2386 	 * of pages to the hugetlb pool and free the extras back to the buddy
2387 	 * allocator.  Commit the entire reservation here to prevent another
2388 	 * process from stealing the pages as they are added to the pool but
2389 	 * before they are reserved.
2390 	 */
2391 	needed += allocated;
2392 	h->resv_huge_pages += delta;
2393 	ret = 0;
2394 
2395 	/* Free the needed pages to the hugetlb pool */
2396 	list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
2397 		if ((--needed) < 0)
2398 			break;
2399 		/* Add the page to the hugetlb allocator */
2400 		enqueue_huge_page(h, page);
2401 	}
2402 free:
2403 	spin_unlock_irq(&hugetlb_lock);
2404 
2405 	/*
2406 	 * Free unnecessary surplus pages to the buddy allocator.
2407 	 * Pages have no ref count, call free_huge_page directly.
2408 	 */
2409 	list_for_each_entry_safe(page, tmp, &surplus_list, lru)
2410 		free_huge_page(page);
2411 	spin_lock_irq(&hugetlb_lock);
2412 
2413 	return ret;
2414 }
2415 
2416 /*
2417  * This routine has two main purposes:
2418  * 1) Decrement the reservation count (resv_huge_pages) by the value passed
2419  *    in unused_resv_pages.  This corresponds to the prior adjustments made
2420  *    to the associated reservation map.
2421  * 2) Free any unused surplus pages that may have been allocated to satisfy
2422  *    the reservation.  As many as unused_resv_pages may be freed.
2423  */
2424 static void return_unused_surplus_pages(struct hstate *h,
2425 					unsigned long unused_resv_pages)
2426 {
2427 	unsigned long nr_pages;
2428 	struct page *page;
2429 	LIST_HEAD(page_list);
2430 
2431 	lockdep_assert_held(&hugetlb_lock);
2432 	/* Uncommit the reservation */
2433 	h->resv_huge_pages -= unused_resv_pages;
2434 
2435 	/* Cannot return gigantic pages currently */
2436 	if (hstate_is_gigantic(h))
2437 		goto out;
2438 
2439 	/*
2440 	 * Part (or even all) of the reservation could have been backed
2441 	 * by pre-allocated pages. Only free surplus pages.
2442 	 */
2443 	nr_pages = min(unused_resv_pages, h->surplus_huge_pages);
2444 
2445 	/*
2446 	 * We want to release as many surplus pages as possible, spread
2447 	 * evenly across all nodes with memory. Iterate across these nodes
2448 	 * until we can no longer free unreserved surplus pages. This occurs
2449 	 * when the nodes with surplus pages have no free pages.
2450 	 * remove_pool_huge_page() will balance the freed pages across the
2451 	 * on-line nodes with memory and will handle the hstate accounting.
2452 	 */
2453 	while (nr_pages--) {
2454 		page = remove_pool_huge_page(h, &node_states[N_MEMORY], 1);
2455 		if (!page)
2456 			goto out;
2457 
2458 		list_add(&page->lru, &page_list);
2459 	}
2460 
2461 out:
2462 	spin_unlock_irq(&hugetlb_lock);
2463 	update_and_free_pages_bulk(h, &page_list);
2464 	spin_lock_irq(&hugetlb_lock);
2465 }
2466 
2467 
2468 /*
2469  * vma_needs_reservation, vma_commit_reservation and vma_end_reservation
2470  * are used by the huge page allocation routines to manage reservations.
2471  *
2472  * vma_needs_reservation is called to determine if the huge page at addr
2473  * within the vma has an associated reservation.  If a reservation is
2474  * needed, the value 1 is returned.  The caller is then responsible for
2475  * managing the global reservation and subpool usage counts.  After
2476  * the huge page has been allocated, vma_commit_reservation is called
2477  * to add the page to the reservation map.  If the page allocation fails,
2478  * the reservation must be ended instead of committed.  vma_end_reservation
2479  * is called in such cases.
2480  *
2481  * In the normal case, vma_commit_reservation returns the same value
2482  * as the preceding vma_needs_reservation call.  The only time this
2483  * is not the case is if a reserve map was changed between calls.  It
2484  * is the responsibility of the caller to notice the difference and
2485  * take appropriate action.
2486  *
2487  * vma_add_reservation is used in error paths where a reservation must
2488  * be restored when a newly allocated huge page must be freed.  It is
2489  * to be called after calling vma_needs_reservation to determine if a
2490  * reservation exists.
2491  *
2492  * vma_del_reservation is used in error paths where an entry in the reserve
2493  * map was created during huge page allocation and must be removed.  It is to
2494  * be called after calling vma_needs_reservation to determine if a reservation
2495  * exists.
2496  */
2497 enum vma_resv_mode {
2498 	VMA_NEEDS_RESV,
2499 	VMA_COMMIT_RESV,
2500 	VMA_END_RESV,
2501 	VMA_ADD_RESV,
2502 	VMA_DEL_RESV,
2503 };
2504 static long __vma_reservation_common(struct hstate *h,
2505 				struct vm_area_struct *vma, unsigned long addr,
2506 				enum vma_resv_mode mode)
2507 {
2508 	struct resv_map *resv;
2509 	pgoff_t idx;
2510 	long ret;
2511 	long dummy_out_regions_needed;
2512 
2513 	resv = vma_resv_map(vma);
2514 	if (!resv)
2515 		return 1;
2516 
2517 	idx = vma_hugecache_offset(h, vma, addr);
2518 	switch (mode) {
2519 	case VMA_NEEDS_RESV:
2520 		ret = region_chg(resv, idx, idx + 1, &dummy_out_regions_needed);
2521 		/* We assume that vma_reservation_* routines always operate on
2522 		 * 1 page, and that adding to resv map a 1 page entry can only
2523 		 * ever require 1 region.
2524 		 */
2525 		VM_BUG_ON(dummy_out_regions_needed != 1);
2526 		break;
2527 	case VMA_COMMIT_RESV:
2528 		ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2529 		/* region_add calls of range 1 should never fail. */
2530 		VM_BUG_ON(ret < 0);
2531 		break;
2532 	case VMA_END_RESV:
2533 		region_abort(resv, idx, idx + 1, 1);
2534 		ret = 0;
2535 		break;
2536 	case VMA_ADD_RESV:
2537 		if (vma->vm_flags & VM_MAYSHARE) {
2538 			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2539 			/* region_add calls of range 1 should never fail. */
2540 			VM_BUG_ON(ret < 0);
2541 		} else {
2542 			region_abort(resv, idx, idx + 1, 1);
2543 			ret = region_del(resv, idx, idx + 1);
2544 		}
2545 		break;
2546 	case VMA_DEL_RESV:
2547 		if (vma->vm_flags & VM_MAYSHARE) {
2548 			region_abort(resv, idx, idx + 1, 1);
2549 			ret = region_del(resv, idx, idx + 1);
2550 		} else {
2551 			ret = region_add(resv, idx, idx + 1, 1, NULL, NULL);
2552 			/* region_add calls of range 1 should never fail. */
2553 			VM_BUG_ON(ret < 0);
2554 		}
2555 		break;
2556 	default:
2557 		BUG();
2558 	}
2559 
2560 	if (vma->vm_flags & VM_MAYSHARE || mode == VMA_DEL_RESV)
2561 		return ret;
2562 	/*
2563 	 * We know private mapping must have HPAGE_RESV_OWNER set.
2564 	 *
2565 	 * In most cases, reserves always exist for private mappings.
2566 	 * However, a file associated with mapping could have been
2567 	 * hole punched or truncated after reserves were consumed.
2568 	 * As subsequent fault on such a range will not use reserves.
2569 	 * Subtle - The reserve map for private mappings has the
2570 	 * opposite meaning than that of shared mappings.  If NO
2571 	 * entry is in the reserve map, it means a reservation exists.
2572 	 * If an entry exists in the reserve map, it means the
2573 	 * reservation has already been consumed.  As a result, the
2574 	 * return value of this routine is the opposite of the
2575 	 * value returned from reserve map manipulation routines above.
2576 	 */
2577 	if (ret > 0)
2578 		return 0;
2579 	if (ret == 0)
2580 		return 1;
2581 	return ret;
2582 }
2583 
2584 static long vma_needs_reservation(struct hstate *h,
2585 			struct vm_area_struct *vma, unsigned long addr)
2586 {
2587 	return __vma_reservation_common(h, vma, addr, VMA_NEEDS_RESV);
2588 }
2589 
2590 static long vma_commit_reservation(struct hstate *h,
2591 			struct vm_area_struct *vma, unsigned long addr)
2592 {
2593 	return __vma_reservation_common(h, vma, addr, VMA_COMMIT_RESV);
2594 }
2595 
2596 static void vma_end_reservation(struct hstate *h,
2597 			struct vm_area_struct *vma, unsigned long addr)
2598 {
2599 	(void)__vma_reservation_common(h, vma, addr, VMA_END_RESV);
2600 }
2601 
2602 static long vma_add_reservation(struct hstate *h,
2603 			struct vm_area_struct *vma, unsigned long addr)
2604 {
2605 	return __vma_reservation_common(h, vma, addr, VMA_ADD_RESV);
2606 }
2607 
2608 static long vma_del_reservation(struct hstate *h,
2609 			struct vm_area_struct *vma, unsigned long addr)
2610 {
2611 	return __vma_reservation_common(h, vma, addr, VMA_DEL_RESV);
2612 }
2613 
2614 /*
2615  * This routine is called to restore reservation information on error paths.
2616  * It should ONLY be called for pages allocated via alloc_huge_page(), and
2617  * the hugetlb mutex should remain held when calling this routine.
2618  *
2619  * It handles two specific cases:
2620  * 1) A reservation was in place and the page consumed the reservation.
2621  *    HPageRestoreReserve is set in the page.
2622  * 2) No reservation was in place for the page, so HPageRestoreReserve is
2623  *    not set.  However, alloc_huge_page always updates the reserve map.
2624  *
2625  * In case 1, free_huge_page later in the error path will increment the
2626  * global reserve count.  But, free_huge_page does not have enough context
2627  * to adjust the reservation map.  This case deals primarily with private
2628  * mappings.  Adjust the reserve map here to be consistent with global
2629  * reserve count adjustments to be made by free_huge_page.  Make sure the
2630  * reserve map indicates there is a reservation present.
2631  *
2632  * In case 2, simply undo reserve map modifications done by alloc_huge_page.
2633  */
2634 void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma,
2635 			unsigned long address, struct page *page)
2636 {
2637 	long rc = vma_needs_reservation(h, vma, address);
2638 
2639 	if (HPageRestoreReserve(page)) {
2640 		if (unlikely(rc < 0))
2641 			/*
2642 			 * Rare out of memory condition in reserve map
2643 			 * manipulation.  Clear HPageRestoreReserve so that
2644 			 * global reserve count will not be incremented
2645 			 * by free_huge_page.  This will make it appear
2646 			 * as though the reservation for this page was
2647 			 * consumed.  This may prevent the task from
2648 			 * faulting in the page at a later time.  This
2649 			 * is better than inconsistent global huge page
2650 			 * accounting of reserve counts.
2651 			 */
2652 			ClearHPageRestoreReserve(page);
2653 		else if (rc)
2654 			(void)vma_add_reservation(h, vma, address);
2655 		else
2656 			vma_end_reservation(h, vma, address);
2657 	} else {
2658 		if (!rc) {
2659 			/*
2660 			 * This indicates there is an entry in the reserve map
2661 			 * not added by alloc_huge_page.  We know it was added
2662 			 * before the alloc_huge_page call, otherwise
2663 			 * HPageRestoreReserve would be set on the page.
2664 			 * Remove the entry so that a subsequent allocation
2665 			 * does not consume a reservation.
2666 			 */
2667 			rc = vma_del_reservation(h, vma, address);
2668 			if (rc < 0)
2669 				/*
2670 				 * VERY rare out of memory condition.  Since
2671 				 * we can not delete the entry, set
2672 				 * HPageRestoreReserve so that the reserve
2673 				 * count will be incremented when the page
2674 				 * is freed.  This reserve will be consumed
2675 				 * on a subsequent allocation.
2676 				 */
2677 				SetHPageRestoreReserve(page);
2678 		} else if (rc < 0) {
2679 			/*
2680 			 * Rare out of memory condition from
2681 			 * vma_needs_reservation call.  Memory allocation is
2682 			 * only attempted if a new entry is needed.  Therefore,
2683 			 * this implies there is not an entry in the
2684 			 * reserve map.
2685 			 *
2686 			 * For shared mappings, no entry in the map indicates
2687 			 * no reservation.  We are done.
2688 			 */
2689 			if (!(vma->vm_flags & VM_MAYSHARE))
2690 				/*
2691 				 * For private mappings, no entry indicates
2692 				 * a reservation is present.  Since we can
2693 				 * not add an entry, set SetHPageRestoreReserve
2694 				 * on the page so reserve count will be
2695 				 * incremented when freed.  This reserve will
2696 				 * be consumed on a subsequent allocation.
2697 				 */
2698 				SetHPageRestoreReserve(page);
2699 		} else
2700 			/*
2701 			 * No reservation present, do nothing
2702 			 */
2703 			 vma_end_reservation(h, vma, address);
2704 	}
2705 }
2706 
2707 /*
2708  * alloc_and_dissolve_huge_page - Allocate a new page and dissolve the old one
2709  * @h: struct hstate old page belongs to
2710  * @old_page: Old page to dissolve
2711  * @list: List to isolate the page in case we need to
2712  * Returns 0 on success, otherwise negated error.
2713  */
2714 static int alloc_and_dissolve_huge_page(struct hstate *h, struct page *old_page,
2715 					struct list_head *list)
2716 {
2717 	gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
2718 	int nid = page_to_nid(old_page);
2719 	bool alloc_retry = false;
2720 	struct page *new_page;
2721 	int ret = 0;
2722 
2723 	/*
2724 	 * Before dissolving the page, we need to allocate a new one for the
2725 	 * pool to remain stable.  Here, we allocate the page and 'prep' it
2726 	 * by doing everything but actually updating counters and adding to
2727 	 * the pool.  This simplifies and let us do most of the processing
2728 	 * under the lock.
2729 	 */
2730 alloc_retry:
2731 	new_page = alloc_buddy_huge_page(h, gfp_mask, nid, NULL, NULL);
2732 	if (!new_page)
2733 		return -ENOMEM;
2734 	/*
2735 	 * If all goes well, this page will be directly added to the free
2736 	 * list in the pool.  For this the ref count needs to be zero.
2737 	 * Attempt to drop now, and retry once if needed.  It is VERY
2738 	 * unlikely there is another ref on the page.
2739 	 *
2740 	 * If someone else has a reference to the page, it will be freed
2741 	 * when they drop their ref.  Abuse temporary page flag to accomplish
2742 	 * this.  Retry once if there is an inflated ref count.
2743 	 */
2744 	SetHPageTemporary(new_page);
2745 	if (!put_page_testzero(new_page)) {
2746 		if (alloc_retry)
2747 			return -EBUSY;
2748 
2749 		alloc_retry = true;
2750 		goto alloc_retry;
2751 	}
2752 	ClearHPageTemporary(new_page);
2753 
2754 	__prep_new_huge_page(h, new_page);
2755 
2756 retry:
2757 	spin_lock_irq(&hugetlb_lock);
2758 	if (!PageHuge(old_page)) {
2759 		/*
2760 		 * Freed from under us. Drop new_page too.
2761 		 */
2762 		goto free_new;
2763 	} else if (page_count(old_page)) {
2764 		/*
2765 		 * Someone has grabbed the page, try to isolate it here.
2766 		 * Fail with -EBUSY if not possible.
2767 		 */
2768 		spin_unlock_irq(&hugetlb_lock);
2769 		if (!isolate_huge_page(old_page, list))
2770 			ret = -EBUSY;
2771 		spin_lock_irq(&hugetlb_lock);
2772 		goto free_new;
2773 	} else if (!HPageFreed(old_page)) {
2774 		/*
2775 		 * Page's refcount is 0 but it has not been enqueued in the
2776 		 * freelist yet. Race window is small, so we can succeed here if
2777 		 * we retry.
2778 		 */
2779 		spin_unlock_irq(&hugetlb_lock);
2780 		cond_resched();
2781 		goto retry;
2782 	} else {
2783 		/*
2784 		 * Ok, old_page is still a genuine free hugepage. Remove it from
2785 		 * the freelist and decrease the counters. These will be
2786 		 * incremented again when calling __prep_account_new_huge_page()
2787 		 * and enqueue_huge_page() for new_page. The counters will remain
2788 		 * stable since this happens under the lock.
2789 		 */
2790 		remove_hugetlb_page(h, old_page, false);
2791 
2792 		/*
2793 		 * Ref count on new page is already zero as it was dropped
2794 		 * earlier.  It can be directly added to the pool free list.
2795 		 */
2796 		__prep_account_new_huge_page(h, nid);
2797 		enqueue_huge_page(h, new_page);
2798 
2799 		/*
2800 		 * Pages have been replaced, we can safely free the old one.
2801 		 */
2802 		spin_unlock_irq(&hugetlb_lock);
2803 		update_and_free_page(h, old_page, false);
2804 	}
2805 
2806 	return ret;
2807 
2808 free_new:
2809 	spin_unlock_irq(&hugetlb_lock);
2810 	/* Page has a zero ref count, but needs a ref to be freed */
2811 	set_page_refcounted(new_page);
2812 	update_and_free_page(h, new_page, false);
2813 
2814 	return ret;
2815 }
2816 
2817 int isolate_or_dissolve_huge_page(struct page *page, struct list_head *list)
2818 {
2819 	struct hstate *h;
2820 	struct page *head;
2821 	int ret = -EBUSY;
2822 
2823 	/*
2824 	 * The page might have been dissolved from under our feet, so make sure
2825 	 * to carefully check the state under the lock.
2826 	 * Return success when racing as if we dissolved the page ourselves.
2827 	 */
2828 	spin_lock_irq(&hugetlb_lock);
2829 	if (PageHuge(page)) {
2830 		head = compound_head(page);
2831 		h = page_hstate(head);
2832 	} else {
2833 		spin_unlock_irq(&hugetlb_lock);
2834 		return 0;
2835 	}
2836 	spin_unlock_irq(&hugetlb_lock);
2837 
2838 	/*
2839 	 * Fence off gigantic pages as there is a cyclic dependency between
2840 	 * alloc_contig_range and them. Return -ENOMEM as this has the effect
2841 	 * of bailing out right away without further retrying.
2842 	 */
2843 	if (hstate_is_gigantic(h))
2844 		return -ENOMEM;
2845 
2846 	if (page_count(head) && isolate_huge_page(head, list))
2847 		ret = 0;
2848 	else if (!page_count(head))
2849 		ret = alloc_and_dissolve_huge_page(h, head, list);
2850 
2851 	return ret;
2852 }
2853 
2854 struct page *alloc_huge_page(struct vm_area_struct *vma,
2855 				    unsigned long addr, int avoid_reserve)
2856 {
2857 	struct hugepage_subpool *spool = subpool_vma(vma);
2858 	struct hstate *h = hstate_vma(vma);
2859 	struct page *page;
2860 	long map_chg, map_commit;
2861 	long gbl_chg;
2862 	int ret, idx;
2863 	struct hugetlb_cgroup *h_cg;
2864 	bool deferred_reserve;
2865 
2866 	idx = hstate_index(h);
2867 	/*
2868 	 * Examine the region/reserve map to determine if the process
2869 	 * has a reservation for the page to be allocated.  A return
2870 	 * code of zero indicates a reservation exists (no change).
2871 	 */
2872 	map_chg = gbl_chg = vma_needs_reservation(h, vma, addr);
2873 	if (map_chg < 0)
2874 		return ERR_PTR(-ENOMEM);
2875 
2876 	/*
2877 	 * Processes that did not create the mapping will have no
2878 	 * reserves as indicated by the region/reserve map. Check
2879 	 * that the allocation will not exceed the subpool limit.
2880 	 * Allocations for MAP_NORESERVE mappings also need to be
2881 	 * checked against any subpool limit.
2882 	 */
2883 	if (map_chg || avoid_reserve) {
2884 		gbl_chg = hugepage_subpool_get_pages(spool, 1);
2885 		if (gbl_chg < 0) {
2886 			vma_end_reservation(h, vma, addr);
2887 			return ERR_PTR(-ENOSPC);
2888 		}
2889 
2890 		/*
2891 		 * Even though there was no reservation in the region/reserve
2892 		 * map, there could be reservations associated with the
2893 		 * subpool that can be used.  This would be indicated if the
2894 		 * return value of hugepage_subpool_get_pages() is zero.
2895 		 * However, if avoid_reserve is specified we still avoid even
2896 		 * the subpool reservations.
2897 		 */
2898 		if (avoid_reserve)
2899 			gbl_chg = 1;
2900 	}
2901 
2902 	/* If this allocation is not consuming a reservation, charge it now.
2903 	 */
2904 	deferred_reserve = map_chg || avoid_reserve;
2905 	if (deferred_reserve) {
2906 		ret = hugetlb_cgroup_charge_cgroup_rsvd(
2907 			idx, pages_per_huge_page(h), &h_cg);
2908 		if (ret)
2909 			goto out_subpool_put;
2910 	}
2911 
2912 	ret = hugetlb_cgroup_charge_cgroup(idx, pages_per_huge_page(h), &h_cg);
2913 	if (ret)
2914 		goto out_uncharge_cgroup_reservation;
2915 
2916 	spin_lock_irq(&hugetlb_lock);
2917 	/*
2918 	 * glb_chg is passed to indicate whether or not a page must be taken
2919 	 * from the global free pool (global change).  gbl_chg == 0 indicates
2920 	 * a reservation exists for the allocation.
2921 	 */
2922 	page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
2923 	if (!page) {
2924 		spin_unlock_irq(&hugetlb_lock);
2925 		page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
2926 		if (!page)
2927 			goto out_uncharge_cgroup;
2928 		if (!avoid_reserve && vma_has_reserves(vma, gbl_chg)) {
2929 			SetHPageRestoreReserve(page);
2930 			h->resv_huge_pages--;
2931 		}
2932 		spin_lock_irq(&hugetlb_lock);
2933 		list_add(&page->lru, &h->hugepage_activelist);
2934 		/* Fall through */
2935 	}
2936 	hugetlb_cgroup_commit_charge(idx, pages_per_huge_page(h), h_cg, page);
2937 	/* If allocation is not consuming a reservation, also store the
2938 	 * hugetlb_cgroup pointer on the page.
2939 	 */
2940 	if (deferred_reserve) {
2941 		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
2942 						  h_cg, page);
2943 	}
2944 
2945 	spin_unlock_irq(&hugetlb_lock);
2946 
2947 	hugetlb_set_page_subpool(page, spool);
2948 
2949 	map_commit = vma_commit_reservation(h, vma, addr);
2950 	if (unlikely(map_chg > map_commit)) {
2951 		/*
2952 		 * The page was added to the reservation map between
2953 		 * vma_needs_reservation and vma_commit_reservation.
2954 		 * This indicates a race with hugetlb_reserve_pages.
2955 		 * Adjust for the subpool count incremented above AND
2956 		 * in hugetlb_reserve_pages for the same page.  Also,
2957 		 * the reservation count added in hugetlb_reserve_pages
2958 		 * no longer applies.
2959 		 */
2960 		long rsv_adjust;
2961 
2962 		rsv_adjust = hugepage_subpool_put_pages(spool, 1);
2963 		hugetlb_acct_memory(h, -rsv_adjust);
2964 		if (deferred_reserve)
2965 			hugetlb_cgroup_uncharge_page_rsvd(hstate_index(h),
2966 					pages_per_huge_page(h), page);
2967 	}
2968 	return page;
2969 
2970 out_uncharge_cgroup:
2971 	hugetlb_cgroup_uncharge_cgroup(idx, pages_per_huge_page(h), h_cg);
2972 out_uncharge_cgroup_reservation:
2973 	if (deferred_reserve)
2974 		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
2975 						    h_cg);
2976 out_subpool_put:
2977 	if (map_chg || avoid_reserve)
2978 		hugepage_subpool_put_pages(spool, 1);
2979 	vma_end_reservation(h, vma, addr);
2980 	return ERR_PTR(-ENOSPC);
2981 }
2982 
2983 int alloc_bootmem_huge_page(struct hstate *h, int nid)
2984 	__attribute__ ((weak, alias("__alloc_bootmem_huge_page")));
2985 int __alloc_bootmem_huge_page(struct hstate *h, int nid)
2986 {
2987 	struct huge_bootmem_page *m = NULL; /* initialize for clang */
2988 	int nr_nodes, node;
2989 
2990 	/* do node specific alloc */
2991 	if (nid != NUMA_NO_NODE) {
2992 		m = memblock_alloc_try_nid_raw(huge_page_size(h), huge_page_size(h),
2993 				0, MEMBLOCK_ALLOC_ACCESSIBLE, nid);
2994 		if (!m)
2995 			return 0;
2996 		goto found;
2997 	}
2998 	/* allocate from next node when distributing huge pages */
2999 	for_each_node_mask_to_alloc(h, nr_nodes, node, &node_states[N_MEMORY]) {
3000 		m = memblock_alloc_try_nid_raw(
3001 				huge_page_size(h), huge_page_size(h),
3002 				0, MEMBLOCK_ALLOC_ACCESSIBLE, node);
3003 		/*
3004 		 * Use the beginning of the huge page to store the
3005 		 * huge_bootmem_page struct (until gather_bootmem
3006 		 * puts them into the mem_map).
3007 		 */
3008 		if (!m)
3009 			return 0;
3010 		goto found;
3011 	}
3012 
3013 found:
3014 	/* Put them into a private list first because mem_map is not up yet */
3015 	INIT_LIST_HEAD(&m->list);
3016 	list_add(&m->list, &huge_boot_pages);
3017 	m->hstate = h;
3018 	return 1;
3019 }
3020 
3021 /*
3022  * Put bootmem huge pages into the standard lists after mem_map is up.
3023  * Note: This only applies to gigantic (order > MAX_ORDER) pages.
3024  */
3025 static void __init gather_bootmem_prealloc(void)
3026 {
3027 	struct huge_bootmem_page *m;
3028 
3029 	list_for_each_entry(m, &huge_boot_pages, list) {
3030 		struct page *page = virt_to_page(m);
3031 		struct hstate *h = m->hstate;
3032 
3033 		VM_BUG_ON(!hstate_is_gigantic(h));
3034 		WARN_ON(page_count(page) != 1);
3035 		if (prep_compound_gigantic_page(page, huge_page_order(h))) {
3036 			WARN_ON(PageReserved(page));
3037 			prep_new_huge_page(h, page, page_to_nid(page));
3038 			put_page(page); /* add to the hugepage allocator */
3039 		} else {
3040 			/* VERY unlikely inflated ref count on a tail page */
3041 			free_gigantic_page(page, huge_page_order(h));
3042 		}
3043 
3044 		/*
3045 		 * We need to restore the 'stolen' pages to totalram_pages
3046 		 * in order to fix confusing memory reports from free(1) and
3047 		 * other side-effects, like CommitLimit going negative.
3048 		 */
3049 		adjust_managed_page_count(page, pages_per_huge_page(h));
3050 		cond_resched();
3051 	}
3052 }
3053 static void __init hugetlb_hstate_alloc_pages_onenode(struct hstate *h, int nid)
3054 {
3055 	unsigned long i;
3056 	char buf[32];
3057 
3058 	for (i = 0; i < h->max_huge_pages_node[nid]; ++i) {
3059 		if (hstate_is_gigantic(h)) {
3060 			if (!alloc_bootmem_huge_page(h, nid))
3061 				break;
3062 		} else {
3063 			struct page *page;
3064 			gfp_t gfp_mask = htlb_alloc_mask(h) | __GFP_THISNODE;
3065 
3066 			page = alloc_fresh_huge_page(h, gfp_mask, nid,
3067 					&node_states[N_MEMORY], NULL);
3068 			if (!page)
3069 				break;
3070 			put_page(page); /* free it into the hugepage allocator */
3071 		}
3072 		cond_resched();
3073 	}
3074 	if (i == h->max_huge_pages_node[nid])
3075 		return;
3076 
3077 	string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3078 	pr_warn("HugeTLB: allocating %u of page size %s failed node%d.  Only allocated %lu hugepages.\n",
3079 		h->max_huge_pages_node[nid], buf, nid, i);
3080 	h->max_huge_pages -= (h->max_huge_pages_node[nid] - i);
3081 	h->max_huge_pages_node[nid] = i;
3082 }
3083 
3084 static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
3085 {
3086 	unsigned long i;
3087 	nodemask_t *node_alloc_noretry;
3088 	bool node_specific_alloc = false;
3089 
3090 	/* skip gigantic hugepages allocation if hugetlb_cma enabled */
3091 	if (hstate_is_gigantic(h) && hugetlb_cma_size) {
3092 		pr_warn_once("HugeTLB: hugetlb_cma is enabled, skip boot time allocation\n");
3093 		return;
3094 	}
3095 
3096 	/* do node specific alloc */
3097 	for_each_online_node(i) {
3098 		if (h->max_huge_pages_node[i] > 0) {
3099 			hugetlb_hstate_alloc_pages_onenode(h, i);
3100 			node_specific_alloc = true;
3101 		}
3102 	}
3103 
3104 	if (node_specific_alloc)
3105 		return;
3106 
3107 	/* below will do all node balanced alloc */
3108 	if (!hstate_is_gigantic(h)) {
3109 		/*
3110 		 * Bit mask controlling how hard we retry per-node allocations.
3111 		 * Ignore errors as lower level routines can deal with
3112 		 * node_alloc_noretry == NULL.  If this kmalloc fails at boot
3113 		 * time, we are likely in bigger trouble.
3114 		 */
3115 		node_alloc_noretry = kmalloc(sizeof(*node_alloc_noretry),
3116 						GFP_KERNEL);
3117 	} else {
3118 		/* allocations done at boot time */
3119 		node_alloc_noretry = NULL;
3120 	}
3121 
3122 	/* bit mask controlling how hard we retry per-node allocations */
3123 	if (node_alloc_noretry)
3124 		nodes_clear(*node_alloc_noretry);
3125 
3126 	for (i = 0; i < h->max_huge_pages; ++i) {
3127 		if (hstate_is_gigantic(h)) {
3128 			if (!alloc_bootmem_huge_page(h, NUMA_NO_NODE))
3129 				break;
3130 		} else if (!alloc_pool_huge_page(h,
3131 					 &node_states[N_MEMORY],
3132 					 node_alloc_noretry))
3133 			break;
3134 		cond_resched();
3135 	}
3136 	if (i < h->max_huge_pages) {
3137 		char buf[32];
3138 
3139 		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3140 		pr_warn("HugeTLB: allocating %lu of page size %s failed.  Only allocated %lu hugepages.\n",
3141 			h->max_huge_pages, buf, i);
3142 		h->max_huge_pages = i;
3143 	}
3144 	kfree(node_alloc_noretry);
3145 }
3146 
3147 static void __init hugetlb_init_hstates(void)
3148 {
3149 	struct hstate *h, *h2;
3150 
3151 	for_each_hstate(h) {
3152 		if (minimum_order > huge_page_order(h))
3153 			minimum_order = huge_page_order(h);
3154 
3155 		/* oversize hugepages were init'ed in early boot */
3156 		if (!hstate_is_gigantic(h))
3157 			hugetlb_hstate_alloc_pages(h);
3158 
3159 		/*
3160 		 * Set demote order for each hstate.  Note that
3161 		 * h->demote_order is initially 0.
3162 		 * - We can not demote gigantic pages if runtime freeing
3163 		 *   is not supported, so skip this.
3164 		 * - If CMA allocation is possible, we can not demote
3165 		 *   HUGETLB_PAGE_ORDER or smaller size pages.
3166 		 */
3167 		if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3168 			continue;
3169 		if (hugetlb_cma_size && h->order <= HUGETLB_PAGE_ORDER)
3170 			continue;
3171 		for_each_hstate(h2) {
3172 			if (h2 == h)
3173 				continue;
3174 			if (h2->order < h->order &&
3175 			    h2->order > h->demote_order)
3176 				h->demote_order = h2->order;
3177 		}
3178 	}
3179 	VM_BUG_ON(minimum_order == UINT_MAX);
3180 }
3181 
3182 static void __init report_hugepages(void)
3183 {
3184 	struct hstate *h;
3185 
3186 	for_each_hstate(h) {
3187 		char buf[32];
3188 
3189 		string_get_size(huge_page_size(h), 1, STRING_UNITS_2, buf, 32);
3190 		pr_info("HugeTLB registered %s page size, pre-allocated %ld pages\n",
3191 			buf, h->free_huge_pages);
3192 	}
3193 }
3194 
3195 #ifdef CONFIG_HIGHMEM
3196 static void try_to_free_low(struct hstate *h, unsigned long count,
3197 						nodemask_t *nodes_allowed)
3198 {
3199 	int i;
3200 	LIST_HEAD(page_list);
3201 
3202 	lockdep_assert_held(&hugetlb_lock);
3203 	if (hstate_is_gigantic(h))
3204 		return;
3205 
3206 	/*
3207 	 * Collect pages to be freed on a list, and free after dropping lock
3208 	 */
3209 	for_each_node_mask(i, *nodes_allowed) {
3210 		struct page *page, *next;
3211 		struct list_head *freel = &h->hugepage_freelists[i];
3212 		list_for_each_entry_safe(page, next, freel, lru) {
3213 			if (count >= h->nr_huge_pages)
3214 				goto out;
3215 			if (PageHighMem(page))
3216 				continue;
3217 			remove_hugetlb_page(h, page, false);
3218 			list_add(&page->lru, &page_list);
3219 		}
3220 	}
3221 
3222 out:
3223 	spin_unlock_irq(&hugetlb_lock);
3224 	update_and_free_pages_bulk(h, &page_list);
3225 	spin_lock_irq(&hugetlb_lock);
3226 }
3227 #else
3228 static inline void try_to_free_low(struct hstate *h, unsigned long count,
3229 						nodemask_t *nodes_allowed)
3230 {
3231 }
3232 #endif
3233 
3234 /*
3235  * Increment or decrement surplus_huge_pages.  Keep node-specific counters
3236  * balanced by operating on them in a round-robin fashion.
3237  * Returns 1 if an adjustment was made.
3238  */
3239 static int adjust_pool_surplus(struct hstate *h, nodemask_t *nodes_allowed,
3240 				int delta)
3241 {
3242 	int nr_nodes, node;
3243 
3244 	lockdep_assert_held(&hugetlb_lock);
3245 	VM_BUG_ON(delta != -1 && delta != 1);
3246 
3247 	if (delta < 0) {
3248 		for_each_node_mask_to_alloc(h, nr_nodes, node, nodes_allowed) {
3249 			if (h->surplus_huge_pages_node[node])
3250 				goto found;
3251 		}
3252 	} else {
3253 		for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
3254 			if (h->surplus_huge_pages_node[node] <
3255 					h->nr_huge_pages_node[node])
3256 				goto found;
3257 		}
3258 	}
3259 	return 0;
3260 
3261 found:
3262 	h->surplus_huge_pages += delta;
3263 	h->surplus_huge_pages_node[node] += delta;
3264 	return 1;
3265 }
3266 
3267 #define persistent_huge_pages(h) (h->nr_huge_pages - h->surplus_huge_pages)
3268 static int set_max_huge_pages(struct hstate *h, unsigned long count, int nid,
3269 			      nodemask_t *nodes_allowed)
3270 {
3271 	unsigned long min_count, ret;
3272 	struct page *page;
3273 	LIST_HEAD(page_list);
3274 	NODEMASK_ALLOC(nodemask_t, node_alloc_noretry, GFP_KERNEL);
3275 
3276 	/*
3277 	 * Bit mask controlling how hard we retry per-node allocations.
3278 	 * If we can not allocate the bit mask, do not attempt to allocate
3279 	 * the requested huge pages.
3280 	 */
3281 	if (node_alloc_noretry)
3282 		nodes_clear(*node_alloc_noretry);
3283 	else
3284 		return -ENOMEM;
3285 
3286 	/*
3287 	 * resize_lock mutex prevents concurrent adjustments to number of
3288 	 * pages in hstate via the proc/sysfs interfaces.
3289 	 */
3290 	mutex_lock(&h->resize_lock);
3291 	flush_free_hpage_work(h);
3292 	spin_lock_irq(&hugetlb_lock);
3293 
3294 	/*
3295 	 * Check for a node specific request.
3296 	 * Changing node specific huge page count may require a corresponding
3297 	 * change to the global count.  In any case, the passed node mask
3298 	 * (nodes_allowed) will restrict alloc/free to the specified node.
3299 	 */
3300 	if (nid != NUMA_NO_NODE) {
3301 		unsigned long old_count = count;
3302 
3303 		count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
3304 		/*
3305 		 * User may have specified a large count value which caused the
3306 		 * above calculation to overflow.  In this case, they wanted
3307 		 * to allocate as many huge pages as possible.  Set count to
3308 		 * largest possible value to align with their intention.
3309 		 */
3310 		if (count < old_count)
3311 			count = ULONG_MAX;
3312 	}
3313 
3314 	/*
3315 	 * Gigantic pages runtime allocation depend on the capability for large
3316 	 * page range allocation.
3317 	 * If the system does not provide this feature, return an error when
3318 	 * the user tries to allocate gigantic pages but let the user free the
3319 	 * boottime allocated gigantic pages.
3320 	 */
3321 	if (hstate_is_gigantic(h) && !IS_ENABLED(CONFIG_CONTIG_ALLOC)) {
3322 		if (count > persistent_huge_pages(h)) {
3323 			spin_unlock_irq(&hugetlb_lock);
3324 			mutex_unlock(&h->resize_lock);
3325 			NODEMASK_FREE(node_alloc_noretry);
3326 			return -EINVAL;
3327 		}
3328 		/* Fall through to decrease pool */
3329 	}
3330 
3331 	/*
3332 	 * Increase the pool size
3333 	 * First take pages out of surplus state.  Then make up the
3334 	 * remaining difference by allocating fresh huge pages.
3335 	 *
3336 	 * We might race with alloc_surplus_huge_page() here and be unable
3337 	 * to convert a surplus huge page to a normal huge page. That is
3338 	 * not critical, though, it just means the overall size of the
3339 	 * pool might be one hugepage larger than it needs to be, but
3340 	 * within all the constraints specified by the sysctls.
3341 	 */
3342 	while (h->surplus_huge_pages && count > persistent_huge_pages(h)) {
3343 		if (!adjust_pool_surplus(h, nodes_allowed, -1))
3344 			break;
3345 	}
3346 
3347 	while (count > persistent_huge_pages(h)) {
3348 		/*
3349 		 * If this allocation races such that we no longer need the
3350 		 * page, free_huge_page will handle it by freeing the page
3351 		 * and reducing the surplus.
3352 		 */
3353 		spin_unlock_irq(&hugetlb_lock);
3354 
3355 		/* yield cpu to avoid soft lockup */
3356 		cond_resched();
3357 
3358 		ret = alloc_pool_huge_page(h, nodes_allowed,
3359 						node_alloc_noretry);
3360 		spin_lock_irq(&hugetlb_lock);
3361 		if (!ret)
3362 			goto out;
3363 
3364 		/* Bail for signals. Probably ctrl-c from user */
3365 		if (signal_pending(current))
3366 			goto out;
3367 	}
3368 
3369 	/*
3370 	 * Decrease the pool size
3371 	 * First return free pages to the buddy allocator (being careful
3372 	 * to keep enough around to satisfy reservations).  Then place
3373 	 * pages into surplus state as needed so the pool will shrink
3374 	 * to the desired size as pages become free.
3375 	 *
3376 	 * By placing pages into the surplus state independent of the
3377 	 * overcommit value, we are allowing the surplus pool size to
3378 	 * exceed overcommit. There are few sane options here. Since
3379 	 * alloc_surplus_huge_page() is checking the global counter,
3380 	 * though, we'll note that we're not allowed to exceed surplus
3381 	 * and won't grow the pool anywhere else. Not until one of the
3382 	 * sysctls are changed, or the surplus pages go out of use.
3383 	 */
3384 	min_count = h->resv_huge_pages + h->nr_huge_pages - h->free_huge_pages;
3385 	min_count = max(count, min_count);
3386 	try_to_free_low(h, min_count, nodes_allowed);
3387 
3388 	/*
3389 	 * Collect pages to be removed on list without dropping lock
3390 	 */
3391 	while (min_count < persistent_huge_pages(h)) {
3392 		page = remove_pool_huge_page(h, nodes_allowed, 0);
3393 		if (!page)
3394 			break;
3395 
3396 		list_add(&page->lru, &page_list);
3397 	}
3398 	/* free the pages after dropping lock */
3399 	spin_unlock_irq(&hugetlb_lock);
3400 	update_and_free_pages_bulk(h, &page_list);
3401 	flush_free_hpage_work(h);
3402 	spin_lock_irq(&hugetlb_lock);
3403 
3404 	while (count < persistent_huge_pages(h)) {
3405 		if (!adjust_pool_surplus(h, nodes_allowed, 1))
3406 			break;
3407 	}
3408 out:
3409 	h->max_huge_pages = persistent_huge_pages(h);
3410 	spin_unlock_irq(&hugetlb_lock);
3411 	mutex_unlock(&h->resize_lock);
3412 
3413 	NODEMASK_FREE(node_alloc_noretry);
3414 
3415 	return 0;
3416 }
3417 
3418 static int demote_free_huge_page(struct hstate *h, struct page *page)
3419 {
3420 	int i, nid = page_to_nid(page);
3421 	struct hstate *target_hstate;
3422 	int rc = 0;
3423 
3424 	target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order);
3425 
3426 	remove_hugetlb_page_for_demote(h, page, false);
3427 	spin_unlock_irq(&hugetlb_lock);
3428 
3429 	rc = hugetlb_vmemmap_alloc(h, page);
3430 	if (rc) {
3431 		/* Allocation of vmemmmap failed, we can not demote page */
3432 		spin_lock_irq(&hugetlb_lock);
3433 		set_page_refcounted(page);
3434 		add_hugetlb_page(h, page, false);
3435 		return rc;
3436 	}
3437 
3438 	/*
3439 	 * Use destroy_compound_hugetlb_page_for_demote for all huge page
3440 	 * sizes as it will not ref count pages.
3441 	 */
3442 	destroy_compound_hugetlb_page_for_demote(page, huge_page_order(h));
3443 
3444 	/*
3445 	 * Taking target hstate mutex synchronizes with set_max_huge_pages.
3446 	 * Without the mutex, pages added to target hstate could be marked
3447 	 * as surplus.
3448 	 *
3449 	 * Note that we already hold h->resize_lock.  To prevent deadlock,
3450 	 * use the convention of always taking larger size hstate mutex first.
3451 	 */
3452 	mutex_lock(&target_hstate->resize_lock);
3453 	for (i = 0; i < pages_per_huge_page(h);
3454 				i += pages_per_huge_page(target_hstate)) {
3455 		if (hstate_is_gigantic(target_hstate))
3456 			prep_compound_gigantic_page_for_demote(page + i,
3457 							target_hstate->order);
3458 		else
3459 			prep_compound_page(page + i, target_hstate->order);
3460 		set_page_private(page + i, 0);
3461 		set_page_refcounted(page + i);
3462 		prep_new_huge_page(target_hstate, page + i, nid);
3463 		put_page(page + i);
3464 	}
3465 	mutex_unlock(&target_hstate->resize_lock);
3466 
3467 	spin_lock_irq(&hugetlb_lock);
3468 
3469 	/*
3470 	 * Not absolutely necessary, but for consistency update max_huge_pages
3471 	 * based on pool changes for the demoted page.
3472 	 */
3473 	h->max_huge_pages--;
3474 	target_hstate->max_huge_pages += pages_per_huge_page(h);
3475 
3476 	return rc;
3477 }
3478 
3479 static int demote_pool_huge_page(struct hstate *h, nodemask_t *nodes_allowed)
3480 	__must_hold(&hugetlb_lock)
3481 {
3482 	int nr_nodes, node;
3483 	struct page *page;
3484 
3485 	lockdep_assert_held(&hugetlb_lock);
3486 
3487 	/* We should never get here if no demote order */
3488 	if (!h->demote_order) {
3489 		pr_warn("HugeTLB: NULL demote order passed to demote_pool_huge_page.\n");
3490 		return -EINVAL;		/* internal error */
3491 	}
3492 
3493 	for_each_node_mask_to_free(h, nr_nodes, node, nodes_allowed) {
3494 		list_for_each_entry(page, &h->hugepage_freelists[node], lru) {
3495 			if (PageHWPoison(page))
3496 				continue;
3497 
3498 			return demote_free_huge_page(h, page);
3499 		}
3500 	}
3501 
3502 	/*
3503 	 * Only way to get here is if all pages on free lists are poisoned.
3504 	 * Return -EBUSY so that caller will not retry.
3505 	 */
3506 	return -EBUSY;
3507 }
3508 
3509 #define HSTATE_ATTR_RO(_name) \
3510 	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
3511 
3512 #define HSTATE_ATTR_WO(_name) \
3513 	static struct kobj_attribute _name##_attr = __ATTR_WO(_name)
3514 
3515 #define HSTATE_ATTR(_name) \
3516 	static struct kobj_attribute _name##_attr = __ATTR_RW(_name)
3517 
3518 static struct kobject *hugepages_kobj;
3519 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
3520 
3521 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
3522 
3523 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
3524 {
3525 	int i;
3526 
3527 	for (i = 0; i < HUGE_MAX_HSTATE; i++)
3528 		if (hstate_kobjs[i] == kobj) {
3529 			if (nidp)
3530 				*nidp = NUMA_NO_NODE;
3531 			return &hstates[i];
3532 		}
3533 
3534 	return kobj_to_node_hstate(kobj, nidp);
3535 }
3536 
3537 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
3538 					struct kobj_attribute *attr, char *buf)
3539 {
3540 	struct hstate *h;
3541 	unsigned long nr_huge_pages;
3542 	int nid;
3543 
3544 	h = kobj_to_hstate(kobj, &nid);
3545 	if (nid == NUMA_NO_NODE)
3546 		nr_huge_pages = h->nr_huge_pages;
3547 	else
3548 		nr_huge_pages = h->nr_huge_pages_node[nid];
3549 
3550 	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
3551 }
3552 
3553 static ssize_t __nr_hugepages_store_common(bool obey_mempolicy,
3554 					   struct hstate *h, int nid,
3555 					   unsigned long count, size_t len)
3556 {
3557 	int err;
3558 	nodemask_t nodes_allowed, *n_mask;
3559 
3560 	if (hstate_is_gigantic(h) && !gigantic_page_runtime_supported())
3561 		return -EINVAL;
3562 
3563 	if (nid == NUMA_NO_NODE) {
3564 		/*
3565 		 * global hstate attribute
3566 		 */
3567 		if (!(obey_mempolicy &&
3568 				init_nodemask_of_mempolicy(&nodes_allowed)))
3569 			n_mask = &node_states[N_MEMORY];
3570 		else
3571 			n_mask = &nodes_allowed;
3572 	} else {
3573 		/*
3574 		 * Node specific request.  count adjustment happens in
3575 		 * set_max_huge_pages() after acquiring hugetlb_lock.
3576 		 */
3577 		init_nodemask_of_node(&nodes_allowed, nid);
3578 		n_mask = &nodes_allowed;
3579 	}
3580 
3581 	err = set_max_huge_pages(h, count, nid, n_mask);
3582 
3583 	return err ? err : len;
3584 }
3585 
3586 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
3587 					 struct kobject *kobj, const char *buf,
3588 					 size_t len)
3589 {
3590 	struct hstate *h;
3591 	unsigned long count;
3592 	int nid;
3593 	int err;
3594 
3595 	err = kstrtoul(buf, 10, &count);
3596 	if (err)
3597 		return err;
3598 
3599 	h = kobj_to_hstate(kobj, &nid);
3600 	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
3601 }
3602 
3603 static ssize_t nr_hugepages_show(struct kobject *kobj,
3604 				       struct kobj_attribute *attr, char *buf)
3605 {
3606 	return nr_hugepages_show_common(kobj, attr, buf);
3607 }
3608 
3609 static ssize_t nr_hugepages_store(struct kobject *kobj,
3610 	       struct kobj_attribute *attr, const char *buf, size_t len)
3611 {
3612 	return nr_hugepages_store_common(false, kobj, buf, len);
3613 }
3614 HSTATE_ATTR(nr_hugepages);
3615 
3616 #ifdef CONFIG_NUMA
3617 
3618 /*
3619  * hstate attribute for optionally mempolicy-based constraint on persistent
3620  * huge page alloc/free.
3621  */
3622 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
3623 					   struct kobj_attribute *attr,
3624 					   char *buf)
3625 {
3626 	return nr_hugepages_show_common(kobj, attr, buf);
3627 }
3628 
3629 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
3630 	       struct kobj_attribute *attr, const char *buf, size_t len)
3631 {
3632 	return nr_hugepages_store_common(true, kobj, buf, len);
3633 }
3634 HSTATE_ATTR(nr_hugepages_mempolicy);
3635 #endif
3636 
3637 
3638 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
3639 					struct kobj_attribute *attr, char *buf)
3640 {
3641 	struct hstate *h = kobj_to_hstate(kobj, NULL);
3642 	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
3643 }
3644 
3645 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
3646 		struct kobj_attribute *attr, const char *buf, size_t count)
3647 {
3648 	int err;
3649 	unsigned long input;
3650 	struct hstate *h = kobj_to_hstate(kobj, NULL);
3651 
3652 	if (hstate_is_gigantic(h))
3653 		return -EINVAL;
3654 
3655 	err = kstrtoul(buf, 10, &input);
3656 	if (err)
3657 		return err;
3658 
3659 	spin_lock_irq(&hugetlb_lock);
3660 	h->nr_overcommit_huge_pages = input;
3661 	spin_unlock_irq(&hugetlb_lock);
3662 
3663 	return count;
3664 }
3665 HSTATE_ATTR(nr_overcommit_hugepages);
3666 
3667 static ssize_t free_hugepages_show(struct kobject *kobj,
3668 					struct kobj_attribute *attr, char *buf)
3669 {
3670 	struct hstate *h;
3671 	unsigned long free_huge_pages;
3672 	int nid;
3673 
3674 	h = kobj_to_hstate(kobj, &nid);
3675 	if (nid == NUMA_NO_NODE)
3676 		free_huge_pages = h->free_huge_pages;
3677 	else
3678 		free_huge_pages = h->free_huge_pages_node[nid];
3679 
3680 	return sysfs_emit(buf, "%lu\n", free_huge_pages);
3681 }
3682 HSTATE_ATTR_RO(free_hugepages);
3683 
3684 static ssize_t resv_hugepages_show(struct kobject *kobj,
3685 					struct kobj_attribute *attr, char *buf)
3686 {
3687 	struct hstate *h = kobj_to_hstate(kobj, NULL);
3688 	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
3689 }
3690 HSTATE_ATTR_RO(resv_hugepages);
3691 
3692 static ssize_t surplus_hugepages_show(struct kobject *kobj,
3693 					struct kobj_attribute *attr, char *buf)
3694 {
3695 	struct hstate *h;
3696 	unsigned long surplus_huge_pages;
3697 	int nid;
3698 
3699 	h = kobj_to_hstate(kobj, &nid);
3700 	if (nid == NUMA_NO_NODE)
3701 		surplus_huge_pages = h->surplus_huge_pages;
3702 	else
3703 		surplus_huge_pages = h->surplus_huge_pages_node[nid];
3704 
3705 	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
3706 }
3707 HSTATE_ATTR_RO(surplus_hugepages);
3708 
3709 static ssize_t demote_store(struct kobject *kobj,
3710 	       struct kobj_attribute *attr, const char *buf, size_t len)
3711 {
3712 	unsigned long nr_demote;
3713 	unsigned long nr_available;
3714 	nodemask_t nodes_allowed, *n_mask;
3715 	struct hstate *h;
3716 	int err = 0;
3717 	int nid;
3718 
3719 	err = kstrtoul(buf, 10, &nr_demote);
3720 	if (err)
3721 		return err;
3722 	h = kobj_to_hstate(kobj, &nid);
3723 
3724 	if (nid != NUMA_NO_NODE) {
3725 		init_nodemask_of_node(&nodes_allowed, nid);
3726 		n_mask = &nodes_allowed;
3727 	} else {
3728 		n_mask = &node_states[N_MEMORY];
3729 	}
3730 
3731 	/* Synchronize with other sysfs operations modifying huge pages */
3732 	mutex_lock(&h->resize_lock);
3733 	spin_lock_irq(&hugetlb_lock);
3734 
3735 	while (nr_demote) {
3736 		/*
3737 		 * Check for available pages to demote each time thorough the
3738 		 * loop as demote_pool_huge_page will drop hugetlb_lock.
3739 		 */
3740 		if (nid != NUMA_NO_NODE)
3741 			nr_available = h->free_huge_pages_node[nid];
3742 		else
3743 			nr_available = h->free_huge_pages;
3744 		nr_available -= h->resv_huge_pages;
3745 		if (!nr_available)
3746 			break;
3747 
3748 		err = demote_pool_huge_page(h, n_mask);
3749 		if (err)
3750 			break;
3751 
3752 		nr_demote--;
3753 	}
3754 
3755 	spin_unlock_irq(&hugetlb_lock);
3756 	mutex_unlock(&h->resize_lock);
3757 
3758 	if (err)
3759 		return err;
3760 	return len;
3761 }
3762 HSTATE_ATTR_WO(demote);
3763 
3764 static ssize_t demote_size_show(struct kobject *kobj,
3765 					struct kobj_attribute *attr, char *buf)
3766 {
3767 	int nid;
3768 	struct hstate *h = kobj_to_hstate(kobj, &nid);
3769 	unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;
3770 
3771 	return sysfs_emit(buf, "%lukB\n", demote_size);
3772 }
3773 
3774 static ssize_t demote_size_store(struct kobject *kobj,
3775 					struct kobj_attribute *attr,
3776 					const char *buf, size_t count)
3777 {
3778 	struct hstate *h, *demote_hstate;
3779 	unsigned long demote_size;
3780 	unsigned int demote_order;
3781 	int nid;
3782 
3783 	demote_size = (unsigned long)memparse(buf, NULL);
3784 
3785 	demote_hstate = size_to_hstate(demote_size);
3786 	if (!demote_hstate)
3787 		return -EINVAL;
3788 	demote_order = demote_hstate->order;
3789 	if (demote_order < HUGETLB_PAGE_ORDER)
3790 		return -EINVAL;
3791 
3792 	/* demote order must be smaller than hstate order */
3793 	h = kobj_to_hstate(kobj, &nid);
3794 	if (demote_order >= h->order)
3795 		return -EINVAL;
3796 
3797 	/* resize_lock synchronizes access to demote size and writes */
3798 	mutex_lock(&h->resize_lock);
3799 	h->demote_order = demote_order;
3800 	mutex_unlock(&h->resize_lock);
3801 
3802 	return count;
3803 }
3804 HSTATE_ATTR(demote_size);
3805 
3806 static struct attribute *hstate_attrs[] = {
3807 	&nr_hugepages_attr.attr,
3808 	&nr_overcommit_hugepages_attr.attr,
3809 	&free_hugepages_attr.attr,
3810 	&resv_hugepages_attr.attr,
3811 	&surplus_hugepages_attr.attr,
3812 #ifdef CONFIG_NUMA
3813 	&nr_hugepages_mempolicy_attr.attr,
3814 #endif
3815 	NULL,
3816 };
3817 
3818 static const struct attribute_group hstate_attr_group = {
3819 	.attrs = hstate_attrs,
3820 };
3821 
3822 static struct attribute *hstate_demote_attrs[] = {
3823 	&demote_size_attr.attr,
3824 	&demote_attr.attr,
3825 	NULL,
3826 };
3827 
3828 static const struct attribute_group hstate_demote_attr_group = {
3829 	.attrs = hstate_demote_attrs,
3830 };
3831 
3832 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
3833 				    struct kobject **hstate_kobjs,
3834 				    const struct attribute_group *hstate_attr_group)
3835 {
3836 	int retval;
3837 	int hi = hstate_index(h);
3838 
3839 	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
3840 	if (!hstate_kobjs[hi])
3841 		return -ENOMEM;
3842 
3843 	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
3844 	if (retval) {
3845 		kobject_put(hstate_kobjs[hi]);
3846 		hstate_kobjs[hi] = NULL;
3847 	}
3848 
3849 	if (h->demote_order) {
3850 		if (sysfs_create_group(hstate_kobjs[hi],
3851 					&hstate_demote_attr_group))
3852 			pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
3853 	}
3854 
3855 	return retval;
3856 }
3857 
3858 static void __init hugetlb_sysfs_init(void)
3859 {
3860 	struct hstate *h;
3861 	int err;
3862 
3863 	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
3864 	if (!hugepages_kobj)
3865 		return;
3866 
3867 	for_each_hstate(h) {
3868 		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
3869 					 hstate_kobjs, &hstate_attr_group);
3870 		if (err)
3871 			pr_err("HugeTLB: Unable to add hstate %s", h->name);
3872 	}
3873 }
3874 
3875 #ifdef CONFIG_NUMA
3876 
3877 /*
3878  * node_hstate/s - associate per node hstate attributes, via their kobjects,
3879  * with node devices in node_devices[] using a parallel array.  The array
3880  * index of a node device or _hstate == node id.
3881  * This is here to avoid any static dependency of the node device driver, in
3882  * the base kernel, on the hugetlb module.
3883  */
3884 struct node_hstate {
3885 	struct kobject		*hugepages_kobj;
3886 	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
3887 };
3888 static struct node_hstate node_hstates[MAX_NUMNODES];
3889 
3890 /*
3891  * A subset of global hstate attributes for node devices
3892  */
3893 static struct attribute *per_node_hstate_attrs[] = {
3894 	&nr_hugepages_attr.attr,
3895 	&free_hugepages_attr.attr,
3896 	&surplus_hugepages_attr.attr,
3897 	NULL,
3898 };
3899 
3900 static const struct attribute_group per_node_hstate_attr_group = {
3901 	.attrs = per_node_hstate_attrs,
3902 };
3903 
3904 /*
3905  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
3906  * Returns node id via non-NULL nidp.
3907  */
3908 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
3909 {
3910 	int nid;
3911 
3912 	for (nid = 0; nid < nr_node_ids; nid++) {
3913 		struct node_hstate *nhs = &node_hstates[nid];
3914 		int i;
3915 		for (i = 0; i < HUGE_MAX_HSTATE; i++)
3916 			if (nhs->hstate_kobjs[i] == kobj) {
3917 				if (nidp)
3918 					*nidp = nid;
3919 				return &hstates[i];
3920 			}
3921 	}
3922 
3923 	BUG();
3924 	return NULL;
3925 }
3926 
3927 /*
3928  * Unregister hstate attributes from a single node device.
3929  * No-op if no hstate attributes attached.
3930  */
3931 static void hugetlb_unregister_node(struct node *node)
3932 {
3933 	struct hstate *h;
3934 	struct node_hstate *nhs = &node_hstates[node->dev.id];
3935 
3936 	if (!nhs->hugepages_kobj)
3937 		return;		/* no hstate attributes */
3938 
3939 	for_each_hstate(h) {
3940 		int idx = hstate_index(h);
3941 		if (nhs->hstate_kobjs[idx]) {
3942 			kobject_put(nhs->hstate_kobjs[idx]);
3943 			nhs->hstate_kobjs[idx] = NULL;
3944 		}
3945 	}
3946 
3947 	kobject_put(nhs->hugepages_kobj);
3948 	nhs->hugepages_kobj = NULL;
3949 }
3950 
3951 
3952 /*
3953  * Register hstate attributes for a single node device.
3954  * No-op if attributes already registered.
3955  */
3956 static void hugetlb_register_node(struct node *node)
3957 {
3958 	struct hstate *h;
3959 	struct node_hstate *nhs = &node_hstates[node->dev.id];
3960 	int err;
3961 
3962 	if (nhs->hugepages_kobj)
3963 		return;		/* already allocated */
3964 
3965 	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
3966 							&node->dev.kobj);
3967 	if (!nhs->hugepages_kobj)
3968 		return;
3969 
3970 	for_each_hstate(h) {
3971 		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
3972 						nhs->hstate_kobjs,
3973 						&per_node_hstate_attr_group);
3974 		if (err) {
3975 			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
3976 				h->name, node->dev.id);
3977 			hugetlb_unregister_node(node);
3978 			break;
3979 		}
3980 	}
3981 }
3982 
3983 /*
3984  * hugetlb init time:  register hstate attributes for all registered node
3985  * devices of nodes that have memory.  All on-line nodes should have
3986  * registered their associated device by this time.
3987  */
3988 static void __init hugetlb_register_all_nodes(void)
3989 {
3990 	int nid;
3991 
3992 	for_each_node_state(nid, N_MEMORY) {
3993 		struct node *node = node_devices[nid];
3994 		if (node->dev.id == nid)
3995 			hugetlb_register_node(node);
3996 	}
3997 
3998 	/*
3999 	 * Let the node device driver know we're here so it can
4000 	 * [un]register hstate attributes on node hotplug.
4001 	 */
4002 	register_hugetlbfs_with_node(hugetlb_register_node,
4003 				     hugetlb_unregister_node);
4004 }
4005 #else	/* !CONFIG_NUMA */
4006 
4007 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
4008 {
4009 	BUG();
4010 	if (nidp)
4011 		*nidp = -1;
4012 	return NULL;
4013 }
4014 
4015 static void hugetlb_register_all_nodes(void) { }
4016 
4017 #endif
4018 
4019 static int __init hugetlb_init(void)
4020 {
4021 	int i;
4022 
4023 	BUILD_BUG_ON(sizeof_field(struct page, private) * BITS_PER_BYTE <
4024 			__NR_HPAGEFLAGS);
4025 
4026 	if (!hugepages_supported()) {
4027 		if (hugetlb_max_hstate || default_hstate_max_huge_pages)
4028 			pr_warn("HugeTLB: huge pages not supported, ignoring associated command-line parameters\n");
4029 		return 0;
4030 	}
4031 
4032 	/*
4033 	 * Make sure HPAGE_SIZE (HUGETLB_PAGE_ORDER) hstate exists.  Some
4034 	 * architectures depend on setup being done here.
4035 	 */
4036 	hugetlb_add_hstate(HUGETLB_PAGE_ORDER);
4037 	if (!parsed_default_hugepagesz) {
4038 		/*
4039 		 * If we did not parse a default huge page size, set
4040 		 * default_hstate_idx to HPAGE_SIZE hstate. And, if the
4041 		 * number of huge pages for this default size was implicitly
4042 		 * specified, set that here as well.
4043 		 * Note that the implicit setting will overwrite an explicit
4044 		 * setting.  A warning will be printed in this case.
4045 		 */
4046 		default_hstate_idx = hstate_index(size_to_hstate(HPAGE_SIZE));
4047 		if (default_hstate_max_huge_pages) {
4048 			if (default_hstate.max_huge_pages) {
4049 				char buf[32];
4050 
4051 				string_get_size(huge_page_size(&default_hstate),
4052 					1, STRING_UNITS_2, buf, 32);
4053 				pr_warn("HugeTLB: Ignoring hugepages=%lu associated with %s page size\n",
4054 					default_hstate.max_huge_pages, buf);
4055 				pr_warn("HugeTLB: Using hugepages=%lu for number of default huge pages\n",
4056 					default_hstate_max_huge_pages);
4057 			}
4058 			default_hstate.max_huge_pages =
4059 				default_hstate_max_huge_pages;
4060 
4061 			for_each_online_node(i)
4062 				default_hstate.max_huge_pages_node[i] =
4063 					default_hugepages_in_node[i];
4064 		}
4065 	}
4066 
4067 	hugetlb_cma_check();
4068 	hugetlb_init_hstates();
4069 	gather_bootmem_prealloc();
4070 	report_hugepages();
4071 
4072 	hugetlb_sysfs_init();
4073 	hugetlb_register_all_nodes();
4074 	hugetlb_cgroup_file_init();
4075 
4076 #ifdef CONFIG_SMP
4077 	num_fault_mutexes = roundup_pow_of_two(8 * num_possible_cpus());
4078 #else
4079 	num_fault_mutexes = 1;
4080 #endif
4081 	hugetlb_fault_mutex_table =
4082 		kmalloc_array(num_fault_mutexes, sizeof(struct mutex),
4083 			      GFP_KERNEL);
4084 	BUG_ON(!hugetlb_fault_mutex_table);
4085 
4086 	for (i = 0; i < num_fault_mutexes; i++)
4087 		mutex_init(&hugetlb_fault_mutex_table[i]);
4088 	return 0;
4089 }
4090 subsys_initcall(hugetlb_init);
4091 
4092 /* Overwritten by architectures with more huge page sizes */
4093 bool __init __attribute((weak)) arch_hugetlb_valid_size(unsigned long size)
4094 {
4095 	return size == HPAGE_SIZE;
4096 }
4097 
4098 void __init hugetlb_add_hstate(unsigned int order)
4099 {
4100 	struct hstate *h;
4101 	unsigned long i;
4102 
4103 	if (size_to_hstate(PAGE_SIZE << order)) {
4104 		return;
4105 	}
4106 	BUG_ON(hugetlb_max_hstate >= HUGE_MAX_HSTATE);
4107 	BUG_ON(order == 0);
4108 	h = &hstates[hugetlb_max_hstate++];
4109 	mutex_init(&h->resize_lock);
4110 	h->order = order;
4111 	h->mask = ~(huge_page_size(h) - 1);
4112 	for (i = 0; i < MAX_NUMNODES; ++i)
4113 		INIT_LIST_HEAD(&h->hugepage_freelists[i]);
4114 	INIT_LIST_HEAD(&h->hugepage_activelist);
4115 	h->next_nid_to_alloc = first_memory_node;
4116 	h->next_nid_to_free = first_memory_node;
4117 	snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
4118 					huge_page_size(h)/1024);
4119 	hugetlb_vmemmap_init(h);
4120 
4121 	parsed_hstate = h;
4122 }
4123 
4124 bool __init __weak hugetlb_node_alloc_supported(void)
4125 {
4126 	return true;
4127 }
4128 
4129 static void __init hugepages_clear_pages_in_node(void)
4130 {
4131 	if (!hugetlb_max_hstate) {
4132 		default_hstate_max_huge_pages = 0;
4133 		memset(default_hugepages_in_node, 0,
4134 			MAX_NUMNODES * sizeof(unsigned int));
4135 	} else {
4136 		parsed_hstate->max_huge_pages = 0;
4137 		memset(parsed_hstate->max_huge_pages_node, 0,
4138 			MAX_NUMNODES * sizeof(unsigned int));
4139 	}
4140 }
4141 
4142 /*
4143  * hugepages command line processing
4144  * hugepages normally follows a valid hugepagsz or default_hugepagsz
4145  * specification.  If not, ignore the hugepages value.  hugepages can also
4146  * be the first huge page command line  option in which case it implicitly
4147  * specifies the number of huge pages for the default size.
4148  */
4149 static int __init hugepages_setup(char *s)
4150 {
4151 	unsigned long *mhp;
4152 	static unsigned long *last_mhp;
4153 	int node = NUMA_NO_NODE;
4154 	int count;
4155 	unsigned long tmp;
4156 	char *p = s;
4157 
4158 	if (!parsed_valid_hugepagesz) {
4159 		pr_warn("HugeTLB: hugepages=%s does not follow a valid hugepagesz, ignoring\n", s);
4160 		parsed_valid_hugepagesz = true;
4161 		return 1;
4162 	}
4163 
4164 	/*
4165 	 * !hugetlb_max_hstate means we haven't parsed a hugepagesz= parameter
4166 	 * yet, so this hugepages= parameter goes to the "default hstate".
4167 	 * Otherwise, it goes with the previously parsed hugepagesz or
4168 	 * default_hugepagesz.
4169 	 */
4170 	else if (!hugetlb_max_hstate)
4171 		mhp = &default_hstate_max_huge_pages;
4172 	else
4173 		mhp = &parsed_hstate->max_huge_pages;
4174 
4175 	if (mhp == last_mhp) {
4176 		pr_warn("HugeTLB: hugepages= specified twice without interleaving hugepagesz=, ignoring hugepages=%s\n", s);
4177 		return 1;
4178 	}
4179 
4180 	while (*p) {
4181 		count = 0;
4182 		if (sscanf(p, "%lu%n", &tmp, &count) != 1)
4183 			goto invalid;
4184 		/* Parameter is node format */
4185 		if (p[count] == ':') {
4186 			if (!hugetlb_node_alloc_supported()) {
4187 				pr_warn("HugeTLB: architecture can't support node specific alloc, ignoring!\n");
4188 				return 1;
4189 			}
4190 			if (tmp >= MAX_NUMNODES || !node_online(tmp))
4191 				goto invalid;
4192 			node = array_index_nospec(tmp, MAX_NUMNODES);
4193 			p += count + 1;
4194 			/* Parse hugepages */
4195 			if (sscanf(p, "%lu%n", &tmp, &count) != 1)
4196 				goto invalid;
4197 			if (!hugetlb_max_hstate)
4198 				default_hugepages_in_node[node] = tmp;
4199 			else
4200 				parsed_hstate->max_huge_pages_node[node] = tmp;
4201 			*mhp += tmp;
4202 			/* Go to parse next node*/
4203 			if (p[count] == ',')
4204 				p += count + 1;
4205 			else
4206 				break;
4207 		} else {
4208 			if (p != s)
4209 				goto invalid;
4210 			*mhp = tmp;
4211 			break;
4212 		}
4213 	}
4214 
4215 	/*
4216 	 * Global state is always initialized later in hugetlb_init.
4217 	 * But we need to allocate gigantic hstates here early to still
4218 	 * use the bootmem allocator.
4219 	 */
4220 	if (hugetlb_max_hstate && hstate_is_gigantic(parsed_hstate))
4221 		hugetlb_hstate_alloc_pages(parsed_hstate);
4222 
4223 	last_mhp = mhp;
4224 
4225 	return 1;
4226 
4227 invalid:
4228 	pr_warn("HugeTLB: Invalid hugepages parameter %s\n", p);
4229 	hugepages_clear_pages_in_node();
4230 	return 1;
4231 }
4232 __setup("hugepages=", hugepages_setup);
4233 
4234 /*
4235  * hugepagesz command line processing
4236  * A specific huge page size can only be specified once with hugepagesz.
4237  * hugepagesz is followed by hugepages on the command line.  The global
4238  * variable 'parsed_valid_hugepagesz' is used to determine if prior
4239  * hugepagesz argument was valid.
4240  */
4241 static int __init hugepagesz_setup(char *s)
4242 {
4243 	unsigned long size;
4244 	struct hstate *h;
4245 
4246 	parsed_valid_hugepagesz = false;
4247 	size = (unsigned long)memparse(s, NULL);
4248 
4249 	if (!arch_hugetlb_valid_size(size)) {
4250 		pr_err("HugeTLB: unsupported hugepagesz=%s\n", s);
4251 		return 1;
4252 	}
4253 
4254 	h = size_to_hstate(size);
4255 	if (h) {
4256 		/*
4257 		 * hstate for this size already exists.  This is normally
4258 		 * an error, but is allowed if the existing hstate is the
4259 		 * default hstate.  More specifically, it is only allowed if
4260 		 * the number of huge pages for the default hstate was not
4261 		 * previously specified.
4262 		 */
4263 		if (!parsed_default_hugepagesz ||  h != &default_hstate ||
4264 		    default_hstate.max_huge_pages) {
4265 			pr_warn("HugeTLB: hugepagesz=%s specified twice, ignoring\n", s);
4266 			return 1;
4267 		}
4268 
4269 		/*
4270 		 * No need to call hugetlb_add_hstate() as hstate already
4271 		 * exists.  But, do set parsed_hstate so that a following
4272 		 * hugepages= parameter will be applied to this hstate.
4273 		 */
4274 		parsed_hstate = h;
4275 		parsed_valid_hugepagesz = true;
4276 		return 1;
4277 	}
4278 
4279 	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4280 	parsed_valid_hugepagesz = true;
4281 	return 1;
4282 }
4283 __setup("hugepagesz=", hugepagesz_setup);
4284 
4285 /*
4286  * default_hugepagesz command line input
4287  * Only one instance of default_hugepagesz allowed on command line.
4288  */
4289 static int __init default_hugepagesz_setup(char *s)
4290 {
4291 	unsigned long size;
4292 	int i;
4293 
4294 	parsed_valid_hugepagesz = false;
4295 	if (parsed_default_hugepagesz) {
4296 		pr_err("HugeTLB: default_hugepagesz previously specified, ignoring %s\n", s);
4297 		return 1;
4298 	}
4299 
4300 	size = (unsigned long)memparse(s, NULL);
4301 
4302 	if (!arch_hugetlb_valid_size(size)) {
4303 		pr_err("HugeTLB: unsupported default_hugepagesz=%s\n", s);
4304 		return 1;
4305 	}
4306 
4307 	hugetlb_add_hstate(ilog2(size) - PAGE_SHIFT);
4308 	parsed_valid_hugepagesz = true;
4309 	parsed_default_hugepagesz = true;
4310 	default_hstate_idx = hstate_index(size_to_hstate(size));
4311 
4312 	/*
4313 	 * The number of default huge pages (for this size) could have been
4314 	 * specified as the first hugetlb parameter: hugepages=X.  If so,
4315 	 * then default_hstate_max_huge_pages is set.  If the default huge
4316 	 * page size is gigantic (>= MAX_ORDER), then the pages must be
4317 	 * allocated here from bootmem allocator.
4318 	 */
4319 	if (default_hstate_max_huge_pages) {
4320 		default_hstate.max_huge_pages = default_hstate_max_huge_pages;
4321 		for_each_online_node(i)
4322 			default_hstate.max_huge_pages_node[i] =
4323 				default_hugepages_in_node[i];
4324 		if (hstate_is_gigantic(&default_hstate))
4325 			hugetlb_hstate_alloc_pages(&default_hstate);
4326 		default_hstate_max_huge_pages = 0;
4327 	}
4328 
4329 	return 1;
4330 }
4331 __setup("default_hugepagesz=", default_hugepagesz_setup);
4332 
4333 static unsigned int allowed_mems_nr(struct hstate *h)
4334 {
4335 	int node;
4336 	unsigned int nr = 0;
4337 	nodemask_t *mpol_allowed;
4338 	unsigned int *array = h->free_huge_pages_node;
4339 	gfp_t gfp_mask = htlb_alloc_mask(h);
4340 
4341 	mpol_allowed = policy_nodemask_current(gfp_mask);
4342 
4343 	for_each_node_mask(node, cpuset_current_mems_allowed) {
4344 		if (!mpol_allowed || node_isset(node, *mpol_allowed))
4345 			nr += array[node];
4346 	}
4347 
4348 	return nr;
4349 }
4350 
4351 #ifdef CONFIG_SYSCTL
4352 static int proc_hugetlb_doulongvec_minmax(struct ctl_table *table, int write,
4353 					  void *buffer, size_t *length,
4354 					  loff_t *ppos, unsigned long *out)
4355 {
4356 	struct ctl_table dup_table;
4357 
4358 	/*
4359 	 * In order to avoid races with __do_proc_doulongvec_minmax(), we
4360 	 * can duplicate the @table and alter the duplicate of it.
4361 	 */
4362 	dup_table = *table;
4363 	dup_table.data = out;
4364 
4365 	return proc_doulongvec_minmax(&dup_table, write, buffer, length, ppos);
4366 }
4367 
4368 static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
4369 			 struct ctl_table *table, int write,
4370 			 void *buffer, size_t *length, loff_t *ppos)
4371 {
4372 	struct hstate *h = &default_hstate;
4373 	unsigned long tmp = h->max_huge_pages;
4374 	int ret;
4375 
4376 	if (!hugepages_supported())
4377 		return -EOPNOTSUPP;
4378 
4379 	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
4380 					     &tmp);
4381 	if (ret)
4382 		goto out;
4383 
4384 	if (write)
4385 		ret = __nr_hugepages_store_common(obey_mempolicy, h,
4386 						  NUMA_NO_NODE, tmp, *length);
4387 out:
4388 	return ret;
4389 }
4390 
4391 int hugetlb_sysctl_handler(struct ctl_table *table, int write,
4392 			  void *buffer, size_t *length, loff_t *ppos)
4393 {
4394 
4395 	return hugetlb_sysctl_handler_common(false, table, write,
4396 							buffer, length, ppos);
4397 }
4398 
4399 #ifdef CONFIG_NUMA
4400 int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
4401 			  void *buffer, size_t *length, loff_t *ppos)
4402 {
4403 	return hugetlb_sysctl_handler_common(true, table, write,
4404 							buffer, length, ppos);
4405 }
4406 #endif /* CONFIG_NUMA */
4407 
4408 int hugetlb_overcommit_handler(struct ctl_table *table, int write,
4409 		void *buffer, size_t *length, loff_t *ppos)
4410 {
4411 	struct hstate *h = &default_hstate;
4412 	unsigned long tmp;
4413 	int ret;
4414 
4415 	if (!hugepages_supported())
4416 		return -EOPNOTSUPP;
4417 
4418 	tmp = h->nr_overcommit_huge_pages;
4419 
4420 	if (write && hstate_is_gigantic(h))
4421 		return -EINVAL;
4422 
4423 	ret = proc_hugetlb_doulongvec_minmax(table, write, buffer, length, ppos,
4424 					     &tmp);
4425 	if (ret)
4426 		goto out;
4427 
4428 	if (write) {
4429 		spin_lock_irq(&hugetlb_lock);
4430 		h->nr_overcommit_huge_pages = tmp;
4431 		spin_unlock_irq(&hugetlb_lock);
4432 	}
4433 out:
4434 	return ret;
4435 }
4436 
4437 #endif /* CONFIG_SYSCTL */
4438 
4439 void hugetlb_report_meminfo(struct seq_file *m)
4440 {
4441 	struct hstate *h;
4442 	unsigned long total = 0;
4443 
4444 	if (!hugepages_supported())
4445 		return;
4446 
4447 	for_each_hstate(h) {
4448 		unsigned long count = h->nr_huge_pages;
4449 
4450 		total += huge_page_size(h) * count;
4451 
4452 		if (h == &default_hstate)
4453 			seq_printf(m,
4454 				   "HugePages_Total:   %5lu\n"
4455 				   "HugePages_Free:    %5lu\n"
4456 				   "HugePages_Rsvd:    %5lu\n"
4457 				   "HugePages_Surp:    %5lu\n"
4458 				   "Hugepagesize:   %8lu kB\n",
4459 				   count,
4460 				   h->free_huge_pages,
4461 				   h->resv_huge_pages,
4462 				   h->surplus_huge_pages,
4463 				   huge_page_size(h) / SZ_1K);
4464 	}
4465 
4466 	seq_printf(m, "Hugetlb:        %8lu kB\n", total / SZ_1K);
4467 }
4468 
4469 int hugetlb_report_node_meminfo(char *buf, int len, int nid)
4470 {
4471 	struct hstate *h = &default_hstate;
4472 
4473 	if (!hugepages_supported())
4474 		return 0;
4475 
4476 	return sysfs_emit_at(buf, len,
4477 			     "Node %d HugePages_Total: %5u\n"
4478 			     "Node %d HugePages_Free:  %5u\n"
4479 			     "Node %d HugePages_Surp:  %5u\n",
4480 			     nid, h->nr_huge_pages_node[nid],
4481 			     nid, h->free_huge_pages_node[nid],
4482 			     nid, h->surplus_huge_pages_node[nid]);
4483 }
4484 
4485 void hugetlb_show_meminfo(void)
4486 {
4487 	struct hstate *h;
4488 	int nid;
4489 
4490 	if (!hugepages_supported())
4491 		return;
4492 
4493 	for_each_node_state(nid, N_MEMORY)
4494 		for_each_hstate(h)
4495 			pr_info("Node %d hugepages_total=%u hugepages_free=%u hugepages_surp=%u hugepages_size=%lukB\n",
4496 				nid,
4497 				h->nr_huge_pages_node[nid],
4498 				h->free_huge_pages_node[nid],
4499 				h->surplus_huge_pages_node[nid],
4500 				huge_page_size(h) / SZ_1K);
4501 }
4502 
4503 void hugetlb_report_usage(struct seq_file *m, struct mm_struct *mm)
4504 {
4505 	seq_printf(m, "HugetlbPages:\t%8lu kB\n",
4506 		   atomic_long_read(&mm->hugetlb_usage) << (PAGE_SHIFT - 10));
4507 }
4508 
4509 /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
4510 unsigned long hugetlb_total_pages(void)
4511 {
4512 	struct hstate *h;
4513 	unsigned long nr_total_pages = 0;
4514 
4515 	for_each_hstate(h)
4516 		nr_total_pages += h->nr_huge_pages * pages_per_huge_page(h);
4517 	return nr_total_pages;
4518 }
4519 
4520 static int hugetlb_acct_memory(struct hstate *h, long delta)
4521 {
4522 	int ret = -ENOMEM;
4523 
4524 	if (!delta)
4525 		return 0;
4526 
4527 	spin_lock_irq(&hugetlb_lock);
4528 	/*
4529 	 * When cpuset is configured, it breaks the strict hugetlb page
4530 	 * reservation as the accounting is done on a global variable. Such
4531 	 * reservation is completely rubbish in the presence of cpuset because
4532 	 * the reservation is not checked against page availability for the
4533 	 * current cpuset. Application can still potentially OOM'ed by kernel
4534 	 * with lack of free htlb page in cpuset that the task is in.
4535 	 * Attempt to enforce strict accounting with cpuset is almost
4536 	 * impossible (or too ugly) because cpuset is too fluid that
4537 	 * task or memory node can be dynamically moved between cpusets.
4538 	 *
4539 	 * The change of semantics for shared hugetlb mapping with cpuset is
4540 	 * undesirable. However, in order to preserve some of the semantics,
4541 	 * we fall back to check against current free page availability as
4542 	 * a best attempt and hopefully to minimize the impact of changing
4543 	 * semantics that cpuset has.
4544 	 *
4545 	 * Apart from cpuset, we also have memory policy mechanism that
4546 	 * also determines from which node the kernel will allocate memory
4547 	 * in a NUMA system. So similar to cpuset, we also should consider
4548 	 * the memory policy of the current task. Similar to the description
4549 	 * above.
4550 	 */
4551 	if (delta > 0) {
4552 		if (gather_surplus_pages(h, delta) < 0)
4553 			goto out;
4554 
4555 		if (delta > allowed_mems_nr(h)) {
4556 			return_unused_surplus_pages(h, delta);
4557 			goto out;
4558 		}
4559 	}
4560 
4561 	ret = 0;
4562 	if (delta < 0)
4563 		return_unused_surplus_pages(h, (unsigned long) -delta);
4564 
4565 out:
4566 	spin_unlock_irq(&hugetlb_lock);
4567 	return ret;
4568 }
4569 
4570 static void hugetlb_vm_op_open(struct vm_area_struct *vma)
4571 {
4572 	struct resv_map *resv = vma_resv_map(vma);
4573 
4574 	/*
4575 	 * This new VMA should share its siblings reservation map if present.
4576 	 * The VMA will only ever have a valid reservation map pointer where
4577 	 * it is being copied for another still existing VMA.  As that VMA
4578 	 * has a reference to the reservation map it cannot disappear until
4579 	 * after this open call completes.  It is therefore safe to take a
4580 	 * new reference here without additional locking.
4581 	 */
4582 	if (resv && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
4583 		resv_map_dup_hugetlb_cgroup_uncharge_info(resv);
4584 		kref_get(&resv->refs);
4585 	}
4586 }
4587 
4588 static void hugetlb_vm_op_close(struct vm_area_struct *vma)
4589 {
4590 	struct hstate *h = hstate_vma(vma);
4591 	struct resv_map *resv = vma_resv_map(vma);
4592 	struct hugepage_subpool *spool = subpool_vma(vma);
4593 	unsigned long reserve, start, end;
4594 	long gbl_reserve;
4595 
4596 	if (!resv || !is_vma_resv_set(vma, HPAGE_RESV_OWNER))
4597 		return;
4598 
4599 	start = vma_hugecache_offset(h, vma, vma->vm_start);
4600 	end = vma_hugecache_offset(h, vma, vma->vm_end);
4601 
4602 	reserve = (end - start) - region_count(resv, start, end);
4603 	hugetlb_cgroup_uncharge_counter(resv, start, end);
4604 	if (reserve) {
4605 		/*
4606 		 * Decrement reserve counts.  The global reserve count may be
4607 		 * adjusted if the subpool has a minimum size.
4608 		 */
4609 		gbl_reserve = hugepage_subpool_put_pages(spool, reserve);
4610 		hugetlb_acct_memory(h, -gbl_reserve);
4611 	}
4612 
4613 	kref_put(&resv->refs, resv_map_release);
4614 }
4615 
4616 static int hugetlb_vm_op_split(struct vm_area_struct *vma, unsigned long addr)
4617 {
4618 	if (addr & ~(huge_page_mask(hstate_vma(vma))))
4619 		return -EINVAL;
4620 	return 0;
4621 }
4622 
4623 static unsigned long hugetlb_vm_op_pagesize(struct vm_area_struct *vma)
4624 {
4625 	return huge_page_size(hstate_vma(vma));
4626 }
4627 
4628 /*
4629  * We cannot handle pagefaults against hugetlb pages at all.  They cause
4630  * handle_mm_fault() to try to instantiate regular-sized pages in the
4631  * hugepage VMA.  do_page_fault() is supposed to trap this, so BUG is we get
4632  * this far.
4633  */
4634 static vm_fault_t hugetlb_vm_op_fault(struct vm_fault *vmf)
4635 {
4636 	BUG();
4637 	return 0;
4638 }
4639 
4640 /*
4641  * When a new function is introduced to vm_operations_struct and added
4642  * to hugetlb_vm_ops, please consider adding the function to shm_vm_ops.
4643  * This is because under System V memory model, mappings created via
4644  * shmget/shmat with "huge page" specified are backed by hugetlbfs files,
4645  * their original vm_ops are overwritten with shm_vm_ops.
4646  */
4647 const struct vm_operations_struct hugetlb_vm_ops = {
4648 	.fault = hugetlb_vm_op_fault,
4649 	.open = hugetlb_vm_op_open,
4650 	.close = hugetlb_vm_op_close,
4651 	.may_split = hugetlb_vm_op_split,
4652 	.pagesize = hugetlb_vm_op_pagesize,
4653 };
4654 
4655 static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
4656 				int writable)
4657 {
4658 	pte_t entry;
4659 	unsigned int shift = huge_page_shift(hstate_vma(vma));
4660 
4661 	if (writable) {
4662 		entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
4663 					 vma->vm_page_prot)));
4664 	} else {
4665 		entry = huge_pte_wrprotect(mk_huge_pte(page,
4666 					   vma->vm_page_prot));
4667 	}
4668 	entry = pte_mkyoung(entry);
4669 	entry = arch_make_huge_pte(entry, shift, vma->vm_flags);
4670 
4671 	return entry;
4672 }
4673 
4674 static void set_huge_ptep_writable(struct vm_area_struct *vma,
4675 				   unsigned long address, pte_t *ptep)
4676 {
4677 	pte_t entry;
4678 
4679 	entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
4680 	if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
4681 		update_mmu_cache(vma, address, ptep);
4682 }
4683 
4684 bool is_hugetlb_entry_migration(pte_t pte)
4685 {
4686 	swp_entry_t swp;
4687 
4688 	if (huge_pte_none(pte) || pte_present(pte))
4689 		return false;
4690 	swp = pte_to_swp_entry(pte);
4691 	if (is_migration_entry(swp))
4692 		return true;
4693 	else
4694 		return false;
4695 }
4696 
4697 static bool is_hugetlb_entry_hwpoisoned(pte_t pte)
4698 {
4699 	swp_entry_t swp;
4700 
4701 	if (huge_pte_none(pte) || pte_present(pte))
4702 		return false;
4703 	swp = pte_to_swp_entry(pte);
4704 	if (is_hwpoison_entry(swp))
4705 		return true;
4706 	else
4707 		return false;
4708 }
4709 
4710 static void
4711 hugetlb_install_page(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr,
4712 		     struct page *new_page)
4713 {
4714 	__SetPageUptodate(new_page);
4715 	hugepage_add_new_anon_rmap(new_page, vma, addr);
4716 	set_huge_pte_at(vma->vm_mm, addr, ptep, make_huge_pte(vma, new_page, 1));
4717 	hugetlb_count_add(pages_per_huge_page(hstate_vma(vma)), vma->vm_mm);
4718 	ClearHPageRestoreReserve(new_page);
4719 	SetHPageMigratable(new_page);
4720 }
4721 
4722 int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
4723 			    struct vm_area_struct *dst_vma,
4724 			    struct vm_area_struct *src_vma)
4725 {
4726 	pte_t *src_pte, *dst_pte, entry, dst_entry;
4727 	struct page *ptepage;
4728 	unsigned long addr;
4729 	bool cow = is_cow_mapping(src_vma->vm_flags);
4730 	struct hstate *h = hstate_vma(src_vma);
4731 	unsigned long sz = huge_page_size(h);
4732 	unsigned long npages = pages_per_huge_page(h);
4733 	struct address_space *mapping = src_vma->vm_file->f_mapping;
4734 	struct mmu_notifier_range range;
4735 	int ret = 0;
4736 
4737 	if (cow) {
4738 		mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src_vma, src,
4739 					src_vma->vm_start,
4740 					src_vma->vm_end);
4741 		mmu_notifier_invalidate_range_start(&range);
4742 		mmap_assert_write_locked(src);
4743 		raw_write_seqcount_begin(&src->write_protect_seq);
4744 	} else {
4745 		/*
4746 		 * For shared mappings i_mmap_rwsem must be held to call
4747 		 * huge_pte_alloc, otherwise the returned ptep could go
4748 		 * away if part of a shared pmd and another thread calls
4749 		 * huge_pmd_unshare.
4750 		 */
4751 		i_mmap_lock_read(mapping);
4752 	}
4753 
4754 	for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
4755 		spinlock_t *src_ptl, *dst_ptl;
4756 		src_pte = huge_pte_offset(src, addr, sz);
4757 		if (!src_pte)
4758 			continue;
4759 		dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
4760 		if (!dst_pte) {
4761 			ret = -ENOMEM;
4762 			break;
4763 		}
4764 
4765 		/*
4766 		 * If the pagetables are shared don't copy or take references.
4767 		 * dst_pte == src_pte is the common case of src/dest sharing.
4768 		 *
4769 		 * However, src could have 'unshared' and dst shares with
4770 		 * another vma.  If dst_pte !none, this implies sharing.
4771 		 * Check here before taking page table lock, and once again
4772 		 * after taking the lock below.
4773 		 */
4774 		dst_entry = huge_ptep_get(dst_pte);
4775 		if ((dst_pte == src_pte) || !huge_pte_none(dst_entry))
4776 			continue;
4777 
4778 		dst_ptl = huge_pte_lock(h, dst, dst_pte);
4779 		src_ptl = huge_pte_lockptr(h, src, src_pte);
4780 		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4781 		entry = huge_ptep_get(src_pte);
4782 		dst_entry = huge_ptep_get(dst_pte);
4783 again:
4784 		if (huge_pte_none(entry) || !huge_pte_none(dst_entry)) {
4785 			/*
4786 			 * Skip if src entry none.  Also, skip in the
4787 			 * unlikely case dst entry !none as this implies
4788 			 * sharing with another vma.
4789 			 */
4790 			;
4791 		} else if (unlikely(is_hugetlb_entry_migration(entry) ||
4792 				    is_hugetlb_entry_hwpoisoned(entry))) {
4793 			swp_entry_t swp_entry = pte_to_swp_entry(entry);
4794 			bool uffd_wp = huge_pte_uffd_wp(entry);
4795 
4796 			if (!is_readable_migration_entry(swp_entry) && cow) {
4797 				/*
4798 				 * COW mappings require pages in both
4799 				 * parent and child to be set to read.
4800 				 */
4801 				swp_entry = make_readable_migration_entry(
4802 							swp_offset(swp_entry));
4803 				entry = swp_entry_to_pte(swp_entry);
4804 				if (userfaultfd_wp(src_vma) && uffd_wp)
4805 					entry = huge_pte_mkuffd_wp(entry);
4806 				set_huge_swap_pte_at(src, addr, src_pte,
4807 						     entry, sz);
4808 			}
4809 			if (!userfaultfd_wp(dst_vma) && uffd_wp)
4810 				entry = huge_pte_clear_uffd_wp(entry);
4811 			set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz);
4812 		} else if (unlikely(is_pte_marker(entry))) {
4813 			/*
4814 			 * We copy the pte marker only if the dst vma has
4815 			 * uffd-wp enabled.
4816 			 */
4817 			if (userfaultfd_wp(dst_vma))
4818 				set_huge_pte_at(dst, addr, dst_pte, entry);
4819 		} else {
4820 			entry = huge_ptep_get(src_pte);
4821 			ptepage = pte_page(entry);
4822 			get_page(ptepage);
4823 
4824 			/*
4825 			 * Failing to duplicate the anon rmap is a rare case
4826 			 * where we see pinned hugetlb pages while they're
4827 			 * prone to COW. We need to do the COW earlier during
4828 			 * fork.
4829 			 *
4830 			 * When pre-allocating the page or copying data, we
4831 			 * need to be without the pgtable locks since we could
4832 			 * sleep during the process.
4833 			 */
4834 			if (!PageAnon(ptepage)) {
4835 				page_dup_file_rmap(ptepage, true);
4836 			} else if (page_try_dup_anon_rmap(ptepage, true,
4837 							  src_vma)) {
4838 				pte_t src_pte_old = entry;
4839 				struct page *new;
4840 
4841 				spin_unlock(src_ptl);
4842 				spin_unlock(dst_ptl);
4843 				/* Do not use reserve as it's private owned */
4844 				new = alloc_huge_page(dst_vma, addr, 1);
4845 				if (IS_ERR(new)) {
4846 					put_page(ptepage);
4847 					ret = PTR_ERR(new);
4848 					break;
4849 				}
4850 				copy_user_huge_page(new, ptepage, addr, dst_vma,
4851 						    npages);
4852 				put_page(ptepage);
4853 
4854 				/* Install the new huge page if src pte stable */
4855 				dst_ptl = huge_pte_lock(h, dst, dst_pte);
4856 				src_ptl = huge_pte_lockptr(h, src, src_pte);
4857 				spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4858 				entry = huge_ptep_get(src_pte);
4859 				if (!pte_same(src_pte_old, entry)) {
4860 					restore_reserve_on_error(h, dst_vma, addr,
4861 								new);
4862 					put_page(new);
4863 					/* dst_entry won't change as in child */
4864 					goto again;
4865 				}
4866 				hugetlb_install_page(dst_vma, dst_pte, addr, new);
4867 				spin_unlock(src_ptl);
4868 				spin_unlock(dst_ptl);
4869 				continue;
4870 			}
4871 
4872 			if (cow) {
4873 				/*
4874 				 * No need to notify as we are downgrading page
4875 				 * table protection not changing it to point
4876 				 * to a new page.
4877 				 *
4878 				 * See Documentation/vm/mmu_notifier.rst
4879 				 */
4880 				huge_ptep_set_wrprotect(src, addr, src_pte);
4881 				entry = huge_pte_wrprotect(entry);
4882 			}
4883 
4884 			set_huge_pte_at(dst, addr, dst_pte, entry);
4885 			hugetlb_count_add(npages, dst);
4886 		}
4887 		spin_unlock(src_ptl);
4888 		spin_unlock(dst_ptl);
4889 	}
4890 
4891 	if (cow) {
4892 		raw_write_seqcount_end(&src->write_protect_seq);
4893 		mmu_notifier_invalidate_range_end(&range);
4894 	} else {
4895 		i_mmap_unlock_read(mapping);
4896 	}
4897 
4898 	return ret;
4899 }
4900 
4901 static void move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr,
4902 			  unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte)
4903 {
4904 	struct hstate *h = hstate_vma(vma);
4905 	struct mm_struct *mm = vma->vm_mm;
4906 	spinlock_t *src_ptl, *dst_ptl;
4907 	pte_t pte;
4908 
4909 	dst_ptl = huge_pte_lock(h, mm, dst_pte);
4910 	src_ptl = huge_pte_lockptr(h, mm, src_pte);
4911 
4912 	/*
4913 	 * We don't have to worry about the ordering of src and dst ptlocks
4914 	 * because exclusive mmap_sem (or the i_mmap_lock) prevents deadlock.
4915 	 */
4916 	if (src_ptl != dst_ptl)
4917 		spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
4918 
4919 	pte = huge_ptep_get_and_clear(mm, old_addr, src_pte);
4920 	set_huge_pte_at(mm, new_addr, dst_pte, pte);
4921 
4922 	if (src_ptl != dst_ptl)
4923 		spin_unlock(src_ptl);
4924 	spin_unlock(dst_ptl);
4925 }
4926 
4927 int move_hugetlb_page_tables(struct vm_area_struct *vma,
4928 			     struct vm_area_struct *new_vma,
4929 			     unsigned long old_addr, unsigned long new_addr,
4930 			     unsigned long len)
4931 {
4932 	struct hstate *h = hstate_vma(vma);
4933 	struct address_space *mapping = vma->vm_file->f_mapping;
4934 	unsigned long sz = huge_page_size(h);
4935 	struct mm_struct *mm = vma->vm_mm;
4936 	unsigned long old_end = old_addr + len;
4937 	unsigned long old_addr_copy;
4938 	pte_t *src_pte, *dst_pte;
4939 	struct mmu_notifier_range range;
4940 	bool shared_pmd = false;
4941 
4942 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, old_addr,
4943 				old_end);
4944 	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
4945 	/*
4946 	 * In case of shared PMDs, we should cover the maximum possible
4947 	 * range.
4948 	 */
4949 	flush_cache_range(vma, range.start, range.end);
4950 
4951 	mmu_notifier_invalidate_range_start(&range);
4952 	/* Prevent race with file truncation */
4953 	i_mmap_lock_write(mapping);
4954 	for (; old_addr < old_end; old_addr += sz, new_addr += sz) {
4955 		src_pte = huge_pte_offset(mm, old_addr, sz);
4956 		if (!src_pte)
4957 			continue;
4958 		if (huge_pte_none(huge_ptep_get(src_pte)))
4959 			continue;
4960 
4961 		/* old_addr arg to huge_pmd_unshare() is a pointer and so the
4962 		 * arg may be modified. Pass a copy instead to preserve the
4963 		 * value in old_addr.
4964 		 */
4965 		old_addr_copy = old_addr;
4966 
4967 		if (huge_pmd_unshare(mm, vma, &old_addr_copy, src_pte)) {
4968 			shared_pmd = true;
4969 			continue;
4970 		}
4971 
4972 		dst_pte = huge_pte_alloc(mm, new_vma, new_addr, sz);
4973 		if (!dst_pte)
4974 			break;
4975 
4976 		move_huge_pte(vma, old_addr, new_addr, src_pte, dst_pte);
4977 	}
4978 
4979 	if (shared_pmd)
4980 		flush_tlb_range(vma, range.start, range.end);
4981 	else
4982 		flush_tlb_range(vma, old_end - len, old_end);
4983 	mmu_notifier_invalidate_range_end(&range);
4984 	i_mmap_unlock_write(mapping);
4985 
4986 	return len + old_addr - old_end;
4987 }
4988 
4989 static void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
4990 				   unsigned long start, unsigned long end,
4991 				   struct page *ref_page, zap_flags_t zap_flags)
4992 {
4993 	struct mm_struct *mm = vma->vm_mm;
4994 	unsigned long address;
4995 	pte_t *ptep;
4996 	pte_t pte;
4997 	spinlock_t *ptl;
4998 	struct page *page;
4999 	struct hstate *h = hstate_vma(vma);
5000 	unsigned long sz = huge_page_size(h);
5001 	struct mmu_notifier_range range;
5002 	bool force_flush = false;
5003 
5004 	WARN_ON(!is_vm_hugetlb_page(vma));
5005 	BUG_ON(start & ~huge_page_mask(h));
5006 	BUG_ON(end & ~huge_page_mask(h));
5007 
5008 	/*
5009 	 * This is a hugetlb vma, all the pte entries should point
5010 	 * to huge page.
5011 	 */
5012 	tlb_change_page_size(tlb, sz);
5013 	tlb_start_vma(tlb, vma);
5014 
5015 	/*
5016 	 * If sharing possible, alert mmu notifiers of worst case.
5017 	 */
5018 	mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0, vma, mm, start,
5019 				end);
5020 	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
5021 	mmu_notifier_invalidate_range_start(&range);
5022 	address = start;
5023 	for (; address < end; address += sz) {
5024 		ptep = huge_pte_offset(mm, address, sz);
5025 		if (!ptep)
5026 			continue;
5027 
5028 		ptl = huge_pte_lock(h, mm, ptep);
5029 		if (huge_pmd_unshare(mm, vma, &address, ptep)) {
5030 			spin_unlock(ptl);
5031 			tlb_flush_pmd_range(tlb, address & PUD_MASK, PUD_SIZE);
5032 			force_flush = true;
5033 			continue;
5034 		}
5035 
5036 		pte = huge_ptep_get(ptep);
5037 		if (huge_pte_none(pte)) {
5038 			spin_unlock(ptl);
5039 			continue;
5040 		}
5041 
5042 		/*
5043 		 * Migrating hugepage or HWPoisoned hugepage is already
5044 		 * unmapped and its refcount is dropped, so just clear pte here.
5045 		 */
5046 		if (unlikely(!pte_present(pte))) {
5047 			/*
5048 			 * If the pte was wr-protected by uffd-wp in any of the
5049 			 * swap forms, meanwhile the caller does not want to
5050 			 * drop the uffd-wp bit in this zap, then replace the
5051 			 * pte with a marker.
5052 			 */
5053 			if (pte_swp_uffd_wp_any(pte) &&
5054 			    !(zap_flags & ZAP_FLAG_DROP_MARKER))
5055 				set_huge_pte_at(mm, address, ptep,
5056 						make_pte_marker(PTE_MARKER_UFFD_WP));
5057 			else
5058 				huge_pte_clear(mm, address, ptep, sz);
5059 			spin_unlock(ptl);
5060 			continue;
5061 		}
5062 
5063 		page = pte_page(pte);
5064 		/*
5065 		 * If a reference page is supplied, it is because a specific
5066 		 * page is being unmapped, not a range. Ensure the page we
5067 		 * are about to unmap is the actual page of interest.
5068 		 */
5069 		if (ref_page) {
5070 			if (page != ref_page) {
5071 				spin_unlock(ptl);
5072 				continue;
5073 			}
5074 			/*
5075 			 * Mark the VMA as having unmapped its page so that
5076 			 * future faults in this VMA will fail rather than
5077 			 * looking like data was lost
5078 			 */
5079 			set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
5080 		}
5081 
5082 		pte = huge_ptep_get_and_clear(mm, address, ptep);
5083 		tlb_remove_huge_tlb_entry(h, tlb, ptep, address);
5084 		if (huge_pte_dirty(pte))
5085 			set_page_dirty(page);
5086 		/* Leave a uffd-wp pte marker if needed */
5087 		if (huge_pte_uffd_wp(pte) &&
5088 		    !(zap_flags & ZAP_FLAG_DROP_MARKER))
5089 			set_huge_pte_at(mm, address, ptep,
5090 					make_pte_marker(PTE_MARKER_UFFD_WP));
5091 		hugetlb_count_sub(pages_per_huge_page(h), mm);
5092 		page_remove_rmap(page, vma, true);
5093 
5094 		spin_unlock(ptl);
5095 		tlb_remove_page_size(tlb, page, huge_page_size(h));
5096 		/*
5097 		 * Bail out after unmapping reference page if supplied
5098 		 */
5099 		if (ref_page)
5100 			break;
5101 	}
5102 	mmu_notifier_invalidate_range_end(&range);
5103 	tlb_end_vma(tlb, vma);
5104 
5105 	/*
5106 	 * If we unshared PMDs, the TLB flush was not recorded in mmu_gather. We
5107 	 * could defer the flush until now, since by holding i_mmap_rwsem we
5108 	 * guaranteed that the last refernece would not be dropped. But we must
5109 	 * do the flushing before we return, as otherwise i_mmap_rwsem will be
5110 	 * dropped and the last reference to the shared PMDs page might be
5111 	 * dropped as well.
5112 	 *
5113 	 * In theory we could defer the freeing of the PMD pages as well, but
5114 	 * huge_pmd_unshare() relies on the exact page_count for the PMD page to
5115 	 * detect sharing, so we cannot defer the release of the page either.
5116 	 * Instead, do flush now.
5117 	 */
5118 	if (force_flush)
5119 		tlb_flush_mmu_tlbonly(tlb);
5120 }
5121 
5122 void __unmap_hugepage_range_final(struct mmu_gather *tlb,
5123 			  struct vm_area_struct *vma, unsigned long start,
5124 			  unsigned long end, struct page *ref_page,
5125 			  zap_flags_t zap_flags)
5126 {
5127 	__unmap_hugepage_range(tlb, vma, start, end, ref_page, zap_flags);
5128 
5129 	/*
5130 	 * Clear this flag so that x86's huge_pmd_share page_table_shareable
5131 	 * test will fail on a vma being torn down, and not grab a page table
5132 	 * on its way out.  We're lucky that the flag has such an appropriate
5133 	 * name, and can in fact be safely cleared here. We could clear it
5134 	 * before the __unmap_hugepage_range above, but all that's necessary
5135 	 * is to clear it before releasing the i_mmap_rwsem. This works
5136 	 * because in the context this is called, the VMA is about to be
5137 	 * destroyed and the i_mmap_rwsem is held.
5138 	 */
5139 	vma->vm_flags &= ~VM_MAYSHARE;
5140 }
5141 
5142 void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
5143 			  unsigned long end, struct page *ref_page,
5144 			  zap_flags_t zap_flags)
5145 {
5146 	struct mmu_gather tlb;
5147 
5148 	tlb_gather_mmu(&tlb, vma->vm_mm);
5149 	__unmap_hugepage_range(&tlb, vma, start, end, ref_page, zap_flags);
5150 	tlb_finish_mmu(&tlb);
5151 }
5152 
5153 /*
5154  * This is called when the original mapper is failing to COW a MAP_PRIVATE
5155  * mapping it owns the reserve page for. The intention is to unmap the page
5156  * from other VMAs and let the children be SIGKILLed if they are faulting the
5157  * same region.
5158  */
5159 static void unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
5160 			      struct page *page, unsigned long address)
5161 {
5162 	struct hstate *h = hstate_vma(vma);
5163 	struct vm_area_struct *iter_vma;
5164 	struct address_space *mapping;
5165 	pgoff_t pgoff;
5166 
5167 	/*
5168 	 * vm_pgoff is in PAGE_SIZE units, hence the different calculation
5169 	 * from page cache lookup which is in HPAGE_SIZE units.
5170 	 */
5171 	address = address & huge_page_mask(h);
5172 	pgoff = ((address - vma->vm_start) >> PAGE_SHIFT) +
5173 			vma->vm_pgoff;
5174 	mapping = vma->vm_file->f_mapping;
5175 
5176 	/*
5177 	 * Take the mapping lock for the duration of the table walk. As
5178 	 * this mapping should be shared between all the VMAs,
5179 	 * __unmap_hugepage_range() is called as the lock is already held
5180 	 */
5181 	i_mmap_lock_write(mapping);
5182 	vma_interval_tree_foreach(iter_vma, &mapping->i_mmap, pgoff, pgoff) {
5183 		/* Do not unmap the current VMA */
5184 		if (iter_vma == vma)
5185 			continue;
5186 
5187 		/*
5188 		 * Shared VMAs have their own reserves and do not affect
5189 		 * MAP_PRIVATE accounting but it is possible that a shared
5190 		 * VMA is using the same page so check and skip such VMAs.
5191 		 */
5192 		if (iter_vma->vm_flags & VM_MAYSHARE)
5193 			continue;
5194 
5195 		/*
5196 		 * Unmap the page from other VMAs without their own reserves.
5197 		 * They get marked to be SIGKILLed if they fault in these
5198 		 * areas. This is because a future no-page fault on this VMA
5199 		 * could insert a zeroed page instead of the data existing
5200 		 * from the time of fork. This would look like data corruption
5201 		 */
5202 		if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
5203 			unmap_hugepage_range(iter_vma, address,
5204 					     address + huge_page_size(h), page, 0);
5205 	}
5206 	i_mmap_unlock_write(mapping);
5207 }
5208 
5209 /*
5210  * hugetlb_wp() should be called with page lock of the original hugepage held.
5211  * Called with hugetlb_fault_mutex_table held and pte_page locked so we
5212  * cannot race with other handlers or page migration.
5213  * Keep the pte_same checks anyway to make transition from the mutex easier.
5214  */
5215 static vm_fault_t hugetlb_wp(struct mm_struct *mm, struct vm_area_struct *vma,
5216 		       unsigned long address, pte_t *ptep, unsigned int flags,
5217 		       struct page *pagecache_page, spinlock_t *ptl)
5218 {
5219 	const bool unshare = flags & FAULT_FLAG_UNSHARE;
5220 	pte_t pte;
5221 	struct hstate *h = hstate_vma(vma);
5222 	struct page *old_page, *new_page;
5223 	int outside_reserve = 0;
5224 	vm_fault_t ret = 0;
5225 	unsigned long haddr = address & huge_page_mask(h);
5226 	struct mmu_notifier_range range;
5227 
5228 	VM_BUG_ON(unshare && (flags & FOLL_WRITE));
5229 	VM_BUG_ON(!unshare && !(flags & FOLL_WRITE));
5230 
5231 	pte = huge_ptep_get(ptep);
5232 	old_page = pte_page(pte);
5233 
5234 	delayacct_wpcopy_start();
5235 
5236 retry_avoidcopy:
5237 	/*
5238 	 * If no-one else is actually using this page, we're the exclusive
5239 	 * owner and can reuse this page.
5240 	 */
5241 	if (page_mapcount(old_page) == 1 && PageAnon(old_page)) {
5242 		if (!PageAnonExclusive(old_page))
5243 			page_move_anon_rmap(old_page, vma);
5244 		if (likely(!unshare))
5245 			set_huge_ptep_writable(vma, haddr, ptep);
5246 
5247 		delayacct_wpcopy_end();
5248 		return 0;
5249 	}
5250 	VM_BUG_ON_PAGE(PageAnon(old_page) && PageAnonExclusive(old_page),
5251 		       old_page);
5252 
5253 	/*
5254 	 * If the process that created a MAP_PRIVATE mapping is about to
5255 	 * perform a COW due to a shared page count, attempt to satisfy
5256 	 * the allocation without using the existing reserves. The pagecache
5257 	 * page is used to determine if the reserve at this address was
5258 	 * consumed or not. If reserves were used, a partial faulted mapping
5259 	 * at the time of fork() could consume its reserves on COW instead
5260 	 * of the full address range.
5261 	 */
5262 	if (is_vma_resv_set(vma, HPAGE_RESV_OWNER) &&
5263 			old_page != pagecache_page)
5264 		outside_reserve = 1;
5265 
5266 	get_page(old_page);
5267 
5268 	/*
5269 	 * Drop page table lock as buddy allocator may be called. It will
5270 	 * be acquired again before returning to the caller, as expected.
5271 	 */
5272 	spin_unlock(ptl);
5273 	new_page = alloc_huge_page(vma, haddr, outside_reserve);
5274 
5275 	if (IS_ERR(new_page)) {
5276 		/*
5277 		 * If a process owning a MAP_PRIVATE mapping fails to COW,
5278 		 * it is due to references held by a child and an insufficient
5279 		 * huge page pool. To guarantee the original mappers
5280 		 * reliability, unmap the page from child processes. The child
5281 		 * may get SIGKILLed if it later faults.
5282 		 */
5283 		if (outside_reserve) {
5284 			struct address_space *mapping = vma->vm_file->f_mapping;
5285 			pgoff_t idx;
5286 			u32 hash;
5287 
5288 			put_page(old_page);
5289 			BUG_ON(huge_pte_none(pte));
5290 			/*
5291 			 * Drop hugetlb_fault_mutex and i_mmap_rwsem before
5292 			 * unmapping.  unmapping needs to hold i_mmap_rwsem
5293 			 * in write mode.  Dropping i_mmap_rwsem in read mode
5294 			 * here is OK as COW mappings do not interact with
5295 			 * PMD sharing.
5296 			 *
5297 			 * Reacquire both after unmap operation.
5298 			 */
5299 			idx = vma_hugecache_offset(h, vma, haddr);
5300 			hash = hugetlb_fault_mutex_hash(mapping, idx);
5301 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5302 			i_mmap_unlock_read(mapping);
5303 
5304 			unmap_ref_private(mm, vma, old_page, haddr);
5305 
5306 			i_mmap_lock_read(mapping);
5307 			mutex_lock(&hugetlb_fault_mutex_table[hash]);
5308 			spin_lock(ptl);
5309 			ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
5310 			if (likely(ptep &&
5311 				   pte_same(huge_ptep_get(ptep), pte)))
5312 				goto retry_avoidcopy;
5313 			/*
5314 			 * race occurs while re-acquiring page table
5315 			 * lock, and our job is done.
5316 			 */
5317 			delayacct_wpcopy_end();
5318 			return 0;
5319 		}
5320 
5321 		ret = vmf_error(PTR_ERR(new_page));
5322 		goto out_release_old;
5323 	}
5324 
5325 	/*
5326 	 * When the original hugepage is shared one, it does not have
5327 	 * anon_vma prepared.
5328 	 */
5329 	if (unlikely(anon_vma_prepare(vma))) {
5330 		ret = VM_FAULT_OOM;
5331 		goto out_release_all;
5332 	}
5333 
5334 	copy_user_huge_page(new_page, old_page, address, vma,
5335 			    pages_per_huge_page(h));
5336 	__SetPageUptodate(new_page);
5337 
5338 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm, haddr,
5339 				haddr + huge_page_size(h));
5340 	mmu_notifier_invalidate_range_start(&range);
5341 
5342 	/*
5343 	 * Retake the page table lock to check for racing updates
5344 	 * before the page tables are altered
5345 	 */
5346 	spin_lock(ptl);
5347 	ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
5348 	if (likely(ptep && pte_same(huge_ptep_get(ptep), pte))) {
5349 		ClearHPageRestoreReserve(new_page);
5350 
5351 		/* Break COW or unshare */
5352 		huge_ptep_clear_flush(vma, haddr, ptep);
5353 		mmu_notifier_invalidate_range(mm, range.start, range.end);
5354 		page_remove_rmap(old_page, vma, true);
5355 		hugepage_add_new_anon_rmap(new_page, vma, haddr);
5356 		set_huge_pte_at(mm, haddr, ptep,
5357 				make_huge_pte(vma, new_page, !unshare));
5358 		SetHPageMigratable(new_page);
5359 		/* Make the old page be freed below */
5360 		new_page = old_page;
5361 	}
5362 	spin_unlock(ptl);
5363 	mmu_notifier_invalidate_range_end(&range);
5364 out_release_all:
5365 	/*
5366 	 * No restore in case of successful pagetable update (Break COW or
5367 	 * unshare)
5368 	 */
5369 	if (new_page != old_page)
5370 		restore_reserve_on_error(h, vma, haddr, new_page);
5371 	put_page(new_page);
5372 out_release_old:
5373 	put_page(old_page);
5374 
5375 	spin_lock(ptl); /* Caller expects lock to be held */
5376 
5377 	delayacct_wpcopy_end();
5378 	return ret;
5379 }
5380 
5381 /* Return the pagecache page at a given address within a VMA */
5382 static struct page *hugetlbfs_pagecache_page(struct hstate *h,
5383 			struct vm_area_struct *vma, unsigned long address)
5384 {
5385 	struct address_space *mapping;
5386 	pgoff_t idx;
5387 
5388 	mapping = vma->vm_file->f_mapping;
5389 	idx = vma_hugecache_offset(h, vma, address);
5390 
5391 	return find_lock_page(mapping, idx);
5392 }
5393 
5394 /*
5395  * Return whether there is a pagecache page to back given address within VMA.
5396  * Caller follow_hugetlb_page() holds page_table_lock so we cannot lock_page.
5397  */
5398 static bool hugetlbfs_pagecache_present(struct hstate *h,
5399 			struct vm_area_struct *vma, unsigned long address)
5400 {
5401 	struct address_space *mapping;
5402 	pgoff_t idx;
5403 	struct page *page;
5404 
5405 	mapping = vma->vm_file->f_mapping;
5406 	idx = vma_hugecache_offset(h, vma, address);
5407 
5408 	page = find_get_page(mapping, idx);
5409 	if (page)
5410 		put_page(page);
5411 	return page != NULL;
5412 }
5413 
5414 int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
5415 			   pgoff_t idx)
5416 {
5417 	struct inode *inode = mapping->host;
5418 	struct hstate *h = hstate_inode(inode);
5419 	int err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
5420 
5421 	if (err)
5422 		return err;
5423 	ClearHPageRestoreReserve(page);
5424 
5425 	/*
5426 	 * set page dirty so that it will not be removed from cache/file
5427 	 * by non-hugetlbfs specific code paths.
5428 	 */
5429 	set_page_dirty(page);
5430 
5431 	spin_lock(&inode->i_lock);
5432 	inode->i_blocks += blocks_per_huge_page(h);
5433 	spin_unlock(&inode->i_lock);
5434 	return 0;
5435 }
5436 
5437 static inline vm_fault_t hugetlb_handle_userfault(struct vm_area_struct *vma,
5438 						  struct address_space *mapping,
5439 						  pgoff_t idx,
5440 						  unsigned int flags,
5441 						  unsigned long haddr,
5442 						  unsigned long addr,
5443 						  unsigned long reason)
5444 {
5445 	vm_fault_t ret;
5446 	u32 hash;
5447 	struct vm_fault vmf = {
5448 		.vma = vma,
5449 		.address = haddr,
5450 		.real_address = addr,
5451 		.flags = flags,
5452 
5453 		/*
5454 		 * Hard to debug if it ends up being
5455 		 * used by a callee that assumes
5456 		 * something about the other
5457 		 * uninitialized fields... same as in
5458 		 * memory.c
5459 		 */
5460 	};
5461 
5462 	/*
5463 	 * hugetlb_fault_mutex and i_mmap_rwsem must be
5464 	 * dropped before handling userfault.  Reacquire
5465 	 * after handling fault to make calling code simpler.
5466 	 */
5467 	hash = hugetlb_fault_mutex_hash(mapping, idx);
5468 	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5469 	i_mmap_unlock_read(mapping);
5470 	ret = handle_userfault(&vmf, reason);
5471 	i_mmap_lock_read(mapping);
5472 	mutex_lock(&hugetlb_fault_mutex_table[hash]);
5473 
5474 	return ret;
5475 }
5476 
5477 static vm_fault_t hugetlb_no_page(struct mm_struct *mm,
5478 			struct vm_area_struct *vma,
5479 			struct address_space *mapping, pgoff_t idx,
5480 			unsigned long address, pte_t *ptep,
5481 			pte_t old_pte, unsigned int flags)
5482 {
5483 	struct hstate *h = hstate_vma(vma);
5484 	vm_fault_t ret = VM_FAULT_SIGBUS;
5485 	int anon_rmap = 0;
5486 	unsigned long size;
5487 	struct page *page;
5488 	pte_t new_pte;
5489 	spinlock_t *ptl;
5490 	unsigned long haddr = address & huge_page_mask(h);
5491 	bool new_page, new_pagecache_page = false;
5492 
5493 	/*
5494 	 * Currently, we are forced to kill the process in the event the
5495 	 * original mapper has unmapped pages from the child due to a failed
5496 	 * COW/unsharing. Warn that such a situation has occurred as it may not
5497 	 * be obvious.
5498 	 */
5499 	if (is_vma_resv_set(vma, HPAGE_RESV_UNMAPPED)) {
5500 		pr_warn_ratelimited("PID %d killed due to inadequate hugepage pool\n",
5501 			   current->pid);
5502 		return ret;
5503 	}
5504 
5505 	/*
5506 	 * We can not race with truncation due to holding i_mmap_rwsem.
5507 	 * i_size is modified when holding i_mmap_rwsem, so check here
5508 	 * once for faults beyond end of file.
5509 	 */
5510 	size = i_size_read(mapping->host) >> huge_page_shift(h);
5511 	if (idx >= size)
5512 		goto out;
5513 
5514 retry:
5515 	new_page = false;
5516 	page = find_lock_page(mapping, idx);
5517 	if (!page) {
5518 		/* Check for page in userfault range */
5519 		if (userfaultfd_missing(vma)) {
5520 			ret = hugetlb_handle_userfault(vma, mapping, idx,
5521 						       flags, haddr, address,
5522 						       VM_UFFD_MISSING);
5523 			goto out;
5524 		}
5525 
5526 		page = alloc_huge_page(vma, haddr, 0);
5527 		if (IS_ERR(page)) {
5528 			/*
5529 			 * Returning error will result in faulting task being
5530 			 * sent SIGBUS.  The hugetlb fault mutex prevents two
5531 			 * tasks from racing to fault in the same page which
5532 			 * could result in false unable to allocate errors.
5533 			 * Page migration does not take the fault mutex, but
5534 			 * does a clear then write of pte's under page table
5535 			 * lock.  Page fault code could race with migration,
5536 			 * notice the clear pte and try to allocate a page
5537 			 * here.  Before returning error, get ptl and make
5538 			 * sure there really is no pte entry.
5539 			 */
5540 			ptl = huge_pte_lock(h, mm, ptep);
5541 			ret = 0;
5542 			if (huge_pte_none(huge_ptep_get(ptep)))
5543 				ret = vmf_error(PTR_ERR(page));
5544 			spin_unlock(ptl);
5545 			goto out;
5546 		}
5547 		clear_huge_page(page, address, pages_per_huge_page(h));
5548 		__SetPageUptodate(page);
5549 		new_page = true;
5550 
5551 		if (vma->vm_flags & VM_MAYSHARE) {
5552 			int err = huge_add_to_page_cache(page, mapping, idx);
5553 			if (err) {
5554 				put_page(page);
5555 				if (err == -EEXIST)
5556 					goto retry;
5557 				goto out;
5558 			}
5559 			new_pagecache_page = true;
5560 		} else {
5561 			lock_page(page);
5562 			if (unlikely(anon_vma_prepare(vma))) {
5563 				ret = VM_FAULT_OOM;
5564 				goto backout_unlocked;
5565 			}
5566 			anon_rmap = 1;
5567 		}
5568 	} else {
5569 		/*
5570 		 * If memory error occurs between mmap() and fault, some process
5571 		 * don't have hwpoisoned swap entry for errored virtual address.
5572 		 * So we need to block hugepage fault by PG_hwpoison bit check.
5573 		 */
5574 		if (unlikely(PageHWPoison(page))) {
5575 			ret = VM_FAULT_HWPOISON_LARGE |
5576 				VM_FAULT_SET_HINDEX(hstate_index(h));
5577 			goto backout_unlocked;
5578 		}
5579 
5580 		/* Check for page in userfault range. */
5581 		if (userfaultfd_minor(vma)) {
5582 			unlock_page(page);
5583 			put_page(page);
5584 			ret = hugetlb_handle_userfault(vma, mapping, idx,
5585 						       flags, haddr, address,
5586 						       VM_UFFD_MINOR);
5587 			goto out;
5588 		}
5589 	}
5590 
5591 	/*
5592 	 * If we are going to COW a private mapping later, we examine the
5593 	 * pending reservations for this page now. This will ensure that
5594 	 * any allocations necessary to record that reservation occur outside
5595 	 * the spinlock.
5596 	 */
5597 	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
5598 		if (vma_needs_reservation(h, vma, haddr) < 0) {
5599 			ret = VM_FAULT_OOM;
5600 			goto backout_unlocked;
5601 		}
5602 		/* Just decrements count, does not deallocate */
5603 		vma_end_reservation(h, vma, haddr);
5604 	}
5605 
5606 	ptl = huge_pte_lock(h, mm, ptep);
5607 	ret = 0;
5608 	/* If pte changed from under us, retry */
5609 	if (!pte_same(huge_ptep_get(ptep), old_pte))
5610 		goto backout;
5611 
5612 	if (anon_rmap) {
5613 		ClearHPageRestoreReserve(page);
5614 		hugepage_add_new_anon_rmap(page, vma, haddr);
5615 	} else
5616 		page_dup_file_rmap(page, true);
5617 	new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
5618 				&& (vma->vm_flags & VM_SHARED)));
5619 	/*
5620 	 * If this pte was previously wr-protected, keep it wr-protected even
5621 	 * if populated.
5622 	 */
5623 	if (unlikely(pte_marker_uffd_wp(old_pte)))
5624 		new_pte = huge_pte_wrprotect(huge_pte_mkuffd_wp(new_pte));
5625 	set_huge_pte_at(mm, haddr, ptep, new_pte);
5626 
5627 	hugetlb_count_add(pages_per_huge_page(h), mm);
5628 	if ((flags & FAULT_FLAG_WRITE) && !(vma->vm_flags & VM_SHARED)) {
5629 		/* Optimization, do the COW without a second fault */
5630 		ret = hugetlb_wp(mm, vma, address, ptep, flags, page, ptl);
5631 	}
5632 
5633 	spin_unlock(ptl);
5634 
5635 	/*
5636 	 * Only set HPageMigratable in newly allocated pages.  Existing pages
5637 	 * found in the pagecache may not have HPageMigratableset if they have
5638 	 * been isolated for migration.
5639 	 */
5640 	if (new_page)
5641 		SetHPageMigratable(page);
5642 
5643 	unlock_page(page);
5644 out:
5645 	return ret;
5646 
5647 backout:
5648 	spin_unlock(ptl);
5649 backout_unlocked:
5650 	unlock_page(page);
5651 	/* restore reserve for newly allocated pages not in page cache */
5652 	if (new_page && !new_pagecache_page)
5653 		restore_reserve_on_error(h, vma, haddr, page);
5654 	put_page(page);
5655 	goto out;
5656 }
5657 
5658 #ifdef CONFIG_SMP
5659 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
5660 {
5661 	unsigned long key[2];
5662 	u32 hash;
5663 
5664 	key[0] = (unsigned long) mapping;
5665 	key[1] = idx;
5666 
5667 	hash = jhash2((u32 *)&key, sizeof(key)/(sizeof(u32)), 0);
5668 
5669 	return hash & (num_fault_mutexes - 1);
5670 }
5671 #else
5672 /*
5673  * For uniprocessor systems we always use a single mutex, so just
5674  * return 0 and avoid the hashing overhead.
5675  */
5676 u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx)
5677 {
5678 	return 0;
5679 }
5680 #endif
5681 
5682 vm_fault_t hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
5683 			unsigned long address, unsigned int flags)
5684 {
5685 	pte_t *ptep, entry;
5686 	spinlock_t *ptl;
5687 	vm_fault_t ret;
5688 	u32 hash;
5689 	pgoff_t idx;
5690 	struct page *page = NULL;
5691 	struct page *pagecache_page = NULL;
5692 	struct hstate *h = hstate_vma(vma);
5693 	struct address_space *mapping;
5694 	int need_wait_lock = 0;
5695 	unsigned long haddr = address & huge_page_mask(h);
5696 
5697 	ptep = huge_pte_offset(mm, haddr, huge_page_size(h));
5698 	if (ptep) {
5699 		/*
5700 		 * Since we hold no locks, ptep could be stale.  That is
5701 		 * OK as we are only making decisions based on content and
5702 		 * not actually modifying content here.
5703 		 */
5704 		entry = huge_ptep_get(ptep);
5705 		if (unlikely(is_hugetlb_entry_migration(entry))) {
5706 			migration_entry_wait_huge(vma, mm, ptep);
5707 			return 0;
5708 		} else if (unlikely(is_hugetlb_entry_hwpoisoned(entry)))
5709 			return VM_FAULT_HWPOISON_LARGE |
5710 				VM_FAULT_SET_HINDEX(hstate_index(h));
5711 	}
5712 
5713 	/*
5714 	 * Acquire i_mmap_rwsem before calling huge_pte_alloc and hold
5715 	 * until finished with ptep.  This serves two purposes:
5716 	 * 1) It prevents huge_pmd_unshare from being called elsewhere
5717 	 *    and making the ptep no longer valid.
5718 	 * 2) It synchronizes us with i_size modifications during truncation.
5719 	 *
5720 	 * ptep could have already be assigned via huge_pte_offset.  That
5721 	 * is OK, as huge_pte_alloc will return the same value unless
5722 	 * something has changed.
5723 	 */
5724 	mapping = vma->vm_file->f_mapping;
5725 	i_mmap_lock_read(mapping);
5726 	ptep = huge_pte_alloc(mm, vma, haddr, huge_page_size(h));
5727 	if (!ptep) {
5728 		i_mmap_unlock_read(mapping);
5729 		return VM_FAULT_OOM;
5730 	}
5731 
5732 	/*
5733 	 * Serialize hugepage allocation and instantiation, so that we don't
5734 	 * get spurious allocation failures if two CPUs race to instantiate
5735 	 * the same page in the page cache.
5736 	 */
5737 	idx = vma_hugecache_offset(h, vma, haddr);
5738 	hash = hugetlb_fault_mutex_hash(mapping, idx);
5739 	mutex_lock(&hugetlb_fault_mutex_table[hash]);
5740 
5741 	entry = huge_ptep_get(ptep);
5742 	/* PTE markers should be handled the same way as none pte */
5743 	if (huge_pte_none_mostly(entry)) {
5744 		ret = hugetlb_no_page(mm, vma, mapping, idx, address, ptep,
5745 				      entry, flags);
5746 		goto out_mutex;
5747 	}
5748 
5749 	ret = 0;
5750 
5751 	/*
5752 	 * entry could be a migration/hwpoison entry at this point, so this
5753 	 * check prevents the kernel from going below assuming that we have
5754 	 * an active hugepage in pagecache. This goto expects the 2nd page
5755 	 * fault, and is_hugetlb_entry_(migration|hwpoisoned) check will
5756 	 * properly handle it.
5757 	 */
5758 	if (!pte_present(entry))
5759 		goto out_mutex;
5760 
5761 	/*
5762 	 * If we are going to COW/unshare the mapping later, we examine the
5763 	 * pending reservations for this page now. This will ensure that any
5764 	 * allocations necessary to record that reservation occur outside the
5765 	 * spinlock. For private mappings, we also lookup the pagecache
5766 	 * page now as it is used to determine if a reservation has been
5767 	 * consumed.
5768 	 */
5769 	if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) &&
5770 	    !huge_pte_write(entry)) {
5771 		if (vma_needs_reservation(h, vma, haddr) < 0) {
5772 			ret = VM_FAULT_OOM;
5773 			goto out_mutex;
5774 		}
5775 		/* Just decrements count, does not deallocate */
5776 		vma_end_reservation(h, vma, haddr);
5777 
5778 		if (!(vma->vm_flags & VM_MAYSHARE))
5779 			pagecache_page = hugetlbfs_pagecache_page(h,
5780 								vma, haddr);
5781 	}
5782 
5783 	ptl = huge_pte_lock(h, mm, ptep);
5784 
5785 	/* Check for a racing update before calling hugetlb_wp() */
5786 	if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
5787 		goto out_ptl;
5788 
5789 	/* Handle userfault-wp first, before trying to lock more pages */
5790 	if (userfaultfd_wp(vma) && huge_pte_uffd_wp(huge_ptep_get(ptep)) &&
5791 	    (flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
5792 		struct vm_fault vmf = {
5793 			.vma = vma,
5794 			.address = haddr,
5795 			.real_address = address,
5796 			.flags = flags,
5797 		};
5798 
5799 		spin_unlock(ptl);
5800 		if (pagecache_page) {
5801 			unlock_page(pagecache_page);
5802 			put_page(pagecache_page);
5803 		}
5804 		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5805 		i_mmap_unlock_read(mapping);
5806 		return handle_userfault(&vmf, VM_UFFD_WP);
5807 	}
5808 
5809 	/*
5810 	 * hugetlb_wp() requires page locks of pte_page(entry) and
5811 	 * pagecache_page, so here we need take the former one
5812 	 * when page != pagecache_page or !pagecache_page.
5813 	 */
5814 	page = pte_page(entry);
5815 	if (page != pagecache_page)
5816 		if (!trylock_page(page)) {
5817 			need_wait_lock = 1;
5818 			goto out_ptl;
5819 		}
5820 
5821 	get_page(page);
5822 
5823 	if (flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) {
5824 		if (!huge_pte_write(entry)) {
5825 			ret = hugetlb_wp(mm, vma, address, ptep, flags,
5826 					 pagecache_page, ptl);
5827 			goto out_put_page;
5828 		} else if (likely(flags & FAULT_FLAG_WRITE)) {
5829 			entry = huge_pte_mkdirty(entry);
5830 		}
5831 	}
5832 	entry = pte_mkyoung(entry);
5833 	if (huge_ptep_set_access_flags(vma, haddr, ptep, entry,
5834 						flags & FAULT_FLAG_WRITE))
5835 		update_mmu_cache(vma, haddr, ptep);
5836 out_put_page:
5837 	if (page != pagecache_page)
5838 		unlock_page(page);
5839 	put_page(page);
5840 out_ptl:
5841 	spin_unlock(ptl);
5842 
5843 	if (pagecache_page) {
5844 		unlock_page(pagecache_page);
5845 		put_page(pagecache_page);
5846 	}
5847 out_mutex:
5848 	mutex_unlock(&hugetlb_fault_mutex_table[hash]);
5849 	i_mmap_unlock_read(mapping);
5850 	/*
5851 	 * Generally it's safe to hold refcount during waiting page lock. But
5852 	 * here we just wait to defer the next page fault to avoid busy loop and
5853 	 * the page is not used after unlocked before returning from the current
5854 	 * page fault. So we are safe from accessing freed page, even if we wait
5855 	 * here without taking refcount.
5856 	 */
5857 	if (need_wait_lock)
5858 		wait_on_page_locked(page);
5859 	return ret;
5860 }
5861 
5862 #ifdef CONFIG_USERFAULTFD
5863 /*
5864  * Used by userfaultfd UFFDIO_COPY.  Based on mcopy_atomic_pte with
5865  * modifications for huge pages.
5866  */
5867 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm,
5868 			    pte_t *dst_pte,
5869 			    struct vm_area_struct *dst_vma,
5870 			    unsigned long dst_addr,
5871 			    unsigned long src_addr,
5872 			    enum mcopy_atomic_mode mode,
5873 			    struct page **pagep,
5874 			    bool wp_copy)
5875 {
5876 	bool is_continue = (mode == MCOPY_ATOMIC_CONTINUE);
5877 	struct hstate *h = hstate_vma(dst_vma);
5878 	struct address_space *mapping = dst_vma->vm_file->f_mapping;
5879 	pgoff_t idx = vma_hugecache_offset(h, dst_vma, dst_addr);
5880 	unsigned long size;
5881 	int vm_shared = dst_vma->vm_flags & VM_SHARED;
5882 	pte_t _dst_pte;
5883 	spinlock_t *ptl;
5884 	int ret = -ENOMEM;
5885 	struct page *page;
5886 	int writable;
5887 	bool page_in_pagecache = false;
5888 
5889 	if (is_continue) {
5890 		ret = -EFAULT;
5891 		page = find_lock_page(mapping, idx);
5892 		if (!page)
5893 			goto out;
5894 		page_in_pagecache = true;
5895 	} else if (!*pagep) {
5896 		/* If a page already exists, then it's UFFDIO_COPY for
5897 		 * a non-missing case. Return -EEXIST.
5898 		 */
5899 		if (vm_shared &&
5900 		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
5901 			ret = -EEXIST;
5902 			goto out;
5903 		}
5904 
5905 		page = alloc_huge_page(dst_vma, dst_addr, 0);
5906 		if (IS_ERR(page)) {
5907 			ret = -ENOMEM;
5908 			goto out;
5909 		}
5910 
5911 		ret = copy_huge_page_from_user(page,
5912 						(const void __user *) src_addr,
5913 						pages_per_huge_page(h), false);
5914 
5915 		/* fallback to copy_from_user outside mmap_lock */
5916 		if (unlikely(ret)) {
5917 			ret = -ENOENT;
5918 			/* Free the allocated page which may have
5919 			 * consumed a reservation.
5920 			 */
5921 			restore_reserve_on_error(h, dst_vma, dst_addr, page);
5922 			put_page(page);
5923 
5924 			/* Allocate a temporary page to hold the copied
5925 			 * contents.
5926 			 */
5927 			page = alloc_huge_page_vma(h, dst_vma, dst_addr);
5928 			if (!page) {
5929 				ret = -ENOMEM;
5930 				goto out;
5931 			}
5932 			*pagep = page;
5933 			/* Set the outparam pagep and return to the caller to
5934 			 * copy the contents outside the lock. Don't free the
5935 			 * page.
5936 			 */
5937 			goto out;
5938 		}
5939 	} else {
5940 		if (vm_shared &&
5941 		    hugetlbfs_pagecache_present(h, dst_vma, dst_addr)) {
5942 			put_page(*pagep);
5943 			ret = -EEXIST;
5944 			*pagep = NULL;
5945 			goto out;
5946 		}
5947 
5948 		page = alloc_huge_page(dst_vma, dst_addr, 0);
5949 		if (IS_ERR(page)) {
5950 			ret = -ENOMEM;
5951 			*pagep = NULL;
5952 			goto out;
5953 		}
5954 		copy_user_huge_page(page, *pagep, dst_addr, dst_vma,
5955 				    pages_per_huge_page(h));
5956 		put_page(*pagep);
5957 		*pagep = NULL;
5958 	}
5959 
5960 	/*
5961 	 * The memory barrier inside __SetPageUptodate makes sure that
5962 	 * preceding stores to the page contents become visible before
5963 	 * the set_pte_at() write.
5964 	 */
5965 	__SetPageUptodate(page);
5966 
5967 	/* Add shared, newly allocated pages to the page cache. */
5968 	if (vm_shared && !is_continue) {
5969 		size = i_size_read(mapping->host) >> huge_page_shift(h);
5970 		ret = -EFAULT;
5971 		if (idx >= size)
5972 			goto out_release_nounlock;
5973 
5974 		/*
5975 		 * Serialization between remove_inode_hugepages() and
5976 		 * huge_add_to_page_cache() below happens through the
5977 		 * hugetlb_fault_mutex_table that here must be hold by
5978 		 * the caller.
5979 		 */
5980 		ret = huge_add_to_page_cache(page, mapping, idx);
5981 		if (ret)
5982 			goto out_release_nounlock;
5983 		page_in_pagecache = true;
5984 	}
5985 
5986 	ptl = huge_pte_lockptr(h, dst_mm, dst_pte);
5987 	spin_lock(ptl);
5988 
5989 	/*
5990 	 * Recheck the i_size after holding PT lock to make sure not
5991 	 * to leave any page mapped (as page_mapped()) beyond the end
5992 	 * of the i_size (remove_inode_hugepages() is strict about
5993 	 * enforcing that). If we bail out here, we'll also leave a
5994 	 * page in the radix tree in the vm_shared case beyond the end
5995 	 * of the i_size, but remove_inode_hugepages() will take care
5996 	 * of it as soon as we drop the hugetlb_fault_mutex_table.
5997 	 */
5998 	size = i_size_read(mapping->host) >> huge_page_shift(h);
5999 	ret = -EFAULT;
6000 	if (idx >= size)
6001 		goto out_release_unlock;
6002 
6003 	ret = -EEXIST;
6004 	/*
6005 	 * We allow to overwrite a pte marker: consider when both MISSING|WP
6006 	 * registered, we firstly wr-protect a none pte which has no page cache
6007 	 * page backing it, then access the page.
6008 	 */
6009 	if (!huge_pte_none_mostly(huge_ptep_get(dst_pte)))
6010 		goto out_release_unlock;
6011 
6012 	if (vm_shared) {
6013 		page_dup_file_rmap(page, true);
6014 	} else {
6015 		ClearHPageRestoreReserve(page);
6016 		hugepage_add_new_anon_rmap(page, dst_vma, dst_addr);
6017 	}
6018 
6019 	/*
6020 	 * For either: (1) CONTINUE on a non-shared VMA, or (2) UFFDIO_COPY
6021 	 * with wp flag set, don't set pte write bit.
6022 	 */
6023 	if (wp_copy || (is_continue && !vm_shared))
6024 		writable = 0;
6025 	else
6026 		writable = dst_vma->vm_flags & VM_WRITE;
6027 
6028 	_dst_pte = make_huge_pte(dst_vma, page, writable);
6029 	/*
6030 	 * Always mark UFFDIO_COPY page dirty; note that this may not be
6031 	 * extremely important for hugetlbfs for now since swapping is not
6032 	 * supported, but we should still be clear in that this page cannot be
6033 	 * thrown away at will, even if write bit not set.
6034 	 */
6035 	_dst_pte = huge_pte_mkdirty(_dst_pte);
6036 	_dst_pte = pte_mkyoung(_dst_pte);
6037 
6038 	if (wp_copy)
6039 		_dst_pte = huge_pte_mkuffd_wp(_dst_pte);
6040 
6041 	set_huge_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
6042 
6043 	(void)huge_ptep_set_access_flags(dst_vma, dst_addr, dst_pte, _dst_pte,
6044 					dst_vma->vm_flags & VM_WRITE);
6045 	hugetlb_count_add(pages_per_huge_page(h), dst_mm);
6046 
6047 	/* No need to invalidate - it was non-present before */
6048 	update_mmu_cache(dst_vma, dst_addr, dst_pte);
6049 
6050 	spin_unlock(ptl);
6051 	if (!is_continue)
6052 		SetHPageMigratable(page);
6053 	if (vm_shared || is_continue)
6054 		unlock_page(page);
6055 	ret = 0;
6056 out:
6057 	return ret;
6058 out_release_unlock:
6059 	spin_unlock(ptl);
6060 	if (vm_shared || is_continue)
6061 		unlock_page(page);
6062 out_release_nounlock:
6063 	if (!page_in_pagecache)
6064 		restore_reserve_on_error(h, dst_vma, dst_addr, page);
6065 	put_page(page);
6066 	goto out;
6067 }
6068 #endif /* CONFIG_USERFAULTFD */
6069 
6070 static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma,
6071 				 int refs, struct page **pages,
6072 				 struct vm_area_struct **vmas)
6073 {
6074 	int nr;
6075 
6076 	for (nr = 0; nr < refs; nr++) {
6077 		if (likely(pages))
6078 			pages[nr] = mem_map_offset(page, nr);
6079 		if (vmas)
6080 			vmas[nr] = vma;
6081 	}
6082 }
6083 
6084 static inline bool __follow_hugetlb_must_fault(unsigned int flags, pte_t *pte,
6085 					       bool *unshare)
6086 {
6087 	pte_t pteval = huge_ptep_get(pte);
6088 
6089 	*unshare = false;
6090 	if (is_swap_pte(pteval))
6091 		return true;
6092 	if (huge_pte_write(pteval))
6093 		return false;
6094 	if (flags & FOLL_WRITE)
6095 		return true;
6096 	if (gup_must_unshare(flags, pte_page(pteval))) {
6097 		*unshare = true;
6098 		return true;
6099 	}
6100 	return false;
6101 }
6102 
6103 long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
6104 			 struct page **pages, struct vm_area_struct **vmas,
6105 			 unsigned long *position, unsigned long *nr_pages,
6106 			 long i, unsigned int flags, int *locked)
6107 {
6108 	unsigned long pfn_offset;
6109 	unsigned long vaddr = *position;
6110 	unsigned long remainder = *nr_pages;
6111 	struct hstate *h = hstate_vma(vma);
6112 	int err = -EFAULT, refs;
6113 
6114 	while (vaddr < vma->vm_end && remainder) {
6115 		pte_t *pte;
6116 		spinlock_t *ptl = NULL;
6117 		bool unshare = false;
6118 		int absent;
6119 		struct page *page;
6120 
6121 		/*
6122 		 * If we have a pending SIGKILL, don't keep faulting pages and
6123 		 * potentially allocating memory.
6124 		 */
6125 		if (fatal_signal_pending(current)) {
6126 			remainder = 0;
6127 			break;
6128 		}
6129 
6130 		/*
6131 		 * Some archs (sparc64, sh*) have multiple pte_ts to
6132 		 * each hugepage.  We have to make sure we get the
6133 		 * first, for the page indexing below to work.
6134 		 *
6135 		 * Note that page table lock is not held when pte is null.
6136 		 */
6137 		pte = huge_pte_offset(mm, vaddr & huge_page_mask(h),
6138 				      huge_page_size(h));
6139 		if (pte)
6140 			ptl = huge_pte_lock(h, mm, pte);
6141 		absent = !pte || huge_pte_none(huge_ptep_get(pte));
6142 
6143 		/*
6144 		 * When coredumping, it suits get_dump_page if we just return
6145 		 * an error where there's an empty slot with no huge pagecache
6146 		 * to back it.  This way, we avoid allocating a hugepage, and
6147 		 * the sparse dumpfile avoids allocating disk blocks, but its
6148 		 * huge holes still show up with zeroes where they need to be.
6149 		 */
6150 		if (absent && (flags & FOLL_DUMP) &&
6151 		    !hugetlbfs_pagecache_present(h, vma, vaddr)) {
6152 			if (pte)
6153 				spin_unlock(ptl);
6154 			remainder = 0;
6155 			break;
6156 		}
6157 
6158 		/*
6159 		 * We need call hugetlb_fault for both hugepages under migration
6160 		 * (in which case hugetlb_fault waits for the migration,) and
6161 		 * hwpoisoned hugepages (in which case we need to prevent the
6162 		 * caller from accessing to them.) In order to do this, we use
6163 		 * here is_swap_pte instead of is_hugetlb_entry_migration and
6164 		 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
6165 		 * both cases, and because we can't follow correct pages
6166 		 * directly from any kind of swap entries.
6167 		 */
6168 		if (absent ||
6169 		    __follow_hugetlb_must_fault(flags, pte, &unshare)) {
6170 			vm_fault_t ret;
6171 			unsigned int fault_flags = 0;
6172 
6173 			if (pte)
6174 				spin_unlock(ptl);
6175 			if (flags & FOLL_WRITE)
6176 				fault_flags |= FAULT_FLAG_WRITE;
6177 			else if (unshare)
6178 				fault_flags |= FAULT_FLAG_UNSHARE;
6179 			if (locked)
6180 				fault_flags |= FAULT_FLAG_ALLOW_RETRY |
6181 					FAULT_FLAG_KILLABLE;
6182 			if (flags & FOLL_NOWAIT)
6183 				fault_flags |= FAULT_FLAG_ALLOW_RETRY |
6184 					FAULT_FLAG_RETRY_NOWAIT;
6185 			if (flags & FOLL_TRIED) {
6186 				/*
6187 				 * Note: FAULT_FLAG_ALLOW_RETRY and
6188 				 * FAULT_FLAG_TRIED can co-exist
6189 				 */
6190 				fault_flags |= FAULT_FLAG_TRIED;
6191 			}
6192 			ret = hugetlb_fault(mm, vma, vaddr, fault_flags);
6193 			if (ret & VM_FAULT_ERROR) {
6194 				err = vm_fault_to_errno(ret, flags);
6195 				remainder = 0;
6196 				break;
6197 			}
6198 			if (ret & VM_FAULT_RETRY) {
6199 				if (locked &&
6200 				    !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
6201 					*locked = 0;
6202 				*nr_pages = 0;
6203 				/*
6204 				 * VM_FAULT_RETRY must not return an
6205 				 * error, it will return zero
6206 				 * instead.
6207 				 *
6208 				 * No need to update "position" as the
6209 				 * caller will not check it after
6210 				 * *nr_pages is set to 0.
6211 				 */
6212 				return i;
6213 			}
6214 			continue;
6215 		}
6216 
6217 		pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
6218 		page = pte_page(huge_ptep_get(pte));
6219 
6220 		VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
6221 			       !PageAnonExclusive(page), page);
6222 
6223 		/*
6224 		 * If subpage information not requested, update counters
6225 		 * and skip the same_page loop below.
6226 		 */
6227 		if (!pages && !vmas && !pfn_offset &&
6228 		    (vaddr + huge_page_size(h) < vma->vm_end) &&
6229 		    (remainder >= pages_per_huge_page(h))) {
6230 			vaddr += huge_page_size(h);
6231 			remainder -= pages_per_huge_page(h);
6232 			i += pages_per_huge_page(h);
6233 			spin_unlock(ptl);
6234 			continue;
6235 		}
6236 
6237 		/* vaddr may not be aligned to PAGE_SIZE */
6238 		refs = min3(pages_per_huge_page(h) - pfn_offset, remainder,
6239 		    (vma->vm_end - ALIGN_DOWN(vaddr, PAGE_SIZE)) >> PAGE_SHIFT);
6240 
6241 		if (pages || vmas)
6242 			record_subpages_vmas(mem_map_offset(page, pfn_offset),
6243 					     vma, refs,
6244 					     likely(pages) ? pages + i : NULL,
6245 					     vmas ? vmas + i : NULL);
6246 
6247 		if (pages) {
6248 			/*
6249 			 * try_grab_folio() should always succeed here,
6250 			 * because: a) we hold the ptl lock, and b) we've just
6251 			 * checked that the huge page is present in the page
6252 			 * tables. If the huge page is present, then the tail
6253 			 * pages must also be present. The ptl prevents the
6254 			 * head page and tail pages from being rearranged in
6255 			 * any way. So this page must be available at this
6256 			 * point, unless the page refcount overflowed:
6257 			 */
6258 			if (WARN_ON_ONCE(!try_grab_folio(pages[i], refs,
6259 							 flags))) {
6260 				spin_unlock(ptl);
6261 				remainder = 0;
6262 				err = -ENOMEM;
6263 				break;
6264 			}
6265 		}
6266 
6267 		vaddr += (refs << PAGE_SHIFT);
6268 		remainder -= refs;
6269 		i += refs;
6270 
6271 		spin_unlock(ptl);
6272 	}
6273 	*nr_pages = remainder;
6274 	/*
6275 	 * setting position is actually required only if remainder is
6276 	 * not zero but it's faster not to add a "if (remainder)"
6277 	 * branch.
6278 	 */
6279 	*position = vaddr;
6280 
6281 	return i ? i : err;
6282 }
6283 
6284 unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
6285 		unsigned long address, unsigned long end,
6286 		pgprot_t newprot, unsigned long cp_flags)
6287 {
6288 	struct mm_struct *mm = vma->vm_mm;
6289 	unsigned long start = address;
6290 	pte_t *ptep;
6291 	pte_t pte;
6292 	struct hstate *h = hstate_vma(vma);
6293 	unsigned long pages = 0, psize = huge_page_size(h);
6294 	bool shared_pmd = false;
6295 	struct mmu_notifier_range range;
6296 	bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
6297 	bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
6298 
6299 	/*
6300 	 * In the case of shared PMDs, the area to flush could be beyond
6301 	 * start/end.  Set range.start/range.end to cover the maximum possible
6302 	 * range if PMD sharing is possible.
6303 	 */
6304 	mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA,
6305 				0, vma, mm, start, end);
6306 	adjust_range_if_pmd_sharing_possible(vma, &range.start, &range.end);
6307 
6308 	BUG_ON(address >= end);
6309 	flush_cache_range(vma, range.start, range.end);
6310 
6311 	mmu_notifier_invalidate_range_start(&range);
6312 	i_mmap_lock_write(vma->vm_file->f_mapping);
6313 	for (; address < end; address += psize) {
6314 		spinlock_t *ptl;
6315 		ptep = huge_pte_offset(mm, address, psize);
6316 		if (!ptep)
6317 			continue;
6318 		ptl = huge_pte_lock(h, mm, ptep);
6319 		if (huge_pmd_unshare(mm, vma, &address, ptep)) {
6320 			/*
6321 			 * When uffd-wp is enabled on the vma, unshare
6322 			 * shouldn't happen at all.  Warn about it if it
6323 			 * happened due to some reason.
6324 			 */
6325 			WARN_ON_ONCE(uffd_wp || uffd_wp_resolve);
6326 			pages++;
6327 			spin_unlock(ptl);
6328 			shared_pmd = true;
6329 			continue;
6330 		}
6331 		pte = huge_ptep_get(ptep);
6332 		if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
6333 			spin_unlock(ptl);
6334 			continue;
6335 		}
6336 		if (unlikely(is_hugetlb_entry_migration(pte))) {
6337 			swp_entry_t entry = pte_to_swp_entry(pte);
6338 			struct page *page = pfn_swap_entry_to_page(entry);
6339 
6340 			if (!is_readable_migration_entry(entry)) {
6341 				pte_t newpte;
6342 
6343 				if (PageAnon(page))
6344 					entry = make_readable_exclusive_migration_entry(
6345 								swp_offset(entry));
6346 				else
6347 					entry = make_readable_migration_entry(
6348 								swp_offset(entry));
6349 				newpte = swp_entry_to_pte(entry);
6350 				if (uffd_wp)
6351 					newpte = pte_swp_mkuffd_wp(newpte);
6352 				else if (uffd_wp_resolve)
6353 					newpte = pte_swp_clear_uffd_wp(newpte);
6354 				set_huge_swap_pte_at(mm, address, ptep,
6355 						     newpte, psize);
6356 				pages++;
6357 			}
6358 			spin_unlock(ptl);
6359 			continue;
6360 		}
6361 		if (unlikely(pte_marker_uffd_wp(pte))) {
6362 			/*
6363 			 * This is changing a non-present pte into a none pte,
6364 			 * no need for huge_ptep_modify_prot_start/commit().
6365 			 */
6366 			if (uffd_wp_resolve)
6367 				huge_pte_clear(mm, address, ptep, psize);
6368 		}
6369 		if (!huge_pte_none(pte)) {
6370 			pte_t old_pte;
6371 			unsigned int shift = huge_page_shift(hstate_vma(vma));
6372 
6373 			old_pte = huge_ptep_modify_prot_start(vma, address, ptep);
6374 			pte = huge_pte_modify(old_pte, newprot);
6375 			pte = arch_make_huge_pte(pte, shift, vma->vm_flags);
6376 			if (uffd_wp)
6377 				pte = huge_pte_mkuffd_wp(huge_pte_wrprotect(pte));
6378 			else if (uffd_wp_resolve)
6379 				pte = huge_pte_clear_uffd_wp(pte);
6380 			huge_ptep_modify_prot_commit(vma, address, ptep, old_pte, pte);
6381 			pages++;
6382 		} else {
6383 			/* None pte */
6384 			if (unlikely(uffd_wp))
6385 				/* Safe to modify directly (none->non-present). */
6386 				set_huge_pte_at(mm, address, ptep,
6387 						make_pte_marker(PTE_MARKER_UFFD_WP));
6388 		}
6389 		spin_unlock(ptl);
6390 	}
6391 	/*
6392 	 * Must flush TLB before releasing i_mmap_rwsem: x86's huge_pmd_unshare
6393 	 * may have cleared our pud entry and done put_page on the page table:
6394 	 * once we release i_mmap_rwsem, another task can do the final put_page
6395 	 * and that page table be reused and filled with junk.  If we actually
6396 	 * did unshare a page of pmds, flush the range corresponding to the pud.
6397 	 */
6398 	if (shared_pmd)
6399 		flush_hugetlb_tlb_range(vma, range.start, range.end);
6400 	else
6401 		flush_hugetlb_tlb_range(vma, start, end);
6402 	/*
6403 	 * No need to call mmu_notifier_invalidate_range() we are downgrading
6404 	 * page table protection not changing it to point to a new page.
6405 	 *
6406 	 * See Documentation/vm/mmu_notifier.rst
6407 	 */
6408 	i_mmap_unlock_write(vma->vm_file->f_mapping);
6409 	mmu_notifier_invalidate_range_end(&range);
6410 
6411 	return pages << h->order;
6412 }
6413 
6414 /* Return true if reservation was successful, false otherwise.  */
6415 bool hugetlb_reserve_pages(struct inode *inode,
6416 					long from, long to,
6417 					struct vm_area_struct *vma,
6418 					vm_flags_t vm_flags)
6419 {
6420 	long chg, add = -1;
6421 	struct hstate *h = hstate_inode(inode);
6422 	struct hugepage_subpool *spool = subpool_inode(inode);
6423 	struct resv_map *resv_map;
6424 	struct hugetlb_cgroup *h_cg = NULL;
6425 	long gbl_reserve, regions_needed = 0;
6426 
6427 	/* This should never happen */
6428 	if (from > to) {
6429 		VM_WARN(1, "%s called with a negative range\n", __func__);
6430 		return false;
6431 	}
6432 
6433 	/*
6434 	 * Only apply hugepage reservation if asked. At fault time, an
6435 	 * attempt will be made for VM_NORESERVE to allocate a page
6436 	 * without using reserves
6437 	 */
6438 	if (vm_flags & VM_NORESERVE)
6439 		return true;
6440 
6441 	/*
6442 	 * Shared mappings base their reservation on the number of pages that
6443 	 * are already allocated on behalf of the file. Private mappings need
6444 	 * to reserve the full area even if read-only as mprotect() may be
6445 	 * called to make the mapping read-write. Assume !vma is a shm mapping
6446 	 */
6447 	if (!vma || vma->vm_flags & VM_MAYSHARE) {
6448 		/*
6449 		 * resv_map can not be NULL as hugetlb_reserve_pages is only
6450 		 * called for inodes for which resv_maps were created (see
6451 		 * hugetlbfs_get_inode).
6452 		 */
6453 		resv_map = inode_resv_map(inode);
6454 
6455 		chg = region_chg(resv_map, from, to, &regions_needed);
6456 
6457 	} else {
6458 		/* Private mapping. */
6459 		resv_map = resv_map_alloc();
6460 		if (!resv_map)
6461 			return false;
6462 
6463 		chg = to - from;
6464 
6465 		set_vma_resv_map(vma, resv_map);
6466 		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
6467 	}
6468 
6469 	if (chg < 0)
6470 		goto out_err;
6471 
6472 	if (hugetlb_cgroup_charge_cgroup_rsvd(hstate_index(h),
6473 				chg * pages_per_huge_page(h), &h_cg) < 0)
6474 		goto out_err;
6475 
6476 	if (vma && !(vma->vm_flags & VM_MAYSHARE) && h_cg) {
6477 		/* For private mappings, the hugetlb_cgroup uncharge info hangs
6478 		 * of the resv_map.
6479 		 */
6480 		resv_map_set_hugetlb_cgroup_uncharge_info(resv_map, h_cg, h);
6481 	}
6482 
6483 	/*
6484 	 * There must be enough pages in the subpool for the mapping. If
6485 	 * the subpool has a minimum size, there may be some global
6486 	 * reservations already in place (gbl_reserve).
6487 	 */
6488 	gbl_reserve = hugepage_subpool_get_pages(spool, chg);
6489 	if (gbl_reserve < 0)
6490 		goto out_uncharge_cgroup;
6491 
6492 	/*
6493 	 * Check enough hugepages are available for the reservation.
6494 	 * Hand the pages back to the subpool if there are not
6495 	 */
6496 	if (hugetlb_acct_memory(h, gbl_reserve) < 0)
6497 		goto out_put_pages;
6498 
6499 	/*
6500 	 * Account for the reservations made. Shared mappings record regions
6501 	 * that have reservations as they are shared by multiple VMAs.
6502 	 * When the last VMA disappears, the region map says how much
6503 	 * the reservation was and the page cache tells how much of
6504 	 * the reservation was consumed. Private mappings are per-VMA and
6505 	 * only the consumed reservations are tracked. When the VMA
6506 	 * disappears, the original reservation is the VMA size and the
6507 	 * consumed reservations are stored in the map. Hence, nothing
6508 	 * else has to be done for private mappings here
6509 	 */
6510 	if (!vma || vma->vm_flags & VM_MAYSHARE) {
6511 		add = region_add(resv_map, from, to, regions_needed, h, h_cg);
6512 
6513 		if (unlikely(add < 0)) {
6514 			hugetlb_acct_memory(h, -gbl_reserve);
6515 			goto out_put_pages;
6516 		} else if (unlikely(chg > add)) {
6517 			/*
6518 			 * pages in this range were added to the reserve
6519 			 * map between region_chg and region_add.  This
6520 			 * indicates a race with alloc_huge_page.  Adjust
6521 			 * the subpool and reserve counts modified above
6522 			 * based on the difference.
6523 			 */
6524 			long rsv_adjust;
6525 
6526 			/*
6527 			 * hugetlb_cgroup_uncharge_cgroup_rsvd() will put the
6528 			 * reference to h_cg->css. See comment below for detail.
6529 			 */
6530 			hugetlb_cgroup_uncharge_cgroup_rsvd(
6531 				hstate_index(h),
6532 				(chg - add) * pages_per_huge_page(h), h_cg);
6533 
6534 			rsv_adjust = hugepage_subpool_put_pages(spool,
6535 								chg - add);
6536 			hugetlb_acct_memory(h, -rsv_adjust);
6537 		} else if (h_cg) {
6538 			/*
6539 			 * The file_regions will hold their own reference to
6540 			 * h_cg->css. So we should release the reference held
6541 			 * via hugetlb_cgroup_charge_cgroup_rsvd() when we are
6542 			 * done.
6543 			 */
6544 			hugetlb_cgroup_put_rsvd_cgroup(h_cg);
6545 		}
6546 	}
6547 	return true;
6548 
6549 out_put_pages:
6550 	/* put back original number of pages, chg */
6551 	(void)hugepage_subpool_put_pages(spool, chg);
6552 out_uncharge_cgroup:
6553 	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
6554 					    chg * pages_per_huge_page(h), h_cg);
6555 out_err:
6556 	if (!vma || vma->vm_flags & VM_MAYSHARE)
6557 		/* Only call region_abort if the region_chg succeeded but the
6558 		 * region_add failed or didn't run.
6559 		 */
6560 		if (chg >= 0 && add < 0)
6561 			region_abort(resv_map, from, to, regions_needed);
6562 	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER))
6563 		kref_put(&resv_map->refs, resv_map_release);
6564 	return false;
6565 }
6566 
6567 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
6568 								long freed)
6569 {
6570 	struct hstate *h = hstate_inode(inode);
6571 	struct resv_map *resv_map = inode_resv_map(inode);
6572 	long chg = 0;
6573 	struct hugepage_subpool *spool = subpool_inode(inode);
6574 	long gbl_reserve;
6575 
6576 	/*
6577 	 * Since this routine can be called in the evict inode path for all
6578 	 * hugetlbfs inodes, resv_map could be NULL.
6579 	 */
6580 	if (resv_map) {
6581 		chg = region_del(resv_map, start, end);
6582 		/*
6583 		 * region_del() can fail in the rare case where a region
6584 		 * must be split and another region descriptor can not be
6585 		 * allocated.  If end == LONG_MAX, it will not fail.
6586 		 */
6587 		if (chg < 0)
6588 			return chg;
6589 	}
6590 
6591 	spin_lock(&inode->i_lock);
6592 	inode->i_blocks -= (blocks_per_huge_page(h) * freed);
6593 	spin_unlock(&inode->i_lock);
6594 
6595 	/*
6596 	 * If the subpool has a minimum size, the number of global
6597 	 * reservations to be released may be adjusted.
6598 	 *
6599 	 * Note that !resv_map implies freed == 0. So (chg - freed)
6600 	 * won't go negative.
6601 	 */
6602 	gbl_reserve = hugepage_subpool_put_pages(spool, (chg - freed));
6603 	hugetlb_acct_memory(h, -gbl_reserve);
6604 
6605 	return 0;
6606 }
6607 
6608 #ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
6609 static unsigned long page_table_shareable(struct vm_area_struct *svma,
6610 				struct vm_area_struct *vma,
6611 				unsigned long addr, pgoff_t idx)
6612 {
6613 	unsigned long saddr = ((idx - svma->vm_pgoff) << PAGE_SHIFT) +
6614 				svma->vm_start;
6615 	unsigned long sbase = saddr & PUD_MASK;
6616 	unsigned long s_end = sbase + PUD_SIZE;
6617 
6618 	/* Allow segments to share if only one is marked locked */
6619 	unsigned long vm_flags = vma->vm_flags & VM_LOCKED_CLEAR_MASK;
6620 	unsigned long svm_flags = svma->vm_flags & VM_LOCKED_CLEAR_MASK;
6621 
6622 	/*
6623 	 * match the virtual addresses, permission and the alignment of the
6624 	 * page table page.
6625 	 */
6626 	if (pmd_index(addr) != pmd_index(saddr) ||
6627 	    vm_flags != svm_flags ||
6628 	    !range_in_vma(svma, sbase, s_end))
6629 		return 0;
6630 
6631 	return saddr;
6632 }
6633 
6634 static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
6635 {
6636 	unsigned long base = addr & PUD_MASK;
6637 	unsigned long end = base + PUD_SIZE;
6638 
6639 	/*
6640 	 * check on proper vm_flags and page table alignment
6641 	 */
6642 	if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
6643 		return true;
6644 	return false;
6645 }
6646 
6647 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
6648 {
6649 #ifdef CONFIG_USERFAULTFD
6650 	if (uffd_disable_huge_pmd_share(vma))
6651 		return false;
6652 #endif
6653 	return vma_shareable(vma, addr);
6654 }
6655 
6656 /*
6657  * Determine if start,end range within vma could be mapped by shared pmd.
6658  * If yes, adjust start and end to cover range associated with possible
6659  * shared pmd mappings.
6660  */
6661 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
6662 				unsigned long *start, unsigned long *end)
6663 {
6664 	unsigned long v_start = ALIGN(vma->vm_start, PUD_SIZE),
6665 		v_end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
6666 
6667 	/*
6668 	 * vma needs to span at least one aligned PUD size, and the range
6669 	 * must be at least partially within in.
6670 	 */
6671 	if (!(vma->vm_flags & VM_MAYSHARE) || !(v_end > v_start) ||
6672 		(*end <= v_start) || (*start >= v_end))
6673 		return;
6674 
6675 	/* Extend the range to be PUD aligned for a worst case scenario */
6676 	if (*start > v_start)
6677 		*start = ALIGN_DOWN(*start, PUD_SIZE);
6678 
6679 	if (*end < v_end)
6680 		*end = ALIGN(*end, PUD_SIZE);
6681 }
6682 
6683 /*
6684  * Search for a shareable pmd page for hugetlb. In any case calls pmd_alloc()
6685  * and returns the corresponding pte. While this is not necessary for the
6686  * !shared pmd case because we can allocate the pmd later as well, it makes the
6687  * code much cleaner.
6688  *
6689  * This routine must be called with i_mmap_rwsem held in at least read mode if
6690  * sharing is possible.  For hugetlbfs, this prevents removal of any page
6691  * table entries associated with the address space.  This is important as we
6692  * are setting up sharing based on existing page table entries (mappings).
6693  */
6694 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
6695 		      unsigned long addr, pud_t *pud)
6696 {
6697 	struct address_space *mapping = vma->vm_file->f_mapping;
6698 	pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
6699 			vma->vm_pgoff;
6700 	struct vm_area_struct *svma;
6701 	unsigned long saddr;
6702 	pte_t *spte = NULL;
6703 	pte_t *pte;
6704 	spinlock_t *ptl;
6705 
6706 	i_mmap_assert_locked(mapping);
6707 	vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
6708 		if (svma == vma)
6709 			continue;
6710 
6711 		saddr = page_table_shareable(svma, vma, addr, idx);
6712 		if (saddr) {
6713 			spte = huge_pte_offset(svma->vm_mm, saddr,
6714 					       vma_mmu_pagesize(svma));
6715 			if (spte) {
6716 				get_page(virt_to_page(spte));
6717 				break;
6718 			}
6719 		}
6720 	}
6721 
6722 	if (!spte)
6723 		goto out;
6724 
6725 	ptl = huge_pte_lock(hstate_vma(vma), mm, spte);
6726 	if (pud_none(*pud)) {
6727 		pud_populate(mm, pud,
6728 				(pmd_t *)((unsigned long)spte & PAGE_MASK));
6729 		mm_inc_nr_pmds(mm);
6730 	} else {
6731 		put_page(virt_to_page(spte));
6732 	}
6733 	spin_unlock(ptl);
6734 out:
6735 	pte = (pte_t *)pmd_alloc(mm, pud, addr);
6736 	return pte;
6737 }
6738 
6739 /*
6740  * unmap huge page backed by shared pte.
6741  *
6742  * Hugetlb pte page is ref counted at the time of mapping.  If pte is shared
6743  * indicated by page_count > 1, unmap is achieved by clearing pud and
6744  * decrementing the ref count. If count == 1, the pte page is not shared.
6745  *
6746  * Called with page table lock held and i_mmap_rwsem held in write mode.
6747  *
6748  * returns: 1 successfully unmapped a shared pte page
6749  *	    0 the underlying pte page is not shared, or it is the last user
6750  */
6751 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
6752 					unsigned long *addr, pte_t *ptep)
6753 {
6754 	pgd_t *pgd = pgd_offset(mm, *addr);
6755 	p4d_t *p4d = p4d_offset(pgd, *addr);
6756 	pud_t *pud = pud_offset(p4d, *addr);
6757 
6758 	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
6759 	BUG_ON(page_count(virt_to_page(ptep)) == 0);
6760 	if (page_count(virt_to_page(ptep)) == 1)
6761 		return 0;
6762 
6763 	pud_clear(pud);
6764 	put_page(virt_to_page(ptep));
6765 	mm_dec_nr_pmds(mm);
6766 	/*
6767 	 * This update of passed address optimizes loops sequentially
6768 	 * processing addresses in increments of huge page size (PMD_SIZE
6769 	 * in this case).  By clearing the pud, a PUD_SIZE area is unmapped.
6770 	 * Update address to the 'last page' in the cleared area so that
6771 	 * calling loop can move to first page past this area.
6772 	 */
6773 	*addr |= PUD_SIZE - PMD_SIZE;
6774 	return 1;
6775 }
6776 
6777 #else /* !CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
6778 pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
6779 		      unsigned long addr, pud_t *pud)
6780 {
6781 	return NULL;
6782 }
6783 
6784 int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
6785 				unsigned long *addr, pte_t *ptep)
6786 {
6787 	return 0;
6788 }
6789 
6790 void adjust_range_if_pmd_sharing_possible(struct vm_area_struct *vma,
6791 				unsigned long *start, unsigned long *end)
6792 {
6793 }
6794 
6795 bool want_pmd_share(struct vm_area_struct *vma, unsigned long addr)
6796 {
6797 	return false;
6798 }
6799 #endif /* CONFIG_ARCH_WANT_HUGE_PMD_SHARE */
6800 
6801 #ifdef CONFIG_ARCH_WANT_GENERAL_HUGETLB
6802 pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
6803 			unsigned long addr, unsigned long sz)
6804 {
6805 	pgd_t *pgd;
6806 	p4d_t *p4d;
6807 	pud_t *pud;
6808 	pte_t *pte = NULL;
6809 
6810 	pgd = pgd_offset(mm, addr);
6811 	p4d = p4d_alloc(mm, pgd, addr);
6812 	if (!p4d)
6813 		return NULL;
6814 	pud = pud_alloc(mm, p4d, addr);
6815 	if (pud) {
6816 		if (sz == PUD_SIZE) {
6817 			pte = (pte_t *)pud;
6818 		} else {
6819 			BUG_ON(sz != PMD_SIZE);
6820 			if (want_pmd_share(vma, addr) && pud_none(*pud))
6821 				pte = huge_pmd_share(mm, vma, addr, pud);
6822 			else
6823 				pte = (pte_t *)pmd_alloc(mm, pud, addr);
6824 		}
6825 	}
6826 	BUG_ON(pte && pte_present(*pte) && !pte_huge(*pte));
6827 
6828 	return pte;
6829 }
6830 
6831 /*
6832  * huge_pte_offset() - Walk the page table to resolve the hugepage
6833  * entry at address @addr
6834  *
6835  * Return: Pointer to page table entry (PUD or PMD) for
6836  * address @addr, or NULL if a !p*d_present() entry is encountered and the
6837  * size @sz doesn't match the hugepage size at this level of the page
6838  * table.
6839  */
6840 pte_t *huge_pte_offset(struct mm_struct *mm,
6841 		       unsigned long addr, unsigned long sz)
6842 {
6843 	pgd_t *pgd;
6844 	p4d_t *p4d;
6845 	pud_t *pud;
6846 	pmd_t *pmd;
6847 
6848 	pgd = pgd_offset(mm, addr);
6849 	if (!pgd_present(*pgd))
6850 		return NULL;
6851 	p4d = p4d_offset(pgd, addr);
6852 	if (!p4d_present(*p4d))
6853 		return NULL;
6854 
6855 	pud = pud_offset(p4d, addr);
6856 	if (sz == PUD_SIZE)
6857 		/* must be pud huge, non-present or none */
6858 		return (pte_t *)pud;
6859 	if (!pud_present(*pud))
6860 		return NULL;
6861 	/* must have a valid entry and size to go further */
6862 
6863 	pmd = pmd_offset(pud, addr);
6864 	/* must be pmd huge, non-present or none */
6865 	return (pte_t *)pmd;
6866 }
6867 
6868 #endif /* CONFIG_ARCH_WANT_GENERAL_HUGETLB */
6869 
6870 /*
6871  * These functions are overwritable if your architecture needs its own
6872  * behavior.
6873  */
6874 struct page * __weak
6875 follow_huge_addr(struct mm_struct *mm, unsigned long address,
6876 			      int write)
6877 {
6878 	return ERR_PTR(-EINVAL);
6879 }
6880 
6881 struct page * __weak
6882 follow_huge_pd(struct vm_area_struct *vma,
6883 	       unsigned long address, hugepd_t hpd, int flags, int pdshift)
6884 {
6885 	WARN(1, "hugepd follow called with no support for hugepage directory format\n");
6886 	return NULL;
6887 }
6888 
6889 struct page * __weak
6890 follow_huge_pmd(struct mm_struct *mm, unsigned long address,
6891 		pmd_t *pmd, int flags)
6892 {
6893 	struct page *page = NULL;
6894 	spinlock_t *ptl;
6895 	pte_t pte;
6896 
6897 	/*
6898 	 * FOLL_PIN is not supported for follow_page(). Ordinary GUP goes via
6899 	 * follow_hugetlb_page().
6900 	 */
6901 	if (WARN_ON_ONCE(flags & FOLL_PIN))
6902 		return NULL;
6903 
6904 retry:
6905 	ptl = pmd_lockptr(mm, pmd);
6906 	spin_lock(ptl);
6907 	/*
6908 	 * make sure that the address range covered by this pmd is not
6909 	 * unmapped from other threads.
6910 	 */
6911 	if (!pmd_huge(*pmd))
6912 		goto out;
6913 	pte = huge_ptep_get((pte_t *)pmd);
6914 	if (pte_present(pte)) {
6915 		page = pmd_page(*pmd) + ((address & ~PMD_MASK) >> PAGE_SHIFT);
6916 		/*
6917 		 * try_grab_page() should always succeed here, because: a) we
6918 		 * hold the pmd (ptl) lock, and b) we've just checked that the
6919 		 * huge pmd (head) page is present in the page tables. The ptl
6920 		 * prevents the head page and tail pages from being rearranged
6921 		 * in any way. So this page must be available at this point,
6922 		 * unless the page refcount overflowed:
6923 		 */
6924 		if (WARN_ON_ONCE(!try_grab_page(page, flags))) {
6925 			page = NULL;
6926 			goto out;
6927 		}
6928 	} else {
6929 		if (is_hugetlb_entry_migration(pte)) {
6930 			spin_unlock(ptl);
6931 			__migration_entry_wait(mm, (pte_t *)pmd, ptl);
6932 			goto retry;
6933 		}
6934 		/*
6935 		 * hwpoisoned entry is treated as no_page_table in
6936 		 * follow_page_mask().
6937 		 */
6938 	}
6939 out:
6940 	spin_unlock(ptl);
6941 	return page;
6942 }
6943 
6944 struct page * __weak
6945 follow_huge_pud(struct mm_struct *mm, unsigned long address,
6946 		pud_t *pud, int flags)
6947 {
6948 	if (flags & (FOLL_GET | FOLL_PIN))
6949 		return NULL;
6950 
6951 	return pte_page(*(pte_t *)pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
6952 }
6953 
6954 struct page * __weak
6955 follow_huge_pgd(struct mm_struct *mm, unsigned long address, pgd_t *pgd, int flags)
6956 {
6957 	if (flags & (FOLL_GET | FOLL_PIN))
6958 		return NULL;
6959 
6960 	return pte_page(*(pte_t *)pgd) + ((address & ~PGDIR_MASK) >> PAGE_SHIFT);
6961 }
6962 
6963 bool isolate_huge_page(struct page *page, struct list_head *list)
6964 {
6965 	bool ret = true;
6966 
6967 	spin_lock_irq(&hugetlb_lock);
6968 	if (!PageHeadHuge(page) ||
6969 	    !HPageMigratable(page) ||
6970 	    !get_page_unless_zero(page)) {
6971 		ret = false;
6972 		goto unlock;
6973 	}
6974 	ClearHPageMigratable(page);
6975 	list_move_tail(&page->lru, list);
6976 unlock:
6977 	spin_unlock_irq(&hugetlb_lock);
6978 	return ret;
6979 }
6980 
6981 int get_hwpoison_huge_page(struct page *page, bool *hugetlb)
6982 {
6983 	int ret = 0;
6984 
6985 	*hugetlb = false;
6986 	spin_lock_irq(&hugetlb_lock);
6987 	if (PageHeadHuge(page)) {
6988 		*hugetlb = true;
6989 		if (HPageFreed(page))
6990 			ret = 0;
6991 		else if (HPageMigratable(page))
6992 			ret = get_page_unless_zero(page);
6993 		else
6994 			ret = -EBUSY;
6995 	}
6996 	spin_unlock_irq(&hugetlb_lock);
6997 	return ret;
6998 }
6999 
7000 int get_huge_page_for_hwpoison(unsigned long pfn, int flags)
7001 {
7002 	int ret;
7003 
7004 	spin_lock_irq(&hugetlb_lock);
7005 	ret = __get_huge_page_for_hwpoison(pfn, flags);
7006 	spin_unlock_irq(&hugetlb_lock);
7007 	return ret;
7008 }
7009 
7010 void putback_active_hugepage(struct page *page)
7011 {
7012 	spin_lock_irq(&hugetlb_lock);
7013 	SetHPageMigratable(page);
7014 	list_move_tail(&page->lru, &(page_hstate(page))->hugepage_activelist);
7015 	spin_unlock_irq(&hugetlb_lock);
7016 	put_page(page);
7017 }
7018 
7019 void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason)
7020 {
7021 	struct hstate *h = page_hstate(oldpage);
7022 
7023 	hugetlb_cgroup_migrate(oldpage, newpage);
7024 	set_page_owner_migrate_reason(newpage, reason);
7025 
7026 	/*
7027 	 * transfer temporary state of the new huge page. This is
7028 	 * reverse to other transitions because the newpage is going to
7029 	 * be final while the old one will be freed so it takes over
7030 	 * the temporary status.
7031 	 *
7032 	 * Also note that we have to transfer the per-node surplus state
7033 	 * here as well otherwise the global surplus count will not match
7034 	 * the per-node's.
7035 	 */
7036 	if (HPageTemporary(newpage)) {
7037 		int old_nid = page_to_nid(oldpage);
7038 		int new_nid = page_to_nid(newpage);
7039 
7040 		SetHPageTemporary(oldpage);
7041 		ClearHPageTemporary(newpage);
7042 
7043 		/*
7044 		 * There is no need to transfer the per-node surplus state
7045 		 * when we do not cross the node.
7046 		 */
7047 		if (new_nid == old_nid)
7048 			return;
7049 		spin_lock_irq(&hugetlb_lock);
7050 		if (h->surplus_huge_pages_node[old_nid]) {
7051 			h->surplus_huge_pages_node[old_nid]--;
7052 			h->surplus_huge_pages_node[new_nid]++;
7053 		}
7054 		spin_unlock_irq(&hugetlb_lock);
7055 	}
7056 }
7057 
7058 /*
7059  * This function will unconditionally remove all the shared pmd pgtable entries
7060  * within the specific vma for a hugetlbfs memory range.
7061  */
7062 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma)
7063 {
7064 	struct hstate *h = hstate_vma(vma);
7065 	unsigned long sz = huge_page_size(h);
7066 	struct mm_struct *mm = vma->vm_mm;
7067 	struct mmu_notifier_range range;
7068 	unsigned long address, start, end;
7069 	spinlock_t *ptl;
7070 	pte_t *ptep;
7071 
7072 	if (!(vma->vm_flags & VM_MAYSHARE))
7073 		return;
7074 
7075 	start = ALIGN(vma->vm_start, PUD_SIZE);
7076 	end = ALIGN_DOWN(vma->vm_end, PUD_SIZE);
7077 
7078 	if (start >= end)
7079 		return;
7080 
7081 	flush_cache_range(vma, start, end);
7082 	/*
7083 	 * No need to call adjust_range_if_pmd_sharing_possible(), because
7084 	 * we have already done the PUD_SIZE alignment.
7085 	 */
7086 	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
7087 				start, end);
7088 	mmu_notifier_invalidate_range_start(&range);
7089 	i_mmap_lock_write(vma->vm_file->f_mapping);
7090 	for (address = start; address < end; address += PUD_SIZE) {
7091 		unsigned long tmp = address;
7092 
7093 		ptep = huge_pte_offset(mm, address, sz);
7094 		if (!ptep)
7095 			continue;
7096 		ptl = huge_pte_lock(h, mm, ptep);
7097 		/* We don't want 'address' to be changed */
7098 		huge_pmd_unshare(mm, vma, &tmp, ptep);
7099 		spin_unlock(ptl);
7100 	}
7101 	flush_hugetlb_tlb_range(vma, start, end);
7102 	i_mmap_unlock_write(vma->vm_file->f_mapping);
7103 	/*
7104 	 * No need to call mmu_notifier_invalidate_range(), see
7105 	 * Documentation/vm/mmu_notifier.rst.
7106 	 */
7107 	mmu_notifier_invalidate_range_end(&range);
7108 }
7109 
7110 #ifdef CONFIG_CMA
7111 static bool cma_reserve_called __initdata;
7112 
7113 static int __init cmdline_parse_hugetlb_cma(char *p)
7114 {
7115 	int nid, count = 0;
7116 	unsigned long tmp;
7117 	char *s = p;
7118 
7119 	while (*s) {
7120 		if (sscanf(s, "%lu%n", &tmp, &count) != 1)
7121 			break;
7122 
7123 		if (s[count] == ':') {
7124 			if (tmp >= MAX_NUMNODES)
7125 				break;
7126 			nid = array_index_nospec(tmp, MAX_NUMNODES);
7127 
7128 			s += count + 1;
7129 			tmp = memparse(s, &s);
7130 			hugetlb_cma_size_in_node[nid] = tmp;
7131 			hugetlb_cma_size += tmp;
7132 
7133 			/*
7134 			 * Skip the separator if have one, otherwise
7135 			 * break the parsing.
7136 			 */
7137 			if (*s == ',')
7138 				s++;
7139 			else
7140 				break;
7141 		} else {
7142 			hugetlb_cma_size = memparse(p, &p);
7143 			break;
7144 		}
7145 	}
7146 
7147 	return 0;
7148 }
7149 
7150 early_param("hugetlb_cma", cmdline_parse_hugetlb_cma);
7151 
7152 void __init hugetlb_cma_reserve(int order)
7153 {
7154 	unsigned long size, reserved, per_node;
7155 	bool node_specific_cma_alloc = false;
7156 	int nid;
7157 
7158 	cma_reserve_called = true;
7159 
7160 	if (!hugetlb_cma_size)
7161 		return;
7162 
7163 	for (nid = 0; nid < MAX_NUMNODES; nid++) {
7164 		if (hugetlb_cma_size_in_node[nid] == 0)
7165 			continue;
7166 
7167 		if (!node_online(nid)) {
7168 			pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
7169 			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
7170 			hugetlb_cma_size_in_node[nid] = 0;
7171 			continue;
7172 		}
7173 
7174 		if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) {
7175 			pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n",
7176 				nid, (PAGE_SIZE << order) / SZ_1M);
7177 			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
7178 			hugetlb_cma_size_in_node[nid] = 0;
7179 		} else {
7180 			node_specific_cma_alloc = true;
7181 		}
7182 	}
7183 
7184 	/* Validate the CMA size again in case some invalid nodes specified. */
7185 	if (!hugetlb_cma_size)
7186 		return;
7187 
7188 	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
7189 		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
7190 			(PAGE_SIZE << order) / SZ_1M);
7191 		hugetlb_cma_size = 0;
7192 		return;
7193 	}
7194 
7195 	if (!node_specific_cma_alloc) {
7196 		/*
7197 		 * If 3 GB area is requested on a machine with 4 numa nodes,
7198 		 * let's allocate 1 GB on first three nodes and ignore the last one.
7199 		 */
7200 		per_node = DIV_ROUND_UP(hugetlb_cma_size, nr_online_nodes);
7201 		pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
7202 			hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
7203 	}
7204 
7205 	reserved = 0;
7206 	for_each_online_node(nid) {
7207 		int res;
7208 		char name[CMA_MAX_NAME];
7209 
7210 		if (node_specific_cma_alloc) {
7211 			if (hugetlb_cma_size_in_node[nid] == 0)
7212 				continue;
7213 
7214 			size = hugetlb_cma_size_in_node[nid];
7215 		} else {
7216 			size = min(per_node, hugetlb_cma_size - reserved);
7217 		}
7218 
7219 		size = round_up(size, PAGE_SIZE << order);
7220 
7221 		snprintf(name, sizeof(name), "hugetlb%d", nid);
7222 		/*
7223 		 * Note that 'order per bit' is based on smallest size that
7224 		 * may be returned to CMA allocator in the case of
7225 		 * huge page demotion.
7226 		 */
7227 		res = cma_declare_contiguous_nid(0, size, 0,
7228 						PAGE_SIZE << HUGETLB_PAGE_ORDER,
7229 						 0, false, name,
7230 						 &hugetlb_cma[nid], nid);
7231 		if (res) {
7232 			pr_warn("hugetlb_cma: reservation failed: err %d, node %d",
7233 				res, nid);
7234 			continue;
7235 		}
7236 
7237 		reserved += size;
7238 		pr_info("hugetlb_cma: reserved %lu MiB on node %d\n",
7239 			size / SZ_1M, nid);
7240 
7241 		if (reserved >= hugetlb_cma_size)
7242 			break;
7243 	}
7244 
7245 	if (!reserved)
7246 		/*
7247 		 * hugetlb_cma_size is used to determine if allocations from
7248 		 * cma are possible.  Set to zero if no cma regions are set up.
7249 		 */
7250 		hugetlb_cma_size = 0;
7251 }
7252 
7253 void __init hugetlb_cma_check(void)
7254 {
7255 	if (!hugetlb_cma_size || cma_reserve_called)
7256 		return;
7257 
7258 	pr_warn("hugetlb_cma: the option isn't supported by current arch\n");
7259 }
7260 
7261 #endif /* CONFIG_CMA */
7262