1 // SPDX-License-Identifier: GPL-2.0
22 	if (!c->parent)  in propagate_protected_usage()
25 	protected = min(usage, READ_ONCE(c->min));  in propagate_protected_usage()
26 	old_protected = atomic_long_read(&c->min_usage);  in propagate_protected_usage()
28 		old_protected = atomic_long_xchg(&c->min_usage, protected);  in propagate_protected_usage()
29 		delta = protected - old_protected;  in propagate_protected_usage()
31 			atomic_long_add(delta, &c->parent->children_min_usage);  in propagate_protected_usage()
34 	protected = min(usage, READ_ONCE(c->low));  in propagate_protected_usage()
35 	old_protected = atomic_long_read(&c->low_usage);  in propagate_protected_usage()
37 		old_protected = atomic_long_xchg(&c->low_usage, protected);  in propagate_protected_usage()
38 		delta = protected - old_protected;  in propagate_protected_usage()
40 			atomic_long_add(delta, &c->parent->children_low_usage);  in propagate_protected_usage()
45  * page_counter_cancel - take pages out of the local counter
47  * @nr_pages: number of pages to cancel
53 	new = atomic_long_sub_return(nr_pages, &counter->usage);  in page_counter_cancel()
58 		atomic_long_set(&counter->usage, new);  in page_counter_cancel()
64  * page_counter_charge - hierarchically charge pages
66  * @nr_pages: number of pages to charge
74 	for (c = counter; c; c = c->parent) {  in page_counter_charge()
77 		new = atomic_long_add_return(nr_pages, &c->usage);  in page_counter_charge()
83 		if (new > READ_ONCE(c->watermark))  in page_counter_charge()
84 			WRITE_ONCE(c->watermark, new);  in page_counter_charge()
89  * page_counter_try_charge - try to hierarchically charge pages
91  * @nr_pages: number of pages to charge
95  * of its ancestors has hit its configured limit.
103 	for (c = counter; c; c = c->parent) {  in page_counter_try_charge()
106 		 * Charge speculatively to avoid an expensive CAS.  If  in page_counter_try_charge()
107 		 * a bigger charge fails, it might falsely lock out a  in page_counter_try_charge()
108 		 * racing smaller charge and send it into reclaim  in page_counter_try_charge()
111 		 * case of a THP locking out a regular page charge.  in page_counter_try_charge()
119 		new = atomic_long_add_return(nr_pages, &c->usage);  in page_counter_try_charge()
120 		if (new > c->max) {  in page_counter_try_charge()
121 			atomic_long_sub(nr_pages, &c->usage);  in page_counter_try_charge()
127 			data_race(c->failcnt++);  in page_counter_try_charge()
136 		if (new > READ_ONCE(c->watermark))  in page_counter_try_charge()
137 			WRITE_ONCE(c->watermark, new);  in page_counter_try_charge()
142 	for (c = counter; c != *fail; c = c->parent)  in page_counter_try_charge()
149  * page_counter_uncharge - hierarchically uncharge pages
151  * @nr_pages: number of pages to uncharge
157 	for (c = counter; c; c = c->parent)  in page_counter_uncharge()
162  * page_counter_set_max - set the maximum number of pages allowed
166  * Returns 0 on success, -EBUSY if the current number of pages on the
179 		 * below the concurrently-changing counter value.  in page_counter_set_max()
182 		 * and after, so the read-swap-read is ordered and  in page_counter_set_max()
191 			return -EBUSY;  in page_counter_set_max()
193 		old = xchg(&counter->max, nr_pages);  in page_counter_set_max()
198 		counter->max = old;  in page_counter_set_max()
204  * page_counter_set_min - set the amount of protected memory
214 	WRITE_ONCE(counter->min, nr_pages);  in page_counter_set_min()
216 	for (c = counter; c; c = c->parent)  in page_counter_set_min()
217 		propagate_protected_usage(c, atomic_long_read(&c->usage));  in page_counter_set_min()
221  * page_counter_set_low - set the amount of protected memory
231 	WRITE_ONCE(counter->low, nr_pages);  in page_counter_set_low()
233 	for (c = counter; c; c = c->parent)  in page_counter_set_low()
234 		propagate_protected_usage(c, atomic_long_read(&c->usage));  in page_counter_set_low()
238  * page_counter_memparse - memparse() for page counter limits
241  * @nr_pages: returns the result in number of pages
243  * Returns -EINVAL, or 0 and @nr_pages on success.  @nr_pages will be
249 	char *end;  in page_counter_memparse()  local
257 	bytes = memparse(buf, &end);  in page_counter_memparse()
258 	if (*end != '\0')  in page_counter_memparse()
259 		return -EINVAL;  in page_counter_memparse()