xref: /openbmc/linux/mm/damon/paddr.c (revision 9b68f30b) 
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * DAMON Primitives for The Physical Address Space
4  *
5  * Author: SeongJae Park <sj@kernel.org>
6  */
7 
8 #define pr_fmt(fmt) "damon-pa: " fmt
9 
10 #include <linux/mmu_notifier.h>
11 #include <linux/page_idle.h>
12 #include <linux/pagemap.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15 
16 #include "../internal.h"
17 #include "ops-common.h"
18 
19 static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma,
20 		unsigned long addr, void *arg)
21 {
22 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
23 
24 	while (page_vma_mapped_walk(&pvmw)) {
25 		addr = pvmw.address;
26 		if (pvmw.pte)
27 			damon_ptep_mkold(pvmw.pte, vma->vm_mm, addr);
28 		else
29 			damon_pmdp_mkold(pvmw.pmd, vma->vm_mm, addr);
30 	}
31 	return true;
32 }
33 
34 static void damon_pa_mkold(unsigned long paddr)
35 {
36 	struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
37 	struct rmap_walk_control rwc = {
38 		.rmap_one = __damon_pa_mkold,
39 		.anon_lock = folio_lock_anon_vma_read,
40 	};
41 	bool need_lock;
42 
43 	if (!folio)
44 		return;
45 
46 	if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
47 		folio_set_idle(folio);
48 		goto out;
49 	}
50 
51 	need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
52 	if (need_lock && !folio_trylock(folio))
53 		goto out;
54 
55 	rmap_walk(folio, &rwc);
56 
57 	if (need_lock)
58 		folio_unlock(folio);
59 
60 out:
61 	folio_put(folio);
62 }
63 
64 static void __damon_pa_prepare_access_check(struct damon_region *r)
65 {
66 	r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
67 
68 	damon_pa_mkold(r->sampling_addr);
69 }
70 
71 static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
72 {
73 	struct damon_target *t;
74 	struct damon_region *r;
75 
76 	damon_for_each_target(t, ctx) {
77 		damon_for_each_region(r, t)
78 			__damon_pa_prepare_access_check(r);
79 	}
80 }
81 
82 static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma,
83 		unsigned long addr, void *arg)
84 {
85 	bool *accessed = arg;
86 	DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
87 
88 	*accessed = false;
89 	while (page_vma_mapped_walk(&pvmw)) {
90 		addr = pvmw.address;
91 		if (pvmw.pte) {
92 			*accessed = pte_young(*pvmw.pte) ||
93 				!folio_test_idle(folio) ||
94 				mmu_notifier_test_young(vma->vm_mm, addr);
95 		} else {
96 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
97 			*accessed = pmd_young(*pvmw.pmd) ||
98 				!folio_test_idle(folio) ||
99 				mmu_notifier_test_young(vma->vm_mm, addr);
100 #else
101 			WARN_ON_ONCE(1);
102 #endif	/* CONFIG_TRANSPARENT_HUGEPAGE */
103 		}
104 		if (*accessed) {
105 			page_vma_mapped_walk_done(&pvmw);
106 			break;
107 		}
108 	}
109 
110 	/* If accessed, stop walking */
111 	return *accessed == false;
112 }
113 
114 static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz)
115 {
116 	struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
117 	bool accessed = false;
118 	struct rmap_walk_control rwc = {
119 		.arg = &accessed,
120 		.rmap_one = __damon_pa_young,
121 		.anon_lock = folio_lock_anon_vma_read,
122 	};
123 	bool need_lock;
124 
125 	if (!folio)
126 		return false;
127 
128 	if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
129 		if (folio_test_idle(folio))
130 			accessed = false;
131 		else
132 			accessed = true;
133 		goto out;
134 	}
135 
136 	need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
137 	if (need_lock && !folio_trylock(folio))
138 		goto out;
139 
140 	rmap_walk(folio, &rwc);
141 
142 	if (need_lock)
143 		folio_unlock(folio);
144 
145 out:
146 	*folio_sz = folio_size(folio);
147 	folio_put(folio);
148 	return accessed;
149 }
150 
151 static void __damon_pa_check_access(struct damon_region *r)
152 {
153 	static unsigned long last_addr;
154 	static unsigned long last_folio_sz = PAGE_SIZE;
155 	static bool last_accessed;
156 
157 	/* If the region is in the last checked page, reuse the result */
158 	if (ALIGN_DOWN(last_addr, last_folio_sz) ==
159 				ALIGN_DOWN(r->sampling_addr, last_folio_sz)) {
160 		if (last_accessed)
161 			r->nr_accesses++;
162 		return;
163 	}
164 
165 	last_accessed = damon_pa_young(r->sampling_addr, &last_folio_sz);
166 	if (last_accessed)
167 		r->nr_accesses++;
168 
169 	last_addr = r->sampling_addr;
170 }
171 
172 static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
173 {
174 	struct damon_target *t;
175 	struct damon_region *r;
176 	unsigned int max_nr_accesses = 0;
177 
178 	damon_for_each_target(t, ctx) {
179 		damon_for_each_region(r, t) {
180 			__damon_pa_check_access(r);
181 			max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
182 		}
183 	}
184 
185 	return max_nr_accesses;
186 }
187 
188 static bool __damos_pa_filter_out(struct damos_filter *filter,
189 		struct folio *folio)
190 {
191 	bool matched = false;
192 	struct mem_cgroup *memcg;
193 
194 	switch (filter->type) {
195 	case DAMOS_FILTER_TYPE_ANON:
196 		matched = folio_test_anon(folio);
197 		break;
198 	case DAMOS_FILTER_TYPE_MEMCG:
199 		rcu_read_lock();
200 		memcg = folio_memcg_check(folio);
201 		if (!memcg)
202 			matched = false;
203 		else
204 			matched = filter->memcg_id == mem_cgroup_id(memcg);
205 		rcu_read_unlock();
206 		break;
207 	default:
208 		break;
209 	}
210 
211 	return matched == filter->matching;
212 }
213 
214 /*
215  * damos_pa_filter_out - Return true if the page should be filtered out.
216  */
217 static bool damos_pa_filter_out(struct damos *scheme, struct folio *folio)
218 {
219 	struct damos_filter *filter;
220 
221 	damos_for_each_filter(filter, scheme) {
222 		if (__damos_pa_filter_out(filter, folio))
223 			return true;
224 	}
225 	return false;
226 }
227 
228 static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s)
229 {
230 	unsigned long addr, applied;
231 	LIST_HEAD(folio_list);
232 
233 	for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
234 		struct folio *folio = damon_get_folio(PHYS_PFN(addr));
235 
236 		if (!folio)
237 			continue;
238 
239 		if (damos_pa_filter_out(s, folio))
240 			goto put_folio;
241 
242 		folio_clear_referenced(folio);
243 		folio_test_clear_young(folio);
244 		if (!folio_isolate_lru(folio))
245 			goto put_folio;
246 		if (folio_test_unevictable(folio))
247 			folio_putback_lru(folio);
248 		else
249 			list_add(&folio->lru, &folio_list);
250 put_folio:
251 		folio_put(folio);
252 	}
253 	applied = reclaim_pages(&folio_list);
254 	cond_resched();
255 	return applied * PAGE_SIZE;
256 }
257 
258 static inline unsigned long damon_pa_mark_accessed_or_deactivate(
259 		struct damon_region *r, struct damos *s, bool mark_accessed)
260 {
261 	unsigned long addr, applied = 0;
262 
263 	for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
264 		struct folio *folio = damon_get_folio(PHYS_PFN(addr));
265 
266 		if (!folio)
267 			continue;
268 
269 		if (damos_pa_filter_out(s, folio))
270 			goto put_folio;
271 
272 		if (mark_accessed)
273 			folio_mark_accessed(folio);
274 		else
275 			folio_deactivate(folio);
276 		applied += folio_nr_pages(folio);
277 put_folio:
278 		folio_put(folio);
279 	}
280 	return applied * PAGE_SIZE;
281 }
282 
283 static unsigned long damon_pa_mark_accessed(struct damon_region *r,
284 	struct damos *s)
285 {
286 	return damon_pa_mark_accessed_or_deactivate(r, s, true);
287 }
288 
289 static unsigned long damon_pa_deactivate_pages(struct damon_region *r,
290 	struct damos *s)
291 {
292 	return damon_pa_mark_accessed_or_deactivate(r, s, false);
293 }
294 
295 static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
296 		struct damon_target *t, struct damon_region *r,
297 		struct damos *scheme)
298 {
299 	switch (scheme->action) {
300 	case DAMOS_PAGEOUT:
301 		return damon_pa_pageout(r, scheme);
302 	case DAMOS_LRU_PRIO:
303 		return damon_pa_mark_accessed(r, scheme);
304 	case DAMOS_LRU_DEPRIO:
305 		return damon_pa_deactivate_pages(r, scheme);
306 	case DAMOS_STAT:
307 		break;
308 	default:
309 		/* DAMOS actions that not yet supported by 'paddr'. */
310 		break;
311 	}
312 	return 0;
313 }
314 
315 static int damon_pa_scheme_score(struct damon_ctx *context,
316 		struct damon_target *t, struct damon_region *r,
317 		struct damos *scheme)
318 {
319 	switch (scheme->action) {
320 	case DAMOS_PAGEOUT:
321 		return damon_cold_score(context, r, scheme);
322 	case DAMOS_LRU_PRIO:
323 		return damon_hot_score(context, r, scheme);
324 	case DAMOS_LRU_DEPRIO:
325 		return damon_cold_score(context, r, scheme);
326 	default:
327 		break;
328 	}
329 
330 	return DAMOS_MAX_SCORE;
331 }
332 
333 static int __init damon_pa_initcall(void)
334 {
335 	struct damon_operations ops = {
336 		.id = DAMON_OPS_PADDR,
337 		.init = NULL,
338 		.update = NULL,
339 		.prepare_access_checks = damon_pa_prepare_access_checks,
340 		.check_accesses = damon_pa_check_accesses,
341 		.reset_aggregated = NULL,
342 		.target_valid = NULL,
343 		.cleanup = NULL,
344 		.apply_scheme = damon_pa_apply_scheme,
345 		.get_scheme_score = damon_pa_scheme_score,
346 	};
347 
348 	return damon_register_ops(&ops);
349 };
350 
351 subsys_initcall(damon_pa_initcall);
352