1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * DAMON Primitives for The Physical Address Space
4 *
5 * Author: SeongJae Park <sj@kernel.org>
6 */
7
8 #define pr_fmt(fmt) "damon-pa: " fmt
9
10 #include <linux/mmu_notifier.h>
11 #include <linux/page_idle.h>
12 #include <linux/pagemap.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15
16 #include "../internal.h"
17 #include "ops-common.h"
18
__damon_pa_mkold(struct folio * folio,struct vm_area_struct * vma,unsigned long addr,void * arg)19 static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma,
20 unsigned long addr, void *arg)
21 {
22 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
23
24 while (page_vma_mapped_walk(&pvmw)) {
25 addr = pvmw.address;
26 if (pvmw.pte)
27 damon_ptep_mkold(pvmw.pte, vma, addr);
28 else
29 damon_pmdp_mkold(pvmw.pmd, vma, addr);
30 }
31 return true;
32 }
33
damon_pa_mkold(unsigned long paddr)34 static void damon_pa_mkold(unsigned long paddr)
35 {
36 struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
37 struct rmap_walk_control rwc = {
38 .rmap_one = __damon_pa_mkold,
39 .anon_lock = folio_lock_anon_vma_read,
40 };
41 bool need_lock;
42
43 if (!folio)
44 return;
45
46 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
47 folio_set_idle(folio);
48 goto out;
49 }
50
51 need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
52 if (need_lock && !folio_trylock(folio))
53 goto out;
54
55 rmap_walk(folio, &rwc);
56
57 if (need_lock)
58 folio_unlock(folio);
59
60 out:
61 folio_put(folio);
62 }
63
__damon_pa_prepare_access_check(struct damon_region * r)64 static void __damon_pa_prepare_access_check(struct damon_region *r)
65 {
66 r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
67
68 damon_pa_mkold(r->sampling_addr);
69 }
70
damon_pa_prepare_access_checks(struct damon_ctx * ctx)71 static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
72 {
73 struct damon_target *t;
74 struct damon_region *r;
75
76 damon_for_each_target(t, ctx) {
77 damon_for_each_region(r, t)
78 __damon_pa_prepare_access_check(r);
79 }
80 }
81
__damon_pa_young(struct folio * folio,struct vm_area_struct * vma,unsigned long addr,void * arg)82 static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma,
83 unsigned long addr, void *arg)
84 {
85 bool *accessed = arg;
86 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
87
88 *accessed = false;
89 while (page_vma_mapped_walk(&pvmw)) {
90 addr = pvmw.address;
91 if (pvmw.pte) {
92 *accessed = pte_young(ptep_get(pvmw.pte)) ||
93 !folio_test_idle(folio) ||
94 mmu_notifier_test_young(vma->vm_mm, addr);
95 } else {
96 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
97 *accessed = pmd_young(pmdp_get(pvmw.pmd)) ||
98 !folio_test_idle(folio) ||
99 mmu_notifier_test_young(vma->vm_mm, addr);
100 #else
101 WARN_ON_ONCE(1);
102 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
103 }
104 if (*accessed) {
105 page_vma_mapped_walk_done(&pvmw);
106 break;
107 }
108 }
109
110 /* If accessed, stop walking */
111 return *accessed == false;
112 }
113
damon_pa_young(unsigned long paddr,unsigned long * folio_sz)114 static bool damon_pa_young(unsigned long paddr, unsigned long *folio_sz)
115 {
116 struct folio *folio = damon_get_folio(PHYS_PFN(paddr));
117 bool accessed = false;
118 struct rmap_walk_control rwc = {
119 .arg = &accessed,
120 .rmap_one = __damon_pa_young,
121 .anon_lock = folio_lock_anon_vma_read,
122 };
123 bool need_lock;
124
125 if (!folio)
126 return false;
127
128 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
129 if (folio_test_idle(folio))
130 accessed = false;
131 else
132 accessed = true;
133 goto out;
134 }
135
136 need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
137 if (need_lock && !folio_trylock(folio))
138 goto out;
139
140 rmap_walk(folio, &rwc);
141
142 if (need_lock)
143 folio_unlock(folio);
144
145 out:
146 *folio_sz = folio_size(folio);
147 folio_put(folio);
148 return accessed;
149 }
150
__damon_pa_check_access(struct damon_region * r)151 static void __damon_pa_check_access(struct damon_region *r)
152 {
153 static unsigned long last_addr;
154 static unsigned long last_folio_sz = PAGE_SIZE;
155 static bool last_accessed;
156
157 /* If the region is in the last checked page, reuse the result */
158 if (ALIGN_DOWN(last_addr, last_folio_sz) ==
159 ALIGN_DOWN(r->sampling_addr, last_folio_sz)) {
160 if (last_accessed)
161 r->nr_accesses++;
162 return;
163 }
164
165 last_accessed = damon_pa_young(r->sampling_addr, &last_folio_sz);
166 if (last_accessed)
167 r->nr_accesses++;
168
169 last_addr = r->sampling_addr;
170 }
171
damon_pa_check_accesses(struct damon_ctx * ctx)172 static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
173 {
174 struct damon_target *t;
175 struct damon_region *r;
176 unsigned int max_nr_accesses = 0;
177
178 damon_for_each_target(t, ctx) {
179 damon_for_each_region(r, t) {
180 __damon_pa_check_access(r);
181 max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
182 }
183 }
184
185 return max_nr_accesses;
186 }
187
__damos_pa_filter_out(struct damos_filter * filter,struct folio * folio)188 static bool __damos_pa_filter_out(struct damos_filter *filter,
189 struct folio *folio)
190 {
191 bool matched = false;
192 struct mem_cgroup *memcg;
193
194 switch (filter->type) {
195 case DAMOS_FILTER_TYPE_ANON:
196 matched = folio_test_anon(folio);
197 break;
198 case DAMOS_FILTER_TYPE_MEMCG:
199 rcu_read_lock();
200 memcg = folio_memcg_check(folio);
201 if (!memcg)
202 matched = false;
203 else
204 matched = filter->memcg_id == mem_cgroup_id(memcg);
205 rcu_read_unlock();
206 break;
207 default:
208 break;
209 }
210
211 return matched == filter->matching;
212 }
213
214 /*
215 * damos_pa_filter_out - Return true if the page should be filtered out.
216 */
damos_pa_filter_out(struct damos * scheme,struct folio * folio)217 static bool damos_pa_filter_out(struct damos *scheme, struct folio *folio)
218 {
219 struct damos_filter *filter;
220
221 damos_for_each_filter(filter, scheme) {
222 if (__damos_pa_filter_out(filter, folio))
223 return true;
224 }
225 return false;
226 }
227
damon_pa_pageout(struct damon_region * r,struct damos * s)228 static unsigned long damon_pa_pageout(struct damon_region *r, struct damos *s)
229 {
230 unsigned long addr, applied;
231 LIST_HEAD(folio_list);
232
233 for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
234 struct folio *folio = damon_get_folio(PHYS_PFN(addr));
235
236 if (!folio)
237 continue;
238
239 if (damos_pa_filter_out(s, folio))
240 goto put_folio;
241
242 folio_clear_referenced(folio);
243 folio_test_clear_young(folio);
244 if (!folio_isolate_lru(folio))
245 goto put_folio;
246 if (folio_test_unevictable(folio))
247 folio_putback_lru(folio);
248 else
249 list_add(&folio->lru, &folio_list);
250 put_folio:
251 folio_put(folio);
252 }
253 applied = reclaim_pages(&folio_list);
254 cond_resched();
255 return applied * PAGE_SIZE;
256 }
257
damon_pa_mark_accessed_or_deactivate(struct damon_region * r,struct damos * s,bool mark_accessed)258 static inline unsigned long damon_pa_mark_accessed_or_deactivate(
259 struct damon_region *r, struct damos *s, bool mark_accessed)
260 {
261 unsigned long addr, applied = 0;
262
263 for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
264 struct folio *folio = damon_get_folio(PHYS_PFN(addr));
265
266 if (!folio)
267 continue;
268
269 if (damos_pa_filter_out(s, folio))
270 goto put_folio;
271
272 if (mark_accessed)
273 folio_mark_accessed(folio);
274 else
275 folio_deactivate(folio);
276 applied += folio_nr_pages(folio);
277 put_folio:
278 folio_put(folio);
279 }
280 return applied * PAGE_SIZE;
281 }
282
damon_pa_mark_accessed(struct damon_region * r,struct damos * s)283 static unsigned long damon_pa_mark_accessed(struct damon_region *r,
284 struct damos *s)
285 {
286 return damon_pa_mark_accessed_or_deactivate(r, s, true);
287 }
288
damon_pa_deactivate_pages(struct damon_region * r,struct damos * s)289 static unsigned long damon_pa_deactivate_pages(struct damon_region *r,
290 struct damos *s)
291 {
292 return damon_pa_mark_accessed_or_deactivate(r, s, false);
293 }
294
damon_pa_apply_scheme(struct damon_ctx * ctx,struct damon_target * t,struct damon_region * r,struct damos * scheme)295 static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
296 struct damon_target *t, struct damon_region *r,
297 struct damos *scheme)
298 {
299 switch (scheme->action) {
300 case DAMOS_PAGEOUT:
301 return damon_pa_pageout(r, scheme);
302 case DAMOS_LRU_PRIO:
303 return damon_pa_mark_accessed(r, scheme);
304 case DAMOS_LRU_DEPRIO:
305 return damon_pa_deactivate_pages(r, scheme);
306 case DAMOS_STAT:
307 break;
308 default:
309 /* DAMOS actions that not yet supported by 'paddr'. */
310 break;
311 }
312 return 0;
313 }
314
damon_pa_scheme_score(struct damon_ctx * context,struct damon_target * t,struct damon_region * r,struct damos * scheme)315 static int damon_pa_scheme_score(struct damon_ctx *context,
316 struct damon_target *t, struct damon_region *r,
317 struct damos *scheme)
318 {
319 switch (scheme->action) {
320 case DAMOS_PAGEOUT:
321 return damon_cold_score(context, r, scheme);
322 case DAMOS_LRU_PRIO:
323 return damon_hot_score(context, r, scheme);
324 case DAMOS_LRU_DEPRIO:
325 return damon_cold_score(context, r, scheme);
326 default:
327 break;
328 }
329
330 return DAMOS_MAX_SCORE;
331 }
332
damon_pa_initcall(void)333 static int __init damon_pa_initcall(void)
334 {
335 struct damon_operations ops = {
336 .id = DAMON_OPS_PADDR,
337 .init = NULL,
338 .update = NULL,
339 .prepare_access_checks = damon_pa_prepare_access_checks,
340 .check_accesses = damon_pa_check_accesses,
341 .reset_aggregated = NULL,
342 .target_valid = NULL,
343 .cleanup = NULL,
344 .apply_scheme = damon_pa_apply_scheme,
345 .get_scheme_score = damon_pa_scheme_score,
346 };
347
348 return damon_register_ops(&ops);
349 };
350
351 subsys_initcall(damon_pa_initcall);
352