12224d848SSeongJae Park // SPDX-License-Identifier: GPL-2.0
22224d848SSeongJae Park /*
32224d848SSeongJae Park * Data Access Monitor
42224d848SSeongJae Park *
52224d848SSeongJae Park * Author: SeongJae Park <sjpark@amazon.de>
62224d848SSeongJae Park */
72224d848SSeongJae Park
82224d848SSeongJae Park #define pr_fmt(fmt) "damon: " fmt
92224d848SSeongJae Park
102224d848SSeongJae Park #include <linux/damon.h>
112224d848SSeongJae Park #include <linux/delay.h>
122224d848SSeongJae Park #include <linux/kthread.h>
13ee801b7dSSeongJae Park #include <linux/mm.h>
142224d848SSeongJae Park #include <linux/slab.h>
1538683e00SSeongJae Park #include <linux/string.h>
162224d848SSeongJae Park
172fcb9362SSeongJae Park #define CREATE_TRACE_POINTS
182fcb9362SSeongJae Park #include <trace/events/damon.h>
192fcb9362SSeongJae Park
2017ccae8bSSeongJae Park #ifdef CONFIG_DAMON_KUNIT_TEST
2117ccae8bSSeongJae Park #undef DAMON_MIN_REGION
2217ccae8bSSeongJae Park #define DAMON_MIN_REGION 1
2317ccae8bSSeongJae Park #endif
2417ccae8bSSeongJae Park
252224d848SSeongJae Park static DEFINE_MUTEX(damon_lock);
262224d848SSeongJae Park static int nr_running_ctxs;
278b9b0d33SSeongJae Park static bool running_exclusive_ctxs;
282224d848SSeongJae Park
299f7b053aSSeongJae Park static DEFINE_MUTEX(damon_ops_lock);
309f7b053aSSeongJae Park static struct damon_operations damon_registered_ops[NR_DAMON_OPS];
319f7b053aSSeongJae Park
32a1870944SDawei Li static struct kmem_cache *damon_region_cache __ro_after_init;
33a1870944SDawei Li
349f7b053aSSeongJae Park /* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */
__damon_is_registered_ops(enum damon_ops_id id)35152e5617SSeongJae Park static bool __damon_is_registered_ops(enum damon_ops_id id)
369f7b053aSSeongJae Park {
379f7b053aSSeongJae Park struct damon_operations empty_ops = {};
389f7b053aSSeongJae Park
399f7b053aSSeongJae Park if (!memcmp(&empty_ops, &damon_registered_ops[id], sizeof(empty_ops)))
409f7b053aSSeongJae Park return false;
419f7b053aSSeongJae Park return true;
429f7b053aSSeongJae Park }
439f7b053aSSeongJae Park
449f7b053aSSeongJae Park /**
45152e5617SSeongJae Park * damon_is_registered_ops() - Check if a given damon_operations is registered.
46152e5617SSeongJae Park * @id: Id of the damon_operations to check if registered.
47152e5617SSeongJae Park *
48152e5617SSeongJae Park * Return: true if the ops is set, false otherwise.
49152e5617SSeongJae Park */
damon_is_registered_ops(enum damon_ops_id id)50152e5617SSeongJae Park bool damon_is_registered_ops(enum damon_ops_id id)
51152e5617SSeongJae Park {
52152e5617SSeongJae Park bool registered;
53152e5617SSeongJae Park
54152e5617SSeongJae Park if (id >= NR_DAMON_OPS)
55152e5617SSeongJae Park return false;
56152e5617SSeongJae Park mutex_lock(&damon_ops_lock);
57152e5617SSeongJae Park registered = __damon_is_registered_ops(id);
58152e5617SSeongJae Park mutex_unlock(&damon_ops_lock);
59152e5617SSeongJae Park return registered;
60152e5617SSeongJae Park }
61152e5617SSeongJae Park
62152e5617SSeongJae Park /**
639f7b053aSSeongJae Park * damon_register_ops() - Register a monitoring operations set to DAMON.
649f7b053aSSeongJae Park * @ops: monitoring operations set to register.
659f7b053aSSeongJae Park *
669f7b053aSSeongJae Park * This function registers a monitoring operations set of valid &struct
679f7b053aSSeongJae Park * damon_operations->id so that others can find and use them later.
689f7b053aSSeongJae Park *
699f7b053aSSeongJae Park * Return: 0 on success, negative error code otherwise.
709f7b053aSSeongJae Park */
damon_register_ops(struct damon_operations * ops)719f7b053aSSeongJae Park int damon_register_ops(struct damon_operations *ops)
729f7b053aSSeongJae Park {
739f7b053aSSeongJae Park int err = 0;
749f7b053aSSeongJae Park
759f7b053aSSeongJae Park if (ops->id >= NR_DAMON_OPS)
769f7b053aSSeongJae Park return -EINVAL;
779f7b053aSSeongJae Park mutex_lock(&damon_ops_lock);
789f7b053aSSeongJae Park /* Fail for already registered ops */
79152e5617SSeongJae Park if (__damon_is_registered_ops(ops->id)) {
809f7b053aSSeongJae Park err = -EINVAL;
819f7b053aSSeongJae Park goto out;
829f7b053aSSeongJae Park }
839f7b053aSSeongJae Park damon_registered_ops[ops->id] = *ops;
849f7b053aSSeongJae Park out:
859f7b053aSSeongJae Park mutex_unlock(&damon_ops_lock);
869f7b053aSSeongJae Park return err;
879f7b053aSSeongJae Park }
889f7b053aSSeongJae Park
899f7b053aSSeongJae Park /**
909f7b053aSSeongJae Park * damon_select_ops() - Select a monitoring operations to use with the context.
919f7b053aSSeongJae Park * @ctx: monitoring context to use the operations.
929f7b053aSSeongJae Park * @id: id of the registered monitoring operations to select.
939f7b053aSSeongJae Park *
949f7b053aSSeongJae Park * This function finds registered monitoring operations set of @id and make
959f7b053aSSeongJae Park * @ctx to use it.
969f7b053aSSeongJae Park *
979f7b053aSSeongJae Park * Return: 0 on success, negative error code otherwise.
989f7b053aSSeongJae Park */
damon_select_ops(struct damon_ctx * ctx,enum damon_ops_id id)999f7b053aSSeongJae Park int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id)
1009f7b053aSSeongJae Park {
1019f7b053aSSeongJae Park int err = 0;
1029f7b053aSSeongJae Park
1039f7b053aSSeongJae Park if (id >= NR_DAMON_OPS)
1049f7b053aSSeongJae Park return -EINVAL;
1059f7b053aSSeongJae Park
1069f7b053aSSeongJae Park mutex_lock(&damon_ops_lock);
107152e5617SSeongJae Park if (!__damon_is_registered_ops(id))
1089f7b053aSSeongJae Park err = -EINVAL;
1099f7b053aSSeongJae Park else
1109f7b053aSSeongJae Park ctx->ops = damon_registered_ops[id];
1119f7b053aSSeongJae Park mutex_unlock(&damon_ops_lock);
1129f7b053aSSeongJae Park return err;
1139f7b053aSSeongJae Park }
1149f7b053aSSeongJae Park
115f23b8eeeSSeongJae Park /*
116f23b8eeeSSeongJae Park * Construct a damon_region struct
117f23b8eeeSSeongJae Park *
118f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise
119f23b8eeeSSeongJae Park */
damon_new_region(unsigned long start,unsigned long end)120f23b8eeeSSeongJae Park struct damon_region *damon_new_region(unsigned long start, unsigned long end)
121f23b8eeeSSeongJae Park {
122f23b8eeeSSeongJae Park struct damon_region *region;
123f23b8eeeSSeongJae Park
124a1870944SDawei Li region = kmem_cache_alloc(damon_region_cache, GFP_KERNEL);
125f23b8eeeSSeongJae Park if (!region)
126f23b8eeeSSeongJae Park return NULL;
127f23b8eeeSSeongJae Park
128f23b8eeeSSeongJae Park region->ar.start = start;
129f23b8eeeSSeongJae Park region->ar.end = end;
130f23b8eeeSSeongJae Park region->nr_accesses = 0;
131f23b8eeeSSeongJae Park INIT_LIST_HEAD(®ion->list);
132f23b8eeeSSeongJae Park
133fda504faSSeongJae Park region->age = 0;
134fda504faSSeongJae Park region->last_nr_accesses = 0;
135fda504faSSeongJae Park
136f23b8eeeSSeongJae Park return region;
137f23b8eeeSSeongJae Park }
138f23b8eeeSSeongJae Park
damon_add_region(struct damon_region * r,struct damon_target * t)139f23b8eeeSSeongJae Park void damon_add_region(struct damon_region *r, struct damon_target *t)
140f23b8eeeSSeongJae Park {
141f23b8eeeSSeongJae Park list_add_tail(&r->list, &t->regions_list);
142b9a6ac4eSSeongJae Park t->nr_regions++;
143f23b8eeeSSeongJae Park }
144f23b8eeeSSeongJae Park
damon_del_region(struct damon_region * r,struct damon_target * t)145b9a6ac4eSSeongJae Park static void damon_del_region(struct damon_region *r, struct damon_target *t)
146f23b8eeeSSeongJae Park {
147f23b8eeeSSeongJae Park list_del(&r->list);
148b9a6ac4eSSeongJae Park t->nr_regions--;
149f23b8eeeSSeongJae Park }
150f23b8eeeSSeongJae Park
damon_free_region(struct damon_region * r)151f23b8eeeSSeongJae Park static void damon_free_region(struct damon_region *r)
152f23b8eeeSSeongJae Park {
153a1870944SDawei Li kmem_cache_free(damon_region_cache, r);
154f23b8eeeSSeongJae Park }
155f23b8eeeSSeongJae Park
damon_destroy_region(struct damon_region * r,struct damon_target * t)156b9a6ac4eSSeongJae Park void damon_destroy_region(struct damon_region *r, struct damon_target *t)
157f23b8eeeSSeongJae Park {
158b9a6ac4eSSeongJae Park damon_del_region(r, t);
159f23b8eeeSSeongJae Park damon_free_region(r);
160f23b8eeeSSeongJae Park }
161f23b8eeeSSeongJae Park
162d0723bc0SSeongJae Park /*
163d0723bc0SSeongJae Park * Check whether a region is intersecting an address range
164d0723bc0SSeongJae Park *
165d0723bc0SSeongJae Park * Returns true if it is.
166d0723bc0SSeongJae Park */
damon_intersect(struct damon_region * r,struct damon_addr_range * re)167d0723bc0SSeongJae Park static bool damon_intersect(struct damon_region *r,
168d0723bc0SSeongJae Park struct damon_addr_range *re)
169d0723bc0SSeongJae Park {
170d0723bc0SSeongJae Park return !(r->ar.end <= re->start || re->end <= r->ar.start);
171d0723bc0SSeongJae Park }
172d0723bc0SSeongJae Park
173d0723bc0SSeongJae Park /*
1749c950c22SSeongJae Park * Fill holes in regions with new regions.
1759c950c22SSeongJae Park */
damon_fill_regions_holes(struct damon_region * first,struct damon_region * last,struct damon_target * t)1769c950c22SSeongJae Park static int damon_fill_regions_holes(struct damon_region *first,
1779c950c22SSeongJae Park struct damon_region *last, struct damon_target *t)
1789c950c22SSeongJae Park {
1799c950c22SSeongJae Park struct damon_region *r = first;
1809c950c22SSeongJae Park
1819c950c22SSeongJae Park damon_for_each_region_from(r, t) {
1829c950c22SSeongJae Park struct damon_region *next, *newr;
1839c950c22SSeongJae Park
1849c950c22SSeongJae Park if (r == last)
1859c950c22SSeongJae Park break;
1869c950c22SSeongJae Park next = damon_next_region(r);
1879c950c22SSeongJae Park if (r->ar.end != next->ar.start) {
1889c950c22SSeongJae Park newr = damon_new_region(r->ar.end, next->ar.start);
1899c950c22SSeongJae Park if (!newr)
1909c950c22SSeongJae Park return -ENOMEM;
1919c950c22SSeongJae Park damon_insert_region(newr, r, next, t);
1929c950c22SSeongJae Park }
1939c950c22SSeongJae Park }
1949c950c22SSeongJae Park return 0;
1959c950c22SSeongJae Park }
1969c950c22SSeongJae Park
1979c950c22SSeongJae Park /*
198d0723bc0SSeongJae Park * damon_set_regions() - Set regions of a target for given address ranges.
199d0723bc0SSeongJae Park * @t: the given target.
200d0723bc0SSeongJae Park * @ranges: array of new monitoring target ranges.
201d0723bc0SSeongJae Park * @nr_ranges: length of @ranges.
202d0723bc0SSeongJae Park *
203d0723bc0SSeongJae Park * This function adds new regions to, or modify existing regions of a
204d0723bc0SSeongJae Park * monitoring target to fit in specific ranges.
205d0723bc0SSeongJae Park *
206d0723bc0SSeongJae Park * Return: 0 if success, or negative error code otherwise.
207d0723bc0SSeongJae Park */
damon_set_regions(struct damon_target * t,struct damon_addr_range * ranges,unsigned int nr_ranges)208d0723bc0SSeongJae Park int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
209d0723bc0SSeongJae Park unsigned int nr_ranges)
210d0723bc0SSeongJae Park {
211d0723bc0SSeongJae Park struct damon_region *r, *next;
212d0723bc0SSeongJae Park unsigned int i;
2139c950c22SSeongJae Park int err;
214d0723bc0SSeongJae Park
215d0723bc0SSeongJae Park /* Remove regions which are not in the new ranges */
216d0723bc0SSeongJae Park damon_for_each_region_safe(r, next, t) {
217d0723bc0SSeongJae Park for (i = 0; i < nr_ranges; i++) {
218d0723bc0SSeongJae Park if (damon_intersect(r, &ranges[i]))
219d0723bc0SSeongJae Park break;
220d0723bc0SSeongJae Park }
221d0723bc0SSeongJae Park if (i == nr_ranges)
222d0723bc0SSeongJae Park damon_destroy_region(r, t);
223d0723bc0SSeongJae Park }
224d0723bc0SSeongJae Park
22536001cbaSKaixu Xia r = damon_first_region(t);
226d0723bc0SSeongJae Park /* Add new regions or resize existing regions to fit in the ranges */
227d0723bc0SSeongJae Park for (i = 0; i < nr_ranges; i++) {
228d0723bc0SSeongJae Park struct damon_region *first = NULL, *last, *newr;
229d0723bc0SSeongJae Park struct damon_addr_range *range;
230d0723bc0SSeongJae Park
231d0723bc0SSeongJae Park range = &ranges[i];
232d0723bc0SSeongJae Park /* Get the first/last regions intersecting with the range */
23336001cbaSKaixu Xia damon_for_each_region_from(r, t) {
234d0723bc0SSeongJae Park if (damon_intersect(r, range)) {
235d0723bc0SSeongJae Park if (!first)
236d0723bc0SSeongJae Park first = r;
237d0723bc0SSeongJae Park last = r;
238d0723bc0SSeongJae Park }
239d0723bc0SSeongJae Park if (r->ar.start >= range->end)
240d0723bc0SSeongJae Park break;
241d0723bc0SSeongJae Park }
242d0723bc0SSeongJae Park if (!first) {
243d0723bc0SSeongJae Park /* no region intersects with this range */
244d0723bc0SSeongJae Park newr = damon_new_region(
245d0723bc0SSeongJae Park ALIGN_DOWN(range->start,
246d0723bc0SSeongJae Park DAMON_MIN_REGION),
247d0723bc0SSeongJae Park ALIGN(range->end, DAMON_MIN_REGION));
248d0723bc0SSeongJae Park if (!newr)
249d0723bc0SSeongJae Park return -ENOMEM;
250d0723bc0SSeongJae Park damon_insert_region(newr, damon_prev_region(r), r, t);
251d0723bc0SSeongJae Park } else {
252d0723bc0SSeongJae Park /* resize intersecting regions to fit in this range */
253d0723bc0SSeongJae Park first->ar.start = ALIGN_DOWN(range->start,
254d0723bc0SSeongJae Park DAMON_MIN_REGION);
255d0723bc0SSeongJae Park last->ar.end = ALIGN(range->end, DAMON_MIN_REGION);
2569c950c22SSeongJae Park
2579c950c22SSeongJae Park /* fill possible holes in the range */
2589c950c22SSeongJae Park err = damon_fill_regions_holes(first, last, t);
2599c950c22SSeongJae Park if (err)
2609c950c22SSeongJae Park return err;
261d0723bc0SSeongJae Park }
262d0723bc0SSeongJae Park }
263d0723bc0SSeongJae Park return 0;
264d0723bc0SSeongJae Park }
265d0723bc0SSeongJae Park
damos_new_filter(enum damos_filter_type type,bool matching)26698def236SSeongJae Park struct damos_filter *damos_new_filter(enum damos_filter_type type,
26798def236SSeongJae Park bool matching)
26898def236SSeongJae Park {
26998def236SSeongJae Park struct damos_filter *filter;
27098def236SSeongJae Park
27198def236SSeongJae Park filter = kmalloc(sizeof(*filter), GFP_KERNEL);
27298def236SSeongJae Park if (!filter)
27398def236SSeongJae Park return NULL;
27498def236SSeongJae Park filter->type = type;
27598def236SSeongJae Park filter->matching = matching;
2765f1fc67fSSeongJae Park INIT_LIST_HEAD(&filter->list);
27798def236SSeongJae Park return filter;
27898def236SSeongJae Park }
27998def236SSeongJae Park
damos_add_filter(struct damos * s,struct damos_filter * f)28098def236SSeongJae Park void damos_add_filter(struct damos *s, struct damos_filter *f)
28198def236SSeongJae Park {
28298def236SSeongJae Park list_add_tail(&f->list, &s->filters);
28398def236SSeongJae Park }
28498def236SSeongJae Park
damos_del_filter(struct damos_filter * f)28598def236SSeongJae Park static void damos_del_filter(struct damos_filter *f)
28698def236SSeongJae Park {
28798def236SSeongJae Park list_del(&f->list);
28898def236SSeongJae Park }
28998def236SSeongJae Park
damos_free_filter(struct damos_filter * f)29098def236SSeongJae Park static void damos_free_filter(struct damos_filter *f)
29198def236SSeongJae Park {
29298def236SSeongJae Park kfree(f);
29398def236SSeongJae Park }
29498def236SSeongJae Park
damos_destroy_filter(struct damos_filter * f)29598def236SSeongJae Park void damos_destroy_filter(struct damos_filter *f)
29698def236SSeongJae Park {
29798def236SSeongJae Park damos_del_filter(f);
29898def236SSeongJae Park damos_free_filter(f);
29998def236SSeongJae Park }
30098def236SSeongJae Park
30170e0c1d1SSeongJae Park /* initialize private fields of damos_quota and return the pointer */
damos_quota_init_priv(struct damos_quota * quota)30270e0c1d1SSeongJae Park static struct damos_quota *damos_quota_init_priv(struct damos_quota *quota)
30370e0c1d1SSeongJae Park {
30470e0c1d1SSeongJae Park quota->total_charged_sz = 0;
30570e0c1d1SSeongJae Park quota->total_charged_ns = 0;
30670e0c1d1SSeongJae Park quota->esz = 0;
30770e0c1d1SSeongJae Park quota->charged_sz = 0;
30870e0c1d1SSeongJae Park quota->charged_from = 0;
30970e0c1d1SSeongJae Park quota->charge_target_from = NULL;
31070e0c1d1SSeongJae Park quota->charge_addr_from = 0;
31170e0c1d1SSeongJae Park return quota;
31270e0c1d1SSeongJae Park }
31370e0c1d1SSeongJae Park
damon_new_scheme(struct damos_access_pattern * pattern,enum damos_action action,struct damos_quota * quota,struct damos_watermarks * wmarks)314f5a79d7cSYajun Deng struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
315ee801b7dSSeongJae Park enum damos_action action, struct damos_quota *quota,
316ee801b7dSSeongJae Park struct damos_watermarks *wmarks)
3171f366e42SSeongJae Park {
3181f366e42SSeongJae Park struct damos *scheme;
3191f366e42SSeongJae Park
3201f366e42SSeongJae Park scheme = kmalloc(sizeof(*scheme), GFP_KERNEL);
3211f366e42SSeongJae Park if (!scheme)
3221f366e42SSeongJae Park return NULL;
32302f17037SSeongJae Park scheme->pattern = *pattern;
3241f366e42SSeongJae Park scheme->action = action;
32598def236SSeongJae Park INIT_LIST_HEAD(&scheme->filters);
3260e92c2eeSSeongJae Park scheme->stat = (struct damos_stat){};
3271f366e42SSeongJae Park INIT_LIST_HEAD(&scheme->list);
3281f366e42SSeongJae Park
32970e0c1d1SSeongJae Park scheme->quota = *(damos_quota_init_priv(quota));
3302b8a248dSSeongJae Park
33102f17037SSeongJae Park scheme->wmarks = *wmarks;
332ee801b7dSSeongJae Park scheme->wmarks.activated = true;
333ee801b7dSSeongJae Park
3341f366e42SSeongJae Park return scheme;
3351f366e42SSeongJae Park }
3361f366e42SSeongJae Park
damon_add_scheme(struct damon_ctx * ctx,struct damos * s)3371f366e42SSeongJae Park void damon_add_scheme(struct damon_ctx *ctx, struct damos *s)
3381f366e42SSeongJae Park {
3391f366e42SSeongJae Park list_add_tail(&s->list, &ctx->schemes);
3401f366e42SSeongJae Park }
3411f366e42SSeongJae Park
damon_del_scheme(struct damos * s)3421f366e42SSeongJae Park static void damon_del_scheme(struct damos *s)
3431f366e42SSeongJae Park {
3441f366e42SSeongJae Park list_del(&s->list);
3451f366e42SSeongJae Park }
3461f366e42SSeongJae Park
damon_free_scheme(struct damos * s)3471f366e42SSeongJae Park static void damon_free_scheme(struct damos *s)
3481f366e42SSeongJae Park {
3491f366e42SSeongJae Park kfree(s);
3501f366e42SSeongJae Park }
3511f366e42SSeongJae Park
damon_destroy_scheme(struct damos * s)3521f366e42SSeongJae Park void damon_destroy_scheme(struct damos *s)
3531f366e42SSeongJae Park {
35498def236SSeongJae Park struct damos_filter *f, *next;
35598def236SSeongJae Park
35698def236SSeongJae Park damos_for_each_filter_safe(f, next, s)
35798def236SSeongJae Park damos_destroy_filter(f);
3581f366e42SSeongJae Park damon_del_scheme(s);
3591f366e42SSeongJae Park damon_free_scheme(s);
3601f366e42SSeongJae Park }
3611f366e42SSeongJae Park
362f23b8eeeSSeongJae Park /*
363f23b8eeeSSeongJae Park * Construct a damon_target struct
364f23b8eeeSSeongJae Park *
365f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise
366f23b8eeeSSeongJae Park */
damon_new_target(void)3671971bd63SSeongJae Park struct damon_target *damon_new_target(void)
368f23b8eeeSSeongJae Park {
369f23b8eeeSSeongJae Park struct damon_target *t;
370f23b8eeeSSeongJae Park
371f23b8eeeSSeongJae Park t = kmalloc(sizeof(*t), GFP_KERNEL);
372f23b8eeeSSeongJae Park if (!t)
373f23b8eeeSSeongJae Park return NULL;
374f23b8eeeSSeongJae Park
3751971bd63SSeongJae Park t->pid = NULL;
376b9a6ac4eSSeongJae Park t->nr_regions = 0;
377f23b8eeeSSeongJae Park INIT_LIST_HEAD(&t->regions_list);
378b1f44cdaSSeongJae Park INIT_LIST_HEAD(&t->list);
379f23b8eeeSSeongJae Park
380f23b8eeeSSeongJae Park return t;
381f23b8eeeSSeongJae Park }
382f23b8eeeSSeongJae Park
damon_add_target(struct damon_ctx * ctx,struct damon_target * t)383f23b8eeeSSeongJae Park void damon_add_target(struct damon_ctx *ctx, struct damon_target *t)
384f23b8eeeSSeongJae Park {
385b9a6ac4eSSeongJae Park list_add_tail(&t->list, &ctx->adaptive_targets);
386f23b8eeeSSeongJae Park }
387f23b8eeeSSeongJae Park
damon_targets_empty(struct damon_ctx * ctx)388b5ca3e83SXin Hao bool damon_targets_empty(struct damon_ctx *ctx)
389b5ca3e83SXin Hao {
390b5ca3e83SXin Hao return list_empty(&ctx->adaptive_targets);
391b5ca3e83SXin Hao }
392b5ca3e83SXin Hao
damon_del_target(struct damon_target * t)393f23b8eeeSSeongJae Park static void damon_del_target(struct damon_target *t)
394f23b8eeeSSeongJae Park {
395f23b8eeeSSeongJae Park list_del(&t->list);
396f23b8eeeSSeongJae Park }
397f23b8eeeSSeongJae Park
damon_free_target(struct damon_target * t)398f23b8eeeSSeongJae Park void damon_free_target(struct damon_target *t)
399f23b8eeeSSeongJae Park {
400f23b8eeeSSeongJae Park struct damon_region *r, *next;
401f23b8eeeSSeongJae Park
402f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t)
403f23b8eeeSSeongJae Park damon_free_region(r);
404f23b8eeeSSeongJae Park kfree(t);
405f23b8eeeSSeongJae Park }
406f23b8eeeSSeongJae Park
damon_destroy_target(struct damon_target * t)407f23b8eeeSSeongJae Park void damon_destroy_target(struct damon_target *t)
408f23b8eeeSSeongJae Park {
409f23b8eeeSSeongJae Park damon_del_target(t);
410f23b8eeeSSeongJae Park damon_free_target(t);
411f23b8eeeSSeongJae Park }
412f23b8eeeSSeongJae Park
damon_nr_regions(struct damon_target * t)413b9a6ac4eSSeongJae Park unsigned int damon_nr_regions(struct damon_target *t)
414b9a6ac4eSSeongJae Park {
415b9a6ac4eSSeongJae Park return t->nr_regions;
416b9a6ac4eSSeongJae Park }
417b9a6ac4eSSeongJae Park
damon_new_ctx(void)4182224d848SSeongJae Park struct damon_ctx *damon_new_ctx(void)
4192224d848SSeongJae Park {
4202224d848SSeongJae Park struct damon_ctx *ctx;
4212224d848SSeongJae Park
4222224d848SSeongJae Park ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
4232224d848SSeongJae Park if (!ctx)
4242224d848SSeongJae Park return NULL;
4252224d848SSeongJae Park
426e93bcaebSSeongJae Park init_completion(&ctx->kdamond_started);
427e93bcaebSSeongJae Park
428cbeaa77bSSeongJae Park ctx->attrs.sample_interval = 5 * 1000;
429cbeaa77bSSeongJae Park ctx->attrs.aggr_interval = 100 * 1000;
430cbeaa77bSSeongJae Park ctx->attrs.ops_update_interval = 60 * 1000 * 1000;
4312224d848SSeongJae Park
432c708a5e5SSeongJae Park ctx->passed_sample_intervals = 0;
433c708a5e5SSeongJae Park /* These will be set from kdamond_init_intervals_sis() */
434c708a5e5SSeongJae Park ctx->next_aggregation_sis = 0;
435c708a5e5SSeongJae Park ctx->next_ops_update_sis = 0;
4362224d848SSeongJae Park
4372224d848SSeongJae Park mutex_init(&ctx->kdamond_lock);
4382224d848SSeongJae Park
439cbeaa77bSSeongJae Park ctx->attrs.min_nr_regions = 10;
440cbeaa77bSSeongJae Park ctx->attrs.max_nr_regions = 1000;
441b9a6ac4eSSeongJae Park
442b9a6ac4eSSeongJae Park INIT_LIST_HEAD(&ctx->adaptive_targets);
4431f366e42SSeongJae Park INIT_LIST_HEAD(&ctx->schemes);
4442224d848SSeongJae Park
4452224d848SSeongJae Park return ctx;
4462224d848SSeongJae Park }
4472224d848SSeongJae Park
damon_destroy_targets(struct damon_ctx * ctx)448f23b8eeeSSeongJae Park static void damon_destroy_targets(struct damon_ctx *ctx)
449f23b8eeeSSeongJae Park {
450f23b8eeeSSeongJae Park struct damon_target *t, *next_t;
451f23b8eeeSSeongJae Park
452f7d911c3SSeongJae Park if (ctx->ops.cleanup) {
453f7d911c3SSeongJae Park ctx->ops.cleanup(ctx);
454f23b8eeeSSeongJae Park return;
455f23b8eeeSSeongJae Park }
456f23b8eeeSSeongJae Park
457f23b8eeeSSeongJae Park damon_for_each_target_safe(t, next_t, ctx)
458f23b8eeeSSeongJae Park damon_destroy_target(t);
459f23b8eeeSSeongJae Park }
460f23b8eeeSSeongJae Park
damon_destroy_ctx(struct damon_ctx * ctx)4612224d848SSeongJae Park void damon_destroy_ctx(struct damon_ctx *ctx)
4622224d848SSeongJae Park {
4631f366e42SSeongJae Park struct damos *s, *next_s;
4641f366e42SSeongJae Park
465f23b8eeeSSeongJae Park damon_destroy_targets(ctx);
4661f366e42SSeongJae Park
4671f366e42SSeongJae Park damon_for_each_scheme_safe(s, next_s, ctx)
4681f366e42SSeongJae Park damon_destroy_scheme(s);
4691f366e42SSeongJae Park
4702224d848SSeongJae Park kfree(ctx);
4712224d848SSeongJae Park }
4722224d848SSeongJae Park
damon_age_for_new_attrs(unsigned int age,struct damon_attrs * old_attrs,struct damon_attrs * new_attrs)4732f5bef5aSSeongJae Park static unsigned int damon_age_for_new_attrs(unsigned int age,
4742f5bef5aSSeongJae Park struct damon_attrs *old_attrs, struct damon_attrs *new_attrs)
4752f5bef5aSSeongJae Park {
4762f5bef5aSSeongJae Park return age * old_attrs->aggr_interval / new_attrs->aggr_interval;
4772f5bef5aSSeongJae Park }
4782f5bef5aSSeongJae Park
4792f5bef5aSSeongJae Park /* convert access ratio in bp (per 10,000) to nr_accesses */
damon_accesses_bp_to_nr_accesses(unsigned int accesses_bp,struct damon_attrs * attrs)4802f5bef5aSSeongJae Park static unsigned int damon_accesses_bp_to_nr_accesses(
4812f5bef5aSSeongJae Park unsigned int accesses_bp, struct damon_attrs *attrs)
4822f5bef5aSSeongJae Park {
48374ed1097SSeongJae Park return accesses_bp * damon_max_nr_accesses(attrs) / 10000;
4842f5bef5aSSeongJae Park }
4852f5bef5aSSeongJae Park
4862f5bef5aSSeongJae Park /* convert nr_accesses to access ratio in bp (per 10,000) */
damon_nr_accesses_to_accesses_bp(unsigned int nr_accesses,struct damon_attrs * attrs)4872f5bef5aSSeongJae Park static unsigned int damon_nr_accesses_to_accesses_bp(
4882f5bef5aSSeongJae Park unsigned int nr_accesses, struct damon_attrs *attrs)
4892f5bef5aSSeongJae Park {
49074ed1097SSeongJae Park return nr_accesses * 10000 / damon_max_nr_accesses(attrs);
4912f5bef5aSSeongJae Park }
4922f5bef5aSSeongJae Park
damon_nr_accesses_for_new_attrs(unsigned int nr_accesses,struct damon_attrs * old_attrs,struct damon_attrs * new_attrs)4932f5bef5aSSeongJae Park static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses,
4942f5bef5aSSeongJae Park struct damon_attrs *old_attrs, struct damon_attrs *new_attrs)
4952f5bef5aSSeongJae Park {
4962f5bef5aSSeongJae Park return damon_accesses_bp_to_nr_accesses(
4972f5bef5aSSeongJae Park damon_nr_accesses_to_accesses_bp(
4982f5bef5aSSeongJae Park nr_accesses, old_attrs),
4992f5bef5aSSeongJae Park new_attrs);
5002f5bef5aSSeongJae Park }
5012f5bef5aSSeongJae Park
damon_update_monitoring_result(struct damon_region * r,struct damon_attrs * old_attrs,struct damon_attrs * new_attrs)5022f5bef5aSSeongJae Park static void damon_update_monitoring_result(struct damon_region *r,
5032f5bef5aSSeongJae Park struct damon_attrs *old_attrs, struct damon_attrs *new_attrs)
5042f5bef5aSSeongJae Park {
5052f5bef5aSSeongJae Park r->nr_accesses = damon_nr_accesses_for_new_attrs(r->nr_accesses,
5062f5bef5aSSeongJae Park old_attrs, new_attrs);
5072f5bef5aSSeongJae Park r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs);
5082f5bef5aSSeongJae Park }
5092f5bef5aSSeongJae Park
5102f5bef5aSSeongJae Park /*
5112f5bef5aSSeongJae Park * region->nr_accesses is the number of sampling intervals in the last
5122f5bef5aSSeongJae Park * aggregation interval that access to the region has found, and region->age is
5132f5bef5aSSeongJae Park * the number of aggregation intervals that its access pattern has maintained.
5142f5bef5aSSeongJae Park * For the reason, the real meaning of the two fields depend on current
5152f5bef5aSSeongJae Park * sampling interval and aggregation interval. This function updates
5162f5bef5aSSeongJae Park * ->nr_accesses and ->age of given damon_ctx's regions for new damon_attrs.
5172f5bef5aSSeongJae Park */
damon_update_monitoring_results(struct damon_ctx * ctx,struct damon_attrs * new_attrs)5182f5bef5aSSeongJae Park static void damon_update_monitoring_results(struct damon_ctx *ctx,
5192f5bef5aSSeongJae Park struct damon_attrs *new_attrs)
5202f5bef5aSSeongJae Park {
5212f5bef5aSSeongJae Park struct damon_attrs *old_attrs = &ctx->attrs;
5222f5bef5aSSeongJae Park struct damon_target *t;
5232f5bef5aSSeongJae Park struct damon_region *r;
5242f5bef5aSSeongJae Park
5252f5bef5aSSeongJae Park /* if any interval is zero, simply forgive conversion */
5262f5bef5aSSeongJae Park if (!old_attrs->sample_interval || !old_attrs->aggr_interval ||
5272f5bef5aSSeongJae Park !new_attrs->sample_interval ||
5282f5bef5aSSeongJae Park !new_attrs->aggr_interval)
5292f5bef5aSSeongJae Park return;
5302f5bef5aSSeongJae Park
5312f5bef5aSSeongJae Park damon_for_each_target(t, ctx)
5322f5bef5aSSeongJae Park damon_for_each_region(r, t)
5332f5bef5aSSeongJae Park damon_update_monitoring_result(
5342f5bef5aSSeongJae Park r, old_attrs, new_attrs);
5352f5bef5aSSeongJae Park }
5362f5bef5aSSeongJae Park
5372224d848SSeongJae Park /**
5382224d848SSeongJae Park * damon_set_attrs() - Set attributes for the monitoring.
5392224d848SSeongJae Park * @ctx: monitoring context
540bead3b00SSeongJae Park * @attrs: monitoring attributes
5412224d848SSeongJae Park *
5422224d848SSeongJae Park * This function should not be called while the kdamond is running.
5432224d848SSeongJae Park * Every time interval is in micro-seconds.
5442224d848SSeongJae Park *
5452224d848SSeongJae Park * Return: 0 on success, negative error code otherwise.
5462224d848SSeongJae Park */
damon_set_attrs(struct damon_ctx * ctx,struct damon_attrs * attrs)547bead3b00SSeongJae Park int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs)
5482224d848SSeongJae Park {
549c708a5e5SSeongJae Park unsigned long sample_interval = attrs->sample_interval ?
550c708a5e5SSeongJae Park attrs->sample_interval : 1;
551c708a5e5SSeongJae Park
552bead3b00SSeongJae Park if (attrs->min_nr_regions < 3)
553b9a6ac4eSSeongJae Park return -EINVAL;
554bead3b00SSeongJae Park if (attrs->min_nr_regions > attrs->max_nr_regions)
555b9a6ac4eSSeongJae Park return -EINVAL;
5565ff6e2ffSKefeng Wang if (attrs->sample_interval > attrs->aggr_interval)
5575ff6e2ffSKefeng Wang return -EINVAL;
558b9a6ac4eSSeongJae Park
559c708a5e5SSeongJae Park ctx->next_aggregation_sis = ctx->passed_sample_intervals +
560c708a5e5SSeongJae Park attrs->aggr_interval / sample_interval;
561c708a5e5SSeongJae Park ctx->next_ops_update_sis = ctx->passed_sample_intervals +
562c708a5e5SSeongJae Park attrs->ops_update_interval / sample_interval;
563c708a5e5SSeongJae Park
5642f5bef5aSSeongJae Park damon_update_monitoring_results(ctx, attrs);
565bead3b00SSeongJae Park ctx->attrs = *attrs;
5662224d848SSeongJae Park return 0;
5672224d848SSeongJae Park }
5682224d848SSeongJae Park
5694bc05954SSeongJae Park /**
5701f366e42SSeongJae Park * damon_set_schemes() - Set data access monitoring based operation schemes.
5711f366e42SSeongJae Park * @ctx: monitoring context
5721f366e42SSeongJae Park * @schemes: array of the schemes
5731f366e42SSeongJae Park * @nr_schemes: number of entries in @schemes
5741f366e42SSeongJae Park *
5751f366e42SSeongJae Park * This function should not be called while the kdamond of the context is
5761f366e42SSeongJae Park * running.
5771f366e42SSeongJae Park */
damon_set_schemes(struct damon_ctx * ctx,struct damos ** schemes,ssize_t nr_schemes)578cc713520SKaixu Xia void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes,
5791f366e42SSeongJae Park ssize_t nr_schemes)
5801f366e42SSeongJae Park {
5811f366e42SSeongJae Park struct damos *s, *next;
5821f366e42SSeongJae Park ssize_t i;
5831f366e42SSeongJae Park
5841f366e42SSeongJae Park damon_for_each_scheme_safe(s, next, ctx)
5851f366e42SSeongJae Park damon_destroy_scheme(s);
5861f366e42SSeongJae Park for (i = 0; i < nr_schemes; i++)
5871f366e42SSeongJae Park damon_add_scheme(ctx, schemes[i]);
5881f366e42SSeongJae Park }
5891f366e42SSeongJae Park
5901f366e42SSeongJae Park /**
5914bc05954SSeongJae Park * damon_nr_running_ctxs() - Return number of currently running contexts.
5924bc05954SSeongJae Park */
damon_nr_running_ctxs(void)5934bc05954SSeongJae Park int damon_nr_running_ctxs(void)
5944bc05954SSeongJae Park {
5954bc05954SSeongJae Park int nr_ctxs;
5964bc05954SSeongJae Park
5974bc05954SSeongJae Park mutex_lock(&damon_lock);
5984bc05954SSeongJae Park nr_ctxs = nr_running_ctxs;
5994bc05954SSeongJae Park mutex_unlock(&damon_lock);
6004bc05954SSeongJae Park
6014bc05954SSeongJae Park return nr_ctxs;
6024bc05954SSeongJae Park }
6034bc05954SSeongJae Park
604b9a6ac4eSSeongJae Park /* Returns the size upper limit for each monitoring region */
damon_region_sz_limit(struct damon_ctx * ctx)605b9a6ac4eSSeongJae Park static unsigned long damon_region_sz_limit(struct damon_ctx *ctx)
606b9a6ac4eSSeongJae Park {
607b9a6ac4eSSeongJae Park struct damon_target *t;
608b9a6ac4eSSeongJae Park struct damon_region *r;
609b9a6ac4eSSeongJae Park unsigned long sz = 0;
610b9a6ac4eSSeongJae Park
611b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) {
612b9a6ac4eSSeongJae Park damon_for_each_region(r, t)
613ab63f63fSXin Hao sz += damon_sz_region(r);
614b9a6ac4eSSeongJae Park }
615b9a6ac4eSSeongJae Park
616cbeaa77bSSeongJae Park if (ctx->attrs.min_nr_regions)
617cbeaa77bSSeongJae Park sz /= ctx->attrs.min_nr_regions;
618b9a6ac4eSSeongJae Park if (sz < DAMON_MIN_REGION)
619b9a6ac4eSSeongJae Park sz = DAMON_MIN_REGION;
620b9a6ac4eSSeongJae Park
621b9a6ac4eSSeongJae Park return sz;
622b9a6ac4eSSeongJae Park }
623b9a6ac4eSSeongJae Park
6242224d848SSeongJae Park static int kdamond_fn(void *data);
6252224d848SSeongJae Park
6262224d848SSeongJae Park /*
6272224d848SSeongJae Park * __damon_start() - Starts monitoring with given context.
6282224d848SSeongJae Park * @ctx: monitoring context
6292224d848SSeongJae Park *
6302224d848SSeongJae Park * This function should be called while damon_lock is hold.
6312224d848SSeongJae Park *
6322224d848SSeongJae Park * Return: 0 on success, negative error code otherwise.
6332224d848SSeongJae Park */
__damon_start(struct damon_ctx * ctx)6342224d848SSeongJae Park static int __damon_start(struct damon_ctx *ctx)
6352224d848SSeongJae Park {
6362224d848SSeongJae Park int err = -EBUSY;
6372224d848SSeongJae Park
6382224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock);
6392224d848SSeongJae Park if (!ctx->kdamond) {
6402224d848SSeongJae Park err = 0;
641e93bcaebSSeongJae Park reinit_completion(&ctx->kdamond_started);
6422224d848SSeongJae Park ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d",
6432224d848SSeongJae Park nr_running_ctxs);
6442224d848SSeongJae Park if (IS_ERR(ctx->kdamond)) {
6452224d848SSeongJae Park err = PTR_ERR(ctx->kdamond);
6467ec1992bSColin Ian King ctx->kdamond = NULL;
647e93bcaebSSeongJae Park } else {
648e93bcaebSSeongJae Park wait_for_completion(&ctx->kdamond_started);
6492224d848SSeongJae Park }
6502224d848SSeongJae Park }
6512224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock);
6522224d848SSeongJae Park
6532224d848SSeongJae Park return err;
6542224d848SSeongJae Park }
6552224d848SSeongJae Park
6562224d848SSeongJae Park /**
6572224d848SSeongJae Park * damon_start() - Starts the monitorings for a given group of contexts.
6582224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to start monitoring
6592224d848SSeongJae Park * @nr_ctxs: size of @ctxs
6608b9b0d33SSeongJae Park * @exclusive: exclusiveness of this contexts group
6612224d848SSeongJae Park *
6622224d848SSeongJae Park * This function starts a group of monitoring threads for a group of monitoring
6632224d848SSeongJae Park * contexts. One thread per each context is created and run in parallel. The
6648b9b0d33SSeongJae Park * caller should handle synchronization between the threads by itself. If
6658b9b0d33SSeongJae Park * @exclusive is true and a group of threads that created by other
6668b9b0d33SSeongJae Park * 'damon_start()' call is currently running, this function does nothing but
6678b9b0d33SSeongJae Park * returns -EBUSY.
6682224d848SSeongJae Park *
6692224d848SSeongJae Park * Return: 0 on success, negative error code otherwise.
6702224d848SSeongJae Park */
damon_start(struct damon_ctx ** ctxs,int nr_ctxs,bool exclusive)6718b9b0d33SSeongJae Park int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive)
6722224d848SSeongJae Park {
6732224d848SSeongJae Park int i;
6742224d848SSeongJae Park int err = 0;
6752224d848SSeongJae Park
6762224d848SSeongJae Park mutex_lock(&damon_lock);
6778b9b0d33SSeongJae Park if ((exclusive && nr_running_ctxs) ||
6788b9b0d33SSeongJae Park (!exclusive && running_exclusive_ctxs)) {
6792224d848SSeongJae Park mutex_unlock(&damon_lock);
6802224d848SSeongJae Park return -EBUSY;
6812224d848SSeongJae Park }
6822224d848SSeongJae Park
6832224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) {
6842224d848SSeongJae Park err = __damon_start(ctxs[i]);
6852224d848SSeongJae Park if (err)
6862224d848SSeongJae Park break;
6872224d848SSeongJae Park nr_running_ctxs++;
6882224d848SSeongJae Park }
6898b9b0d33SSeongJae Park if (exclusive && nr_running_ctxs)
6908b9b0d33SSeongJae Park running_exclusive_ctxs = true;
6912224d848SSeongJae Park mutex_unlock(&damon_lock);
6922224d848SSeongJae Park
6932224d848SSeongJae Park return err;
6942224d848SSeongJae Park }
6952224d848SSeongJae Park
6962224d848SSeongJae Park /*
6978b9b0d33SSeongJae Park * __damon_stop() - Stops monitoring of a given context.
6982224d848SSeongJae Park * @ctx: monitoring context
6992224d848SSeongJae Park *
7002224d848SSeongJae Park * Return: 0 on success, negative error code otherwise.
7012224d848SSeongJae Park */
__damon_stop(struct damon_ctx * ctx)7022224d848SSeongJae Park static int __damon_stop(struct damon_ctx *ctx)
7032224d848SSeongJae Park {
7040f91d133SChangbin Du struct task_struct *tsk;
7050f91d133SChangbin Du
7062224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock);
7070f91d133SChangbin Du tsk = ctx->kdamond;
7080f91d133SChangbin Du if (tsk) {
7090f91d133SChangbin Du get_task_struct(tsk);
7102224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock);
711a9da6ddaSAndreas Gruenbacher kthread_stop_put(tsk);
7122224d848SSeongJae Park return 0;
7132224d848SSeongJae Park }
7142224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock);
7152224d848SSeongJae Park
7162224d848SSeongJae Park return -EPERM;
7172224d848SSeongJae Park }
7182224d848SSeongJae Park
7192224d848SSeongJae Park /**
7202224d848SSeongJae Park * damon_stop() - Stops the monitorings for a given group of contexts.
7212224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to stop monitoring
7222224d848SSeongJae Park * @nr_ctxs: size of @ctxs
7232224d848SSeongJae Park *
7242224d848SSeongJae Park * Return: 0 on success, negative error code otherwise.
7252224d848SSeongJae Park */
damon_stop(struct damon_ctx ** ctxs,int nr_ctxs)7262224d848SSeongJae Park int damon_stop(struct damon_ctx **ctxs, int nr_ctxs)
7272224d848SSeongJae Park {
7282224d848SSeongJae Park int i, err = 0;
7292224d848SSeongJae Park
7302224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) {
7312224d848SSeongJae Park /* nr_running_ctxs is decremented in kdamond_fn */
7322224d848SSeongJae Park err = __damon_stop(ctxs[i]);
7332224d848SSeongJae Park if (err)
7348b9b0d33SSeongJae Park break;
7352224d848SSeongJae Park }
7362224d848SSeongJae Park return err;
7372224d848SSeongJae Park }
7382224d848SSeongJae Park
7392224d848SSeongJae Park /*
740f23b8eeeSSeongJae Park * Reset the aggregated monitoring results ('nr_accesses' of each region).
741f23b8eeeSSeongJae Park */
kdamond_reset_aggregated(struct damon_ctx * c)742f23b8eeeSSeongJae Park static void kdamond_reset_aggregated(struct damon_ctx *c)
743f23b8eeeSSeongJae Park {
744f23b8eeeSSeongJae Park struct damon_target *t;
74576fd0285SSeongJae Park unsigned int ti = 0; /* target's index */
746f23b8eeeSSeongJae Park
747f23b8eeeSSeongJae Park damon_for_each_target(t, c) {
748f23b8eeeSSeongJae Park struct damon_region *r;
749f23b8eeeSSeongJae Park
7502fcb9362SSeongJae Park damon_for_each_region(r, t) {
75176fd0285SSeongJae Park trace_damon_aggregated(t, ti, r, damon_nr_regions(t));
752fda504faSSeongJae Park r->last_nr_accesses = r->nr_accesses;
753f23b8eeeSSeongJae Park r->nr_accesses = 0;
754f23b8eeeSSeongJae Park }
75576fd0285SSeongJae Park ti++;
756f23b8eeeSSeongJae Park }
7572fcb9362SSeongJae Park }
758f23b8eeeSSeongJae Park
7594ed98243SKaixu Xia static void damon_split_region_at(struct damon_target *t,
7604ed98243SKaixu Xia struct damon_region *r, unsigned long sz_r);
7612b8a248dSSeongJae Park
__damos_valid_target(struct damon_region * r,struct damos * s)76238683e00SSeongJae Park static bool __damos_valid_target(struct damon_region *r, struct damos *s)
76338683e00SSeongJae Park {
76438683e00SSeongJae Park unsigned long sz;
76538683e00SSeongJae Park
766ab63f63fSXin Hao sz = damon_sz_region(r);
767f5a79d7cSYajun Deng return s->pattern.min_sz_region <= sz &&
768f5a79d7cSYajun Deng sz <= s->pattern.max_sz_region &&
769f5a79d7cSYajun Deng s->pattern.min_nr_accesses <= r->nr_accesses &&
770f5a79d7cSYajun Deng r->nr_accesses <= s->pattern.max_nr_accesses &&
771f5a79d7cSYajun Deng s->pattern.min_age_region <= r->age &&
772f5a79d7cSYajun Deng r->age <= s->pattern.max_age_region;
77338683e00SSeongJae Park }
77438683e00SSeongJae Park
damos_valid_target(struct damon_ctx * c,struct damon_target * t,struct damon_region * r,struct damos * s)77538683e00SSeongJae Park static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t,
77638683e00SSeongJae Park struct damon_region *r, struct damos *s)
77738683e00SSeongJae Park {
77838683e00SSeongJae Park bool ret = __damos_valid_target(r, s);
77938683e00SSeongJae Park
780f7d911c3SSeongJae Park if (!ret || !s->quota.esz || !c->ops.get_scheme_score)
78138683e00SSeongJae Park return ret;
78238683e00SSeongJae Park
783f7d911c3SSeongJae Park return c->ops.get_scheme_score(c, t, r, s) >= s->quota.min_score;
78438683e00SSeongJae Park }
78538683e00SSeongJae Park
7862ea34989SSeongJae Park /*
7872ea34989SSeongJae Park * damos_skip_charged_region() - Check if the given region or starting part of
7882ea34989SSeongJae Park * it is already charged for the DAMOS quota.
7892ea34989SSeongJae Park * @t: The target of the region.
7902ea34989SSeongJae Park * @rp: The pointer to the region.
7912ea34989SSeongJae Park * @s: The scheme to be applied.
7922ea34989SSeongJae Park *
7932ea34989SSeongJae Park * If a quota of a scheme has exceeded in a quota charge window, the scheme's
7942ea34989SSeongJae Park * action would applied to only a part of the target access pattern fulfilling
7952ea34989SSeongJae Park * regions. To avoid applying the scheme action to only already applied
7962ea34989SSeongJae Park * regions, DAMON skips applying the scheme action to the regions that charged
7972ea34989SSeongJae Park * in the previous charge window.
7982ea34989SSeongJae Park *
7992ea34989SSeongJae Park * This function checks if a given region should be skipped or not for the
8002ea34989SSeongJae Park * reason. If only the starting part of the region has previously charged,
8012ea34989SSeongJae Park * this function splits the region into two so that the second one covers the
8022ea34989SSeongJae Park * area that not charged in the previous charge widnow and saves the second
8032ea34989SSeongJae Park * region in *rp and returns false, so that the caller can apply DAMON action
8042ea34989SSeongJae Park * to the second one.
8052ea34989SSeongJae Park *
8062ea34989SSeongJae Park * Return: true if the region should be entirely skipped, false otherwise.
8072ea34989SSeongJae Park */
damos_skip_charged_region(struct damon_target * t,struct damon_region ** rp,struct damos * s)8082ea34989SSeongJae Park static bool damos_skip_charged_region(struct damon_target *t,
8092ea34989SSeongJae Park struct damon_region **rp, struct damos *s)
8102ea34989SSeongJae Park {
8112ea34989SSeongJae Park struct damon_region *r = *rp;
8122ea34989SSeongJae Park struct damos_quota *quota = &s->quota;
8132ea34989SSeongJae Park unsigned long sz_to_skip;
8142ea34989SSeongJae Park
8152ea34989SSeongJae Park /* Skip previously charged regions */
8162ea34989SSeongJae Park if (quota->charge_target_from) {
8172ea34989SSeongJae Park if (t != quota->charge_target_from)
8182ea34989SSeongJae Park return true;
8192ea34989SSeongJae Park if (r == damon_last_region(t)) {
8202ea34989SSeongJae Park quota->charge_target_from = NULL;
8212ea34989SSeongJae Park quota->charge_addr_from = 0;
8222ea34989SSeongJae Park return true;
8232ea34989SSeongJae Park }
8242ea34989SSeongJae Park if (quota->charge_addr_from &&
8252ea34989SSeongJae Park r->ar.end <= quota->charge_addr_from)
8262ea34989SSeongJae Park return true;
8272ea34989SSeongJae Park
8282ea34989SSeongJae Park if (quota->charge_addr_from && r->ar.start <
8292ea34989SSeongJae Park quota->charge_addr_from) {
8302ea34989SSeongJae Park sz_to_skip = ALIGN_DOWN(quota->charge_addr_from -
8312ea34989SSeongJae Park r->ar.start, DAMON_MIN_REGION);
8322ea34989SSeongJae Park if (!sz_to_skip) {
8332ea34989SSeongJae Park if (damon_sz_region(r) <= DAMON_MIN_REGION)
8342ea34989SSeongJae Park return true;
8352ea34989SSeongJae Park sz_to_skip = DAMON_MIN_REGION;
8362ea34989SSeongJae Park }
8372ea34989SSeongJae Park damon_split_region_at(t, r, sz_to_skip);
8382ea34989SSeongJae Park r = damon_next_region(r);
8392ea34989SSeongJae Park *rp = r;
8402ea34989SSeongJae Park }
8412ea34989SSeongJae Park quota->charge_target_from = NULL;
8422ea34989SSeongJae Park quota->charge_addr_from = 0;
8432ea34989SSeongJae Park }
8442ea34989SSeongJae Park return false;
8452ea34989SSeongJae Park }
8462ea34989SSeongJae Park
damos_update_stat(struct damos * s,unsigned long sz_tried,unsigned long sz_applied)847d1cbbf62SSeongJae Park static void damos_update_stat(struct damos *s,
848d1cbbf62SSeongJae Park unsigned long sz_tried, unsigned long sz_applied)
849d1cbbf62SSeongJae Park {
850d1cbbf62SSeongJae Park s->stat.nr_tried++;
851d1cbbf62SSeongJae Park s->stat.sz_tried += sz_tried;
852d1cbbf62SSeongJae Park if (sz_applied)
853d1cbbf62SSeongJae Park s->stat.nr_applied++;
854d1cbbf62SSeongJae Park s->stat.sz_applied += sz_applied;
855d1cbbf62SSeongJae Park }
856d1cbbf62SSeongJae Park
__damos_filter_out(struct damon_ctx * ctx,struct damon_target * t,struct damon_region * r,struct damos_filter * filter)85717e7c724SSeongJae Park static bool __damos_filter_out(struct damon_ctx *ctx, struct damon_target *t,
85817e7c724SSeongJae Park struct damon_region *r, struct damos_filter *filter)
859ab9bda00SSeongJae Park {
860ab9bda00SSeongJae Park bool matched = false;
86117e7c724SSeongJae Park struct damon_target *ti;
86217e7c724SSeongJae Park int target_idx = 0;
863ab9bda00SSeongJae Park unsigned long start, end;
864ab9bda00SSeongJae Park
865ab9bda00SSeongJae Park switch (filter->type) {
86617e7c724SSeongJae Park case DAMOS_FILTER_TYPE_TARGET:
86717e7c724SSeongJae Park damon_for_each_target(ti, ctx) {
86817e7c724SSeongJae Park if (ti == t)
86917e7c724SSeongJae Park break;
87017e7c724SSeongJae Park target_idx++;
87117e7c724SSeongJae Park }
87217e7c724SSeongJae Park matched = target_idx == filter->target_idx;
87317e7c724SSeongJae Park break;
874ab9bda00SSeongJae Park case DAMOS_FILTER_TYPE_ADDR:
875ab9bda00SSeongJae Park start = ALIGN_DOWN(filter->addr_range.start, DAMON_MIN_REGION);
876ab9bda00SSeongJae Park end = ALIGN_DOWN(filter->addr_range.end, DAMON_MIN_REGION);
877ab9bda00SSeongJae Park
878ab9bda00SSeongJae Park /* inside the range */
879ab9bda00SSeongJae Park if (start <= r->ar.start && r->ar.end <= end) {
880ab9bda00SSeongJae Park matched = true;
881ab9bda00SSeongJae Park break;
882ab9bda00SSeongJae Park }
883ab9bda00SSeongJae Park /* outside of the range */
884ab9bda00SSeongJae Park if (r->ar.end <= start || end <= r->ar.start) {
885ab9bda00SSeongJae Park matched = false;
886ab9bda00SSeongJae Park break;
887ab9bda00SSeongJae Park }
888ab9bda00SSeongJae Park /* start before the range and overlap */
889ab9bda00SSeongJae Park if (r->ar.start < start) {
890ab9bda00SSeongJae Park damon_split_region_at(t, r, start - r->ar.start);
891ab9bda00SSeongJae Park matched = false;
892ab9bda00SSeongJae Park break;
893ab9bda00SSeongJae Park }
894ab9bda00SSeongJae Park /* start inside the range */
895ab9bda00SSeongJae Park damon_split_region_at(t, r, end - r->ar.start);
896ab9bda00SSeongJae Park matched = true;
897ab9bda00SSeongJae Park break;
898ab9bda00SSeongJae Park default:
8991777bb5cSHyeongtak Ji return false;
900ab9bda00SSeongJae Park }
901ab9bda00SSeongJae Park
902ab9bda00SSeongJae Park return matched == filter->matching;
903ab9bda00SSeongJae Park }
904ab9bda00SSeongJae Park
damos_filter_out(struct damon_ctx * ctx,struct damon_target * t,struct damon_region * r,struct damos * s)90517e7c724SSeongJae Park static bool damos_filter_out(struct damon_ctx *ctx, struct damon_target *t,
90617e7c724SSeongJae Park struct damon_region *r, struct damos *s)
907ab9bda00SSeongJae Park {
908ab9bda00SSeongJae Park struct damos_filter *filter;
909ab9bda00SSeongJae Park
910ab9bda00SSeongJae Park damos_for_each_filter(filter, s) {
91117e7c724SSeongJae Park if (__damos_filter_out(ctx, t, r, filter))
912ab9bda00SSeongJae Park return true;
913ab9bda00SSeongJae Park }
914ab9bda00SSeongJae Park return false;
915ab9bda00SSeongJae Park }
916ab9bda00SSeongJae Park
damos_apply_scheme(struct damon_ctx * c,struct damon_target * t,struct damon_region * r,struct damos * s)917e63a30c5SSeongJae Park static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
918e63a30c5SSeongJae Park struct damon_region *r, struct damos *s)
9191f366e42SSeongJae Park {
9202b8a248dSSeongJae Park struct damos_quota *quota = &s->quota;
921e63a30c5SSeongJae Park unsigned long sz = damon_sz_region(r);
9221cd24303SSeongJae Park struct timespec64 begin, end;
9230e92c2eeSSeongJae Park unsigned long sz_applied = 0;
92444467bbbSSeongJae Park int err = 0;
9252b8a248dSSeongJae Park
926f7d911c3SSeongJae Park if (c->ops.apply_scheme) {
927e63a30c5SSeongJae Park if (quota->esz && quota->charged_sz + sz > quota->esz) {
9281cd24303SSeongJae Park sz = ALIGN_DOWN(quota->esz - quota->charged_sz,
9292b8a248dSSeongJae Park DAMON_MIN_REGION);
9302b8a248dSSeongJae Park if (!sz)
9312b8a248dSSeongJae Park goto update_stat;
9324ed98243SKaixu Xia damon_split_region_at(t, r, sz);
9332b8a248dSSeongJae Park }
93417e7c724SSeongJae Park if (damos_filter_out(c, t, r, s))
935ab9bda00SSeongJae Park return;
9361cd24303SSeongJae Park ktime_get_coarse_ts64(&begin);
93744467bbbSSeongJae Park if (c->callback.before_damos_apply)
93844467bbbSSeongJae Park err = c->callback.before_damos_apply(c, t, r, s);
93944467bbbSSeongJae Park if (!err)
940f7d911c3SSeongJae Park sz_applied = c->ops.apply_scheme(c, t, r, s);
9411cd24303SSeongJae Park ktime_get_coarse_ts64(&end);
9421cd24303SSeongJae Park quota->total_charged_ns += timespec64_to_ns(&end) -
9431cd24303SSeongJae Park timespec64_to_ns(&begin);
9442b8a248dSSeongJae Park quota->charged_sz += sz;
9451cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) {
94650585192SSeongJae Park quota->charge_target_from = t;
94750585192SSeongJae Park quota->charge_addr_from = r->ar.end + 1;
94850585192SSeongJae Park }
9492b8a248dSSeongJae Park }
9502f0b548cSSeongJae Park if (s->action != DAMOS_STAT)
9511f366e42SSeongJae Park r->age = 0;
9522b8a248dSSeongJae Park
9532b8a248dSSeongJae Park update_stat:
954d1cbbf62SSeongJae Park damos_update_stat(s, sz, sz_applied);
9551f366e42SSeongJae Park }
956e63a30c5SSeongJae Park
damon_do_apply_schemes(struct damon_ctx * c,struct damon_target * t,struct damon_region * r)957e63a30c5SSeongJae Park static void damon_do_apply_schemes(struct damon_ctx *c,
958e63a30c5SSeongJae Park struct damon_target *t,
959e63a30c5SSeongJae Park struct damon_region *r)
960e63a30c5SSeongJae Park {
961e63a30c5SSeongJae Park struct damos *s;
962e63a30c5SSeongJae Park
963e63a30c5SSeongJae Park damon_for_each_scheme(s, c) {
964e63a30c5SSeongJae Park struct damos_quota *quota = &s->quota;
965e63a30c5SSeongJae Park
966e63a30c5SSeongJae Park if (!s->wmarks.activated)
967e63a30c5SSeongJae Park continue;
968e63a30c5SSeongJae Park
969e63a30c5SSeongJae Park /* Check the quota */
970e63a30c5SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz)
971e63a30c5SSeongJae Park continue;
972e63a30c5SSeongJae Park
973e63a30c5SSeongJae Park if (damos_skip_charged_region(t, &r, s))
974e63a30c5SSeongJae Park continue;
975e63a30c5SSeongJae Park
976e63a30c5SSeongJae Park if (!damos_valid_target(c, t, r, s))
977e63a30c5SSeongJae Park continue;
978e63a30c5SSeongJae Park
979e63a30c5SSeongJae Park damos_apply_scheme(c, t, r, s);
980e63a30c5SSeongJae Park }
9811f366e42SSeongJae Park }
9821f366e42SSeongJae Park
9831cd24303SSeongJae Park /* Shouldn't be called if quota->ms and quota->sz are zero */
damos_set_effective_quota(struct damos_quota * quota)9841cd24303SSeongJae Park static void damos_set_effective_quota(struct damos_quota *quota)
9851cd24303SSeongJae Park {
9861cd24303SSeongJae Park unsigned long throughput;
9871cd24303SSeongJae Park unsigned long esz;
9881cd24303SSeongJae Park
9891cd24303SSeongJae Park if (!quota->ms) {
9901cd24303SSeongJae Park quota->esz = quota->sz;
9911cd24303SSeongJae Park return;
9921cd24303SSeongJae Park }
9931cd24303SSeongJae Park
9941cd24303SSeongJae Park if (quota->total_charged_ns)
9951cd24303SSeongJae Park throughput = quota->total_charged_sz * 1000000 /
9961cd24303SSeongJae Park quota->total_charged_ns;
9971cd24303SSeongJae Park else
9981cd24303SSeongJae Park throughput = PAGE_SIZE * 1024;
9991cd24303SSeongJae Park esz = throughput * quota->ms;
10001cd24303SSeongJae Park
10011cd24303SSeongJae Park if (quota->sz && quota->sz < esz)
10021cd24303SSeongJae Park esz = quota->sz;
10031cd24303SSeongJae Park quota->esz = esz;
10041cd24303SSeongJae Park }
10051cd24303SSeongJae Park
damos_adjust_quota(struct damon_ctx * c,struct damos * s)1006898810e5SSeongJae Park static void damos_adjust_quota(struct damon_ctx *c, struct damos *s)
10071f366e42SSeongJae Park {
10082b8a248dSSeongJae Park struct damos_quota *quota = &s->quota;
1009898810e5SSeongJae Park struct damon_target *t;
1010898810e5SSeongJae Park struct damon_region *r;
101138683e00SSeongJae Park unsigned long cumulated_sz;
101238683e00SSeongJae Park unsigned int score, max_score = 0;
10132b8a248dSSeongJae Park
10141cd24303SSeongJae Park if (!quota->ms && !quota->sz)
1015898810e5SSeongJae Park return;
10162b8a248dSSeongJae Park
10172b8a248dSSeongJae Park /* New charge window starts */
10182b8a248dSSeongJae Park if (time_after_eq(jiffies, quota->charged_from +
1019898810e5SSeongJae Park msecs_to_jiffies(quota->reset_interval))) {
10206268eac3SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz)
10216268eac3SSeongJae Park s->stat.qt_exceeds++;
10221cd24303SSeongJae Park quota->total_charged_sz += quota->charged_sz;
10232b8a248dSSeongJae Park quota->charged_from = jiffies;
10242b8a248dSSeongJae Park quota->charged_sz = 0;
10251cd24303SSeongJae Park damos_set_effective_quota(quota);
10262b8a248dSSeongJae Park }
102738683e00SSeongJae Park
1028f7d911c3SSeongJae Park if (!c->ops.get_scheme_score)
1029898810e5SSeongJae Park return;
103038683e00SSeongJae Park
103138683e00SSeongJae Park /* Fill up the score histogram */
103238683e00SSeongJae Park memset(quota->histogram, 0, sizeof(quota->histogram));
103338683e00SSeongJae Park damon_for_each_target(t, c) {
103438683e00SSeongJae Park damon_for_each_region(r, t) {
103538683e00SSeongJae Park if (!__damos_valid_target(r, s))
103638683e00SSeongJae Park continue;
1037898810e5SSeongJae Park score = c->ops.get_scheme_score(c, t, r, s);
1038ab63f63fSXin Hao quota->histogram[score] += damon_sz_region(r);
103938683e00SSeongJae Park if (score > max_score)
104038683e00SSeongJae Park max_score = score;
104138683e00SSeongJae Park }
104238683e00SSeongJae Park }
104338683e00SSeongJae Park
104438683e00SSeongJae Park /* Set the min score limit */
104538683e00SSeongJae Park for (cumulated_sz = 0, score = max_score; ; score--) {
104638683e00SSeongJae Park cumulated_sz += quota->histogram[score];
104738683e00SSeongJae Park if (cumulated_sz >= quota->esz || !score)
104838683e00SSeongJae Park break;
104938683e00SSeongJae Park }
105038683e00SSeongJae Park quota->min_score = score;
10512b8a248dSSeongJae Park }
10521f366e42SSeongJae Park
kdamond_apply_schemes(struct damon_ctx * c)1053898810e5SSeongJae Park static void kdamond_apply_schemes(struct damon_ctx *c)
1054898810e5SSeongJae Park {
1055898810e5SSeongJae Park struct damon_target *t;
1056898810e5SSeongJae Park struct damon_region *r, *next_r;
1057898810e5SSeongJae Park struct damos *s;
1058898810e5SSeongJae Park
1059898810e5SSeongJae Park damon_for_each_scheme(s, c) {
1060898810e5SSeongJae Park if (!s->wmarks.activated)
1061898810e5SSeongJae Park continue;
1062898810e5SSeongJae Park
1063898810e5SSeongJae Park damos_adjust_quota(c, s);
1064898810e5SSeongJae Park }
1065898810e5SSeongJae Park
10661f366e42SSeongJae Park damon_for_each_target(t, c) {
10672b8a248dSSeongJae Park damon_for_each_region_safe(r, next_r, t)
10681f366e42SSeongJae Park damon_do_apply_schemes(c, t, r);
10691f366e42SSeongJae Park }
10701f366e42SSeongJae Park }
10711f366e42SSeongJae Park
1072b9a6ac4eSSeongJae Park /*
1073b9a6ac4eSSeongJae Park * Merge two adjacent regions into one region
1074b9a6ac4eSSeongJae Park */
damon_merge_two_regions(struct damon_target * t,struct damon_region * l,struct damon_region * r)1075b9a6ac4eSSeongJae Park static void damon_merge_two_regions(struct damon_target *t,
1076b9a6ac4eSSeongJae Park struct damon_region *l, struct damon_region *r)
1077b9a6ac4eSSeongJae Park {
1078652e0446SXin Hao unsigned long sz_l = damon_sz_region(l), sz_r = damon_sz_region(r);
1079b9a6ac4eSSeongJae Park
1080b9a6ac4eSSeongJae Park l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) /
1081b9a6ac4eSSeongJae Park (sz_l + sz_r);
1082fda504faSSeongJae Park l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r);
1083b9a6ac4eSSeongJae Park l->ar.end = r->ar.end;
1084b9a6ac4eSSeongJae Park damon_destroy_region(r, t);
1085b9a6ac4eSSeongJae Park }
1086b9a6ac4eSSeongJae Park
1087b9a6ac4eSSeongJae Park /*
1088b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies
1089b9a6ac4eSSeongJae Park *
1090b9a6ac4eSSeongJae Park * t target affected by this merge operation
1091b9a6ac4eSSeongJae Park * thres '->nr_accesses' diff threshold for the merge
1092b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region
1093b9a6ac4eSSeongJae Park */
damon_merge_regions_of(struct damon_target * t,unsigned int thres,unsigned long sz_limit)1094b9a6ac4eSSeongJae Park static void damon_merge_regions_of(struct damon_target *t, unsigned int thres,
1095b9a6ac4eSSeongJae Park unsigned long sz_limit)
1096b9a6ac4eSSeongJae Park {
1097b9a6ac4eSSeongJae Park struct damon_region *r, *prev = NULL, *next;
1098b9a6ac4eSSeongJae Park
1099b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) {
1100d720bbbdSXin Hao if (abs(r->nr_accesses - r->last_nr_accesses) > thres)
1101fda504faSSeongJae Park r->age = 0;
1102fda504faSSeongJae Park else
1103fda504faSSeongJae Park r->age++;
1104fda504faSSeongJae Park
1105b9a6ac4eSSeongJae Park if (prev && prev->ar.end == r->ar.start &&
1106d720bbbdSXin Hao abs(prev->nr_accesses - r->nr_accesses) <= thres &&
1107652e0446SXin Hao damon_sz_region(prev) + damon_sz_region(r) <= sz_limit)
1108b9a6ac4eSSeongJae Park damon_merge_two_regions(t, prev, r);
1109b9a6ac4eSSeongJae Park else
1110b9a6ac4eSSeongJae Park prev = r;
1111b9a6ac4eSSeongJae Park }
1112b9a6ac4eSSeongJae Park }
1113b9a6ac4eSSeongJae Park
1114b9a6ac4eSSeongJae Park /*
1115b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies
1116b9a6ac4eSSeongJae Park *
1117b9a6ac4eSSeongJae Park * threshold '->nr_accesses' diff threshold for the merge
1118b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region
1119b9a6ac4eSSeongJae Park *
1120b9a6ac4eSSeongJae Park * This function merges monitoring target regions which are adjacent and their
1121b9a6ac4eSSeongJae Park * access frequencies are similar. This is for minimizing the monitoring
1122b9a6ac4eSSeongJae Park * overhead under the dynamically changeable access pattern. If a merge was
1123b9a6ac4eSSeongJae Park * unnecessarily made, later 'kdamond_split_regions()' will revert it.
1124b9a6ac4eSSeongJae Park */
kdamond_merge_regions(struct damon_ctx * c,unsigned int threshold,unsigned long sz_limit)1125b9a6ac4eSSeongJae Park static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold,
1126b9a6ac4eSSeongJae Park unsigned long sz_limit)
1127b9a6ac4eSSeongJae Park {
1128b9a6ac4eSSeongJae Park struct damon_target *t;
1129b9a6ac4eSSeongJae Park
1130b9a6ac4eSSeongJae Park damon_for_each_target(t, c)
1131b9a6ac4eSSeongJae Park damon_merge_regions_of(t, threshold, sz_limit);
1132b9a6ac4eSSeongJae Park }
1133b9a6ac4eSSeongJae Park
1134b9a6ac4eSSeongJae Park /*
1135b9a6ac4eSSeongJae Park * Split a region in two
1136b9a6ac4eSSeongJae Park *
1137b9a6ac4eSSeongJae Park * r the region to be split
1138b9a6ac4eSSeongJae Park * sz_r size of the first sub-region that will be made
1139b9a6ac4eSSeongJae Park */
damon_split_region_at(struct damon_target * t,struct damon_region * r,unsigned long sz_r)11404ed98243SKaixu Xia static void damon_split_region_at(struct damon_target *t,
11414ed98243SKaixu Xia struct damon_region *r, unsigned long sz_r)
1142b9a6ac4eSSeongJae Park {
1143b9a6ac4eSSeongJae Park struct damon_region *new;
1144b9a6ac4eSSeongJae Park
1145b9a6ac4eSSeongJae Park new = damon_new_region(r->ar.start + sz_r, r->ar.end);
1146b9a6ac4eSSeongJae Park if (!new)
1147b9a6ac4eSSeongJae Park return;
1148b9a6ac4eSSeongJae Park
1149b9a6ac4eSSeongJae Park r->ar.end = new->ar.start;
1150b9a6ac4eSSeongJae Park
1151fda504faSSeongJae Park new->age = r->age;
1152fda504faSSeongJae Park new->last_nr_accesses = r->last_nr_accesses;
1153fda504faSSeongJae Park
1154b9a6ac4eSSeongJae Park damon_insert_region(new, r, damon_next_region(r), t);
1155b9a6ac4eSSeongJae Park }
1156b9a6ac4eSSeongJae Park
1157b9a6ac4eSSeongJae Park /* Split every region in the given target into 'nr_subs' regions */
damon_split_regions_of(struct damon_target * t,int nr_subs)11584ed98243SKaixu Xia static void damon_split_regions_of(struct damon_target *t, int nr_subs)
1159b9a6ac4eSSeongJae Park {
1160b9a6ac4eSSeongJae Park struct damon_region *r, *next;
1161b9a6ac4eSSeongJae Park unsigned long sz_region, sz_sub = 0;
1162b9a6ac4eSSeongJae Park int i;
1163b9a6ac4eSSeongJae Park
1164b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) {
1165ab63f63fSXin Hao sz_region = damon_sz_region(r);
1166b9a6ac4eSSeongJae Park
1167b9a6ac4eSSeongJae Park for (i = 0; i < nr_subs - 1 &&
1168b9a6ac4eSSeongJae Park sz_region > 2 * DAMON_MIN_REGION; i++) {
1169b9a6ac4eSSeongJae Park /*
1170b9a6ac4eSSeongJae Park * Randomly select size of left sub-region to be at
1171b9a6ac4eSSeongJae Park * least 10 percent and at most 90% of original region
1172b9a6ac4eSSeongJae Park */
1173b9a6ac4eSSeongJae Park sz_sub = ALIGN_DOWN(damon_rand(1, 10) *
1174b9a6ac4eSSeongJae Park sz_region / 10, DAMON_MIN_REGION);
1175b9a6ac4eSSeongJae Park /* Do not allow blank region */
1176b9a6ac4eSSeongJae Park if (sz_sub == 0 || sz_sub >= sz_region)
1177b9a6ac4eSSeongJae Park continue;
1178b9a6ac4eSSeongJae Park
11794ed98243SKaixu Xia damon_split_region_at(t, r, sz_sub);
1180b9a6ac4eSSeongJae Park sz_region = sz_sub;
1181b9a6ac4eSSeongJae Park }
1182b9a6ac4eSSeongJae Park }
1183b9a6ac4eSSeongJae Park }
1184b9a6ac4eSSeongJae Park
1185b9a6ac4eSSeongJae Park /*
1186b9a6ac4eSSeongJae Park * Split every target region into randomly-sized small regions
1187b9a6ac4eSSeongJae Park *
1188b9a6ac4eSSeongJae Park * This function splits every target region into random-sized small regions if
1189b9a6ac4eSSeongJae Park * current total number of the regions is equal or smaller than half of the
1190b9a6ac4eSSeongJae Park * user-specified maximum number of regions. This is for maximizing the
1191b9a6ac4eSSeongJae Park * monitoring accuracy under the dynamically changeable access patterns. If a
1192b9a6ac4eSSeongJae Park * split was unnecessarily made, later 'kdamond_merge_regions()' will revert
1193b9a6ac4eSSeongJae Park * it.
1194b9a6ac4eSSeongJae Park */
kdamond_split_regions(struct damon_ctx * ctx)1195b9a6ac4eSSeongJae Park static void kdamond_split_regions(struct damon_ctx *ctx)
1196b9a6ac4eSSeongJae Park {
1197b9a6ac4eSSeongJae Park struct damon_target *t;
1198b9a6ac4eSSeongJae Park unsigned int nr_regions = 0;
1199b9a6ac4eSSeongJae Park static unsigned int last_nr_regions;
1200b9a6ac4eSSeongJae Park int nr_subregions = 2;
1201b9a6ac4eSSeongJae Park
1202b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx)
1203b9a6ac4eSSeongJae Park nr_regions += damon_nr_regions(t);
1204b9a6ac4eSSeongJae Park
1205cbeaa77bSSeongJae Park if (nr_regions > ctx->attrs.max_nr_regions / 2)
1206b9a6ac4eSSeongJae Park return;
1207b9a6ac4eSSeongJae Park
1208b9a6ac4eSSeongJae Park /* Maybe the middle of the region has different access frequency */
1209b9a6ac4eSSeongJae Park if (last_nr_regions == nr_regions &&
1210cbeaa77bSSeongJae Park nr_regions < ctx->attrs.max_nr_regions / 3)
1211b9a6ac4eSSeongJae Park nr_subregions = 3;
1212b9a6ac4eSSeongJae Park
1213b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx)
12144ed98243SKaixu Xia damon_split_regions_of(t, nr_subregions);
1215b9a6ac4eSSeongJae Park
1216b9a6ac4eSSeongJae Park last_nr_regions = nr_regions;
1217b9a6ac4eSSeongJae Park }
1218b9a6ac4eSSeongJae Park
1219f23b8eeeSSeongJae Park /*
12202224d848SSeongJae Park * Check whether current monitoring should be stopped
12212224d848SSeongJae Park *
12222224d848SSeongJae Park * The monitoring is stopped when either the user requested to stop, or all
12232224d848SSeongJae Park * monitoring targets are invalid.
12242224d848SSeongJae Park *
12252224d848SSeongJae Park * Returns true if need to stop current monitoring.
12262224d848SSeongJae Park */
kdamond_need_stop(struct damon_ctx * ctx)12272224d848SSeongJae Park static bool kdamond_need_stop(struct damon_ctx *ctx)
12282224d848SSeongJae Park {
1229f23b8eeeSSeongJae Park struct damon_target *t;
12302224d848SSeongJae Park
12310f91d133SChangbin Du if (kthread_should_stop())
12322224d848SSeongJae Park return true;
12332224d848SSeongJae Park
1234f7d911c3SSeongJae Park if (!ctx->ops.target_valid)
12352224d848SSeongJae Park return false;
12362224d848SSeongJae Park
1237f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) {
1238f7d911c3SSeongJae Park if (ctx->ops.target_valid(t))
1239f23b8eeeSSeongJae Park return false;
1240f23b8eeeSSeongJae Park }
1241f23b8eeeSSeongJae Park
1242f23b8eeeSSeongJae Park return true;
12432224d848SSeongJae Park }
12442224d848SSeongJae Park
damos_wmark_metric_value(enum damos_wmark_metric metric)1245ee801b7dSSeongJae Park static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric)
1246ee801b7dSSeongJae Park {
1247ee801b7dSSeongJae Park struct sysinfo i;
1248ee801b7dSSeongJae Park
1249ee801b7dSSeongJae Park switch (metric) {
1250ee801b7dSSeongJae Park case DAMOS_WMARK_FREE_MEM_RATE:
1251ee801b7dSSeongJae Park si_meminfo(&i);
1252ee801b7dSSeongJae Park return i.freeram * 1000 / i.totalram;
1253ee801b7dSSeongJae Park default:
1254ee801b7dSSeongJae Park break;
1255ee801b7dSSeongJae Park }
1256ee801b7dSSeongJae Park return -EINVAL;
1257ee801b7dSSeongJae Park }
1258ee801b7dSSeongJae Park
1259ee801b7dSSeongJae Park /*
1260ee801b7dSSeongJae Park * Returns zero if the scheme is active. Else, returns time to wait for next
1261ee801b7dSSeongJae Park * watermark check in micro-seconds.
1262ee801b7dSSeongJae Park */
damos_wmark_wait_us(struct damos * scheme)1263ee801b7dSSeongJae Park static unsigned long damos_wmark_wait_us(struct damos *scheme)
1264ee801b7dSSeongJae Park {
1265ee801b7dSSeongJae Park unsigned long metric;
1266ee801b7dSSeongJae Park
1267ee801b7dSSeongJae Park if (scheme->wmarks.metric == DAMOS_WMARK_NONE)
1268ee801b7dSSeongJae Park return 0;
1269ee801b7dSSeongJae Park
1270ee801b7dSSeongJae Park metric = damos_wmark_metric_value(scheme->wmarks.metric);
1271ee801b7dSSeongJae Park /* higher than high watermark or lower than low watermark */
1272ee801b7dSSeongJae Park if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) {
1273ee801b7dSSeongJae Park if (scheme->wmarks.activated)
127401078655SColin Ian King pr_debug("deactivate a scheme (%d) for %s wmark\n",
1275ee801b7dSSeongJae Park scheme->action,
1276ee801b7dSSeongJae Park metric > scheme->wmarks.high ?
1277ee801b7dSSeongJae Park "high" : "low");
1278ee801b7dSSeongJae Park scheme->wmarks.activated = false;
1279ee801b7dSSeongJae Park return scheme->wmarks.interval;
1280ee801b7dSSeongJae Park }
1281ee801b7dSSeongJae Park
1282ee801b7dSSeongJae Park /* inactive and higher than middle watermark */
1283ee801b7dSSeongJae Park if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) &&
1284ee801b7dSSeongJae Park !scheme->wmarks.activated)
1285ee801b7dSSeongJae Park return scheme->wmarks.interval;
1286ee801b7dSSeongJae Park
1287ee801b7dSSeongJae Park if (!scheme->wmarks.activated)
1288ee801b7dSSeongJae Park pr_debug("activate a scheme (%d)\n", scheme->action);
1289ee801b7dSSeongJae Park scheme->wmarks.activated = true;
1290ee801b7dSSeongJae Park return 0;
1291ee801b7dSSeongJae Park }
1292ee801b7dSSeongJae Park
kdamond_usleep(unsigned long usecs)1293ee801b7dSSeongJae Park static void kdamond_usleep(unsigned long usecs)
1294ee801b7dSSeongJae Park {
12954de46a30SSeongJae Park /* See Documentation/timers/timers-howto.rst for the thresholds */
12964de46a30SSeongJae Park if (usecs > 20 * USEC_PER_MSEC)
129770e92748SSeongJae Park schedule_timeout_idle(usecs_to_jiffies(usecs));
1298ee801b7dSSeongJae Park else
129970e92748SSeongJae Park usleep_idle_range(usecs, usecs + 1);
1300ee801b7dSSeongJae Park }
1301ee801b7dSSeongJae Park
1302ee801b7dSSeongJae Park /* Returns negative error code if it's not activated but should return */
kdamond_wait_activation(struct damon_ctx * ctx)1303ee801b7dSSeongJae Park static int kdamond_wait_activation(struct damon_ctx *ctx)
1304ee801b7dSSeongJae Park {
1305ee801b7dSSeongJae Park struct damos *s;
1306ee801b7dSSeongJae Park unsigned long wait_time;
1307ee801b7dSSeongJae Park unsigned long min_wait_time = 0;
130878049e94SJonghyeon Kim bool init_wait_time = false;
1309ee801b7dSSeongJae Park
1310ee801b7dSSeongJae Park while (!kdamond_need_stop(ctx)) {
1311ee801b7dSSeongJae Park damon_for_each_scheme(s, ctx) {
1312ee801b7dSSeongJae Park wait_time = damos_wmark_wait_us(s);
131378049e94SJonghyeon Kim if (!init_wait_time || wait_time < min_wait_time) {
131478049e94SJonghyeon Kim init_wait_time = true;
1315ee801b7dSSeongJae Park min_wait_time = wait_time;
1316ee801b7dSSeongJae Park }
131778049e94SJonghyeon Kim }
1318ee801b7dSSeongJae Park if (!min_wait_time)
1319ee801b7dSSeongJae Park return 0;
1320ee801b7dSSeongJae Park
1321ee801b7dSSeongJae Park kdamond_usleep(min_wait_time);
13226e74d2bfSSeongJae Park
13236e74d2bfSSeongJae Park if (ctx->callback.after_wmarks_check &&
13246e74d2bfSSeongJae Park ctx->callback.after_wmarks_check(ctx))
13256e74d2bfSSeongJae Park break;
1326ee801b7dSSeongJae Park }
1327ee801b7dSSeongJae Park return -EBUSY;
1328ee801b7dSSeongJae Park }
1329ee801b7dSSeongJae Park
kdamond_init_intervals_sis(struct damon_ctx * ctx)1330c708a5e5SSeongJae Park static void kdamond_init_intervals_sis(struct damon_ctx *ctx)
1331c708a5e5SSeongJae Park {
1332c708a5e5SSeongJae Park unsigned long sample_interval = ctx->attrs.sample_interval ?
1333c708a5e5SSeongJae Park ctx->attrs.sample_interval : 1;
1334c708a5e5SSeongJae Park
1335c708a5e5SSeongJae Park ctx->passed_sample_intervals = 0;
1336c708a5e5SSeongJae Park ctx->next_aggregation_sis = ctx->attrs.aggr_interval / sample_interval;
1337c708a5e5SSeongJae Park ctx->next_ops_update_sis = ctx->attrs.ops_update_interval /
1338c708a5e5SSeongJae Park sample_interval;
1339c708a5e5SSeongJae Park }
1340c708a5e5SSeongJae Park
13412224d848SSeongJae Park /*
13422224d848SSeongJae Park * The monitoring daemon that runs as a kernel thread
13432224d848SSeongJae Park */
kdamond_fn(void * data)13442224d848SSeongJae Park static int kdamond_fn(void *data)
13452224d848SSeongJae Park {
1346cef4493fSYu Zhe struct damon_ctx *ctx = data;
1347f23b8eeeSSeongJae Park struct damon_target *t;
1348f23b8eeeSSeongJae Park struct damon_region *r, *next;
1349b9a6ac4eSSeongJae Park unsigned int max_nr_accesses = 0;
1350b9a6ac4eSSeongJae Park unsigned long sz_limit = 0;
13512224d848SSeongJae Park
135242e4cef5SChangbin Du pr_debug("kdamond (%d) starts\n", current->pid);
13532224d848SSeongJae Park
1354e93bcaebSSeongJae Park complete(&ctx->kdamond_started);
1355c708a5e5SSeongJae Park kdamond_init_intervals_sis(ctx);
1356c708a5e5SSeongJae Park
1357f7d911c3SSeongJae Park if (ctx->ops.init)
1358f7d911c3SSeongJae Park ctx->ops.init(ctx);
13592224d848SSeongJae Park if (ctx->callback.before_start && ctx->callback.before_start(ctx))
136029454cf6SKaixu Xia goto done;
13612224d848SSeongJae Park
1362b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx);
1363b9a6ac4eSSeongJae Park
136429454cf6SKaixu Xia while (!kdamond_need_stop(ctx)) {
1365c708a5e5SSeongJae Park /*
1366c708a5e5SSeongJae Park * ctx->attrs and ctx->next_{aggregation,ops_update}_sis could
1367c708a5e5SSeongJae Park * be changed from after_wmarks_check() or after_aggregation()
1368c708a5e5SSeongJae Park * callbacks. Read the values here, and use those for this
1369c708a5e5SSeongJae Park * iteration. That is, damon_set_attrs() updated new values
1370c708a5e5SSeongJae Park * are respected from next iteration.
1371c708a5e5SSeongJae Park */
1372c708a5e5SSeongJae Park unsigned long next_aggregation_sis = ctx->next_aggregation_sis;
1373c708a5e5SSeongJae Park unsigned long next_ops_update_sis = ctx->next_ops_update_sis;
1374c708a5e5SSeongJae Park unsigned long sample_interval = ctx->attrs.sample_interval;
1375c708a5e5SSeongJae Park
137629454cf6SKaixu Xia if (kdamond_wait_activation(ctx))
137729454cf6SKaixu Xia break;
1378ee801b7dSSeongJae Park
1379f7d911c3SSeongJae Park if (ctx->ops.prepare_access_checks)
1380f7d911c3SSeongJae Park ctx->ops.prepare_access_checks(ctx);
13812224d848SSeongJae Park if (ctx->callback.after_sampling &&
138229454cf6SKaixu Xia ctx->callback.after_sampling(ctx))
138329454cf6SKaixu Xia break;
13842224d848SSeongJae Park
1385c708a5e5SSeongJae Park kdamond_usleep(sample_interval);
1386c708a5e5SSeongJae Park ctx->passed_sample_intervals++;
13872224d848SSeongJae Park
1388f7d911c3SSeongJae Park if (ctx->ops.check_accesses)
1389f7d911c3SSeongJae Park max_nr_accesses = ctx->ops.check_accesses(ctx);
13902224d848SSeongJae Park
1391c708a5e5SSeongJae Park sample_interval = ctx->attrs.sample_interval ?
1392c708a5e5SSeongJae Park ctx->attrs.sample_interval : 1;
1393c708a5e5SSeongJae Park if (ctx->passed_sample_intervals == next_aggregation_sis) {
1394c708a5e5SSeongJae Park ctx->next_aggregation_sis = next_aggregation_sis +
1395c708a5e5SSeongJae Park ctx->attrs.aggr_interval / sample_interval;
1396b9a6ac4eSSeongJae Park kdamond_merge_regions(ctx,
1397b9a6ac4eSSeongJae Park max_nr_accesses / 10,
1398b9a6ac4eSSeongJae Park sz_limit);
13992224d848SSeongJae Park if (ctx->callback.after_aggregation &&
140029454cf6SKaixu Xia ctx->callback.after_aggregation(ctx))
140129454cf6SKaixu Xia break;
140264517d6eSHuaisheng Ye if (!list_empty(&ctx->schemes))
14031f366e42SSeongJae Park kdamond_apply_schemes(ctx);
1404f23b8eeeSSeongJae Park kdamond_reset_aggregated(ctx);
1405b9a6ac4eSSeongJae Park kdamond_split_regions(ctx);
1406f7d911c3SSeongJae Park if (ctx->ops.reset_aggregated)
1407f7d911c3SSeongJae Park ctx->ops.reset_aggregated(ctx);
14082224d848SSeongJae Park }
14092224d848SSeongJae Park
1410c708a5e5SSeongJae Park if (ctx->passed_sample_intervals == next_ops_update_sis) {
1411c708a5e5SSeongJae Park ctx->next_ops_update_sis = next_ops_update_sis +
1412c708a5e5SSeongJae Park ctx->attrs.ops_update_interval /
1413c708a5e5SSeongJae Park sample_interval;
1414f7d911c3SSeongJae Park if (ctx->ops.update)
1415f7d911c3SSeongJae Park ctx->ops.update(ctx);
1416b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx);
14172224d848SSeongJae Park }
14182224d848SSeongJae Park }
141929454cf6SKaixu Xia done:
1420f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) {
1421f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t)
1422b9a6ac4eSSeongJae Park damon_destroy_region(r, t);
1423f23b8eeeSSeongJae Park }
14242224d848SSeongJae Park
14250f91d133SChangbin Du if (ctx->callback.before_terminate)
14260f91d133SChangbin Du ctx->callback.before_terminate(ctx);
1427f7d911c3SSeongJae Park if (ctx->ops.cleanup)
1428f7d911c3SSeongJae Park ctx->ops.cleanup(ctx);
14292224d848SSeongJae Park
143042e4cef5SChangbin Du pr_debug("kdamond (%d) finishes\n", current->pid);
14312224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock);
14322224d848SSeongJae Park ctx->kdamond = NULL;
14332224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock);
14342224d848SSeongJae Park
14352224d848SSeongJae Park mutex_lock(&damon_lock);
14362224d848SSeongJae Park nr_running_ctxs--;
14378b9b0d33SSeongJae Park if (!nr_running_ctxs && running_exclusive_ctxs)
14388b9b0d33SSeongJae Park running_exclusive_ctxs = false;
14392224d848SSeongJae Park mutex_unlock(&damon_lock);
14402224d848SSeongJae Park
14415f7fe2b9SChangbin Du return 0;
14422224d848SSeongJae Park }
144317ccae8bSSeongJae Park
14440d83b2d8SXin Hao /*
14450d83b2d8SXin Hao * struct damon_system_ram_region - System RAM resource address region of
14460d83b2d8SXin Hao * [@start, @end).
14470d83b2d8SXin Hao * @start: Start address of the region (inclusive).
14480d83b2d8SXin Hao * @end: End address of the region (exclusive).
14490d83b2d8SXin Hao */
14500d83b2d8SXin Hao struct damon_system_ram_region {
14510d83b2d8SXin Hao unsigned long start;
14520d83b2d8SXin Hao unsigned long end;
14530d83b2d8SXin Hao };
14540d83b2d8SXin Hao
walk_system_ram(struct resource * res,void * arg)14550d83b2d8SXin Hao static int walk_system_ram(struct resource *res, void *arg)
14560d83b2d8SXin Hao {
14570d83b2d8SXin Hao struct damon_system_ram_region *a = arg;
14580d83b2d8SXin Hao
14590d83b2d8SXin Hao if (a->end - a->start < resource_size(res)) {
14600d83b2d8SXin Hao a->start = res->start;
14610d83b2d8SXin Hao a->end = res->end;
14620d83b2d8SXin Hao }
14630d83b2d8SXin Hao return 0;
14640d83b2d8SXin Hao }
14650d83b2d8SXin Hao
14660d83b2d8SXin Hao /*
14670d83b2d8SXin Hao * Find biggest 'System RAM' resource and store its start and end address in
14680d83b2d8SXin Hao * @start and @end, respectively. If no System RAM is found, returns false.
14690d83b2d8SXin Hao */
damon_find_biggest_system_ram(unsigned long * start,unsigned long * end)1470233f0b31SKaixu Xia static bool damon_find_biggest_system_ram(unsigned long *start,
1471233f0b31SKaixu Xia unsigned long *end)
14720d83b2d8SXin Hao
14730d83b2d8SXin Hao {
14740d83b2d8SXin Hao struct damon_system_ram_region arg = {};
14750d83b2d8SXin Hao
14760d83b2d8SXin Hao walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram);
14770d83b2d8SXin Hao if (arg.end <= arg.start)
14780d83b2d8SXin Hao return false;
14790d83b2d8SXin Hao
14800d83b2d8SXin Hao *start = arg.start;
14810d83b2d8SXin Hao *end = arg.end;
14820d83b2d8SXin Hao return true;
14830d83b2d8SXin Hao }
14840d83b2d8SXin Hao
1485233f0b31SKaixu Xia /**
1486233f0b31SKaixu Xia * damon_set_region_biggest_system_ram_default() - Set the region of the given
1487233f0b31SKaixu Xia * monitoring target as requested, or biggest 'System RAM'.
1488233f0b31SKaixu Xia * @t: The monitoring target to set the region.
1489233f0b31SKaixu Xia * @start: The pointer to the start address of the region.
1490233f0b31SKaixu Xia * @end: The pointer to the end address of the region.
1491233f0b31SKaixu Xia *
1492233f0b31SKaixu Xia * This function sets the region of @t as requested by @start and @end. If the
1493233f0b31SKaixu Xia * values of @start and @end are zero, however, this function finds the biggest
1494233f0b31SKaixu Xia * 'System RAM' resource and sets the region to cover the resource. In the
1495233f0b31SKaixu Xia * latter case, this function saves the start and end addresses of the resource
1496233f0b31SKaixu Xia * in @start and @end, respectively.
1497233f0b31SKaixu Xia *
1498233f0b31SKaixu Xia * Return: 0 on success, negative error code otherwise.
1499233f0b31SKaixu Xia */
damon_set_region_biggest_system_ram_default(struct damon_target * t,unsigned long * start,unsigned long * end)1500233f0b31SKaixu Xia int damon_set_region_biggest_system_ram_default(struct damon_target *t,
1501233f0b31SKaixu Xia unsigned long *start, unsigned long *end)
1502233f0b31SKaixu Xia {
1503233f0b31SKaixu Xia struct damon_addr_range addr_range;
1504233f0b31SKaixu Xia
1505233f0b31SKaixu Xia if (*start > *end)
1506233f0b31SKaixu Xia return -EINVAL;
1507233f0b31SKaixu Xia
1508233f0b31SKaixu Xia if (!*start && !*end &&
1509233f0b31SKaixu Xia !damon_find_biggest_system_ram(start, end))
1510233f0b31SKaixu Xia return -EINVAL;
1511233f0b31SKaixu Xia
1512233f0b31SKaixu Xia addr_range.start = *start;
1513233f0b31SKaixu Xia addr_range.end = *end;
1514233f0b31SKaixu Xia return damon_set_regions(t, &addr_range, 1);
1515233f0b31SKaixu Xia }
1516233f0b31SKaixu Xia
damon_init(void)1517a1870944SDawei Li static int __init damon_init(void)
1518a1870944SDawei Li {
1519a1870944SDawei Li damon_region_cache = KMEM_CACHE(damon_region, 0);
1520a1870944SDawei Li if (unlikely(!damon_region_cache)) {
1521a1870944SDawei Li pr_err("creating damon_region_cache fails\n");
1522a1870944SDawei Li return -ENOMEM;
1523a1870944SDawei Li }
1524a1870944SDawei Li
1525a1870944SDawei Li return 0;
1526a1870944SDawei Li }
1527a1870944SDawei Li
1528a1870944SDawei Li subsys_initcall(damon_init);
1529a1870944SDawei Li
153017ccae8bSSeongJae Park #include "core-test.h"
1531