12224d848SSeongJae Park // SPDX-License-Identifier: GPL-2.0 22224d848SSeongJae Park /* 32224d848SSeongJae Park * Data Access Monitor 42224d848SSeongJae Park * 52224d848SSeongJae Park * Author: SeongJae Park <sjpark@amazon.de> 62224d848SSeongJae Park */ 72224d848SSeongJae Park 82224d848SSeongJae Park #define pr_fmt(fmt) "damon: " fmt 92224d848SSeongJae Park 102224d848SSeongJae Park #include <linux/damon.h> 112224d848SSeongJae Park #include <linux/delay.h> 122224d848SSeongJae Park #include <linux/kthread.h> 13ee801b7dSSeongJae Park #include <linux/mm.h> 142224d848SSeongJae Park #include <linux/slab.h> 1538683e00SSeongJae Park #include <linux/string.h> 162224d848SSeongJae Park 172fcb9362SSeongJae Park #define CREATE_TRACE_POINTS 182fcb9362SSeongJae Park #include <trace/events/damon.h> 192fcb9362SSeongJae Park 2017ccae8bSSeongJae Park #ifdef CONFIG_DAMON_KUNIT_TEST 2117ccae8bSSeongJae Park #undef DAMON_MIN_REGION 2217ccae8bSSeongJae Park #define DAMON_MIN_REGION 1 2317ccae8bSSeongJae Park #endif 2417ccae8bSSeongJae Park 252224d848SSeongJae Park static DEFINE_MUTEX(damon_lock); 262224d848SSeongJae Park static int nr_running_ctxs; 278b9b0d33SSeongJae Park static bool running_exclusive_ctxs; 282224d848SSeongJae Park 299f7b053aSSeongJae Park static DEFINE_MUTEX(damon_ops_lock); 309f7b053aSSeongJae Park static struct damon_operations damon_registered_ops[NR_DAMON_OPS]; 319f7b053aSSeongJae Park 32a1870944SDawei Li static struct kmem_cache *damon_region_cache __ro_after_init; 33a1870944SDawei Li 349f7b053aSSeongJae Park /* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */ 35152e5617SSeongJae Park static bool __damon_is_registered_ops(enum damon_ops_id id) 369f7b053aSSeongJae Park { 379f7b053aSSeongJae Park struct damon_operations empty_ops = {}; 389f7b053aSSeongJae Park 399f7b053aSSeongJae Park if (!memcmp(&empty_ops, &damon_registered_ops[id], sizeof(empty_ops))) 409f7b053aSSeongJae Park return false; 419f7b053aSSeongJae Park return true; 429f7b053aSSeongJae Park } 439f7b053aSSeongJae Park 449f7b053aSSeongJae Park /** 45152e5617SSeongJae Park * damon_is_registered_ops() - Check if a given damon_operations is registered. 46152e5617SSeongJae Park * @id: Id of the damon_operations to check if registered. 47152e5617SSeongJae Park * 48152e5617SSeongJae Park * Return: true if the ops is set, false otherwise. 49152e5617SSeongJae Park */ 50152e5617SSeongJae Park bool damon_is_registered_ops(enum damon_ops_id id) 51152e5617SSeongJae Park { 52152e5617SSeongJae Park bool registered; 53152e5617SSeongJae Park 54152e5617SSeongJae Park if (id >= NR_DAMON_OPS) 55152e5617SSeongJae Park return false; 56152e5617SSeongJae Park mutex_lock(&damon_ops_lock); 57152e5617SSeongJae Park registered = __damon_is_registered_ops(id); 58152e5617SSeongJae Park mutex_unlock(&damon_ops_lock); 59152e5617SSeongJae Park return registered; 60152e5617SSeongJae Park } 61152e5617SSeongJae Park 62152e5617SSeongJae Park /** 639f7b053aSSeongJae Park * damon_register_ops() - Register a monitoring operations set to DAMON. 649f7b053aSSeongJae Park * @ops: monitoring operations set to register. 659f7b053aSSeongJae Park * 669f7b053aSSeongJae Park * This function registers a monitoring operations set of valid &struct 679f7b053aSSeongJae Park * damon_operations->id so that others can find and use them later. 689f7b053aSSeongJae Park * 699f7b053aSSeongJae Park * Return: 0 on success, negative error code otherwise. 709f7b053aSSeongJae Park */ 719f7b053aSSeongJae Park int damon_register_ops(struct damon_operations *ops) 729f7b053aSSeongJae Park { 739f7b053aSSeongJae Park int err = 0; 749f7b053aSSeongJae Park 759f7b053aSSeongJae Park if (ops->id >= NR_DAMON_OPS) 769f7b053aSSeongJae Park return -EINVAL; 779f7b053aSSeongJae Park mutex_lock(&damon_ops_lock); 789f7b053aSSeongJae Park /* Fail for already registered ops */ 79152e5617SSeongJae Park if (__damon_is_registered_ops(ops->id)) { 809f7b053aSSeongJae Park err = -EINVAL; 819f7b053aSSeongJae Park goto out; 829f7b053aSSeongJae Park } 839f7b053aSSeongJae Park damon_registered_ops[ops->id] = *ops; 849f7b053aSSeongJae Park out: 859f7b053aSSeongJae Park mutex_unlock(&damon_ops_lock); 869f7b053aSSeongJae Park return err; 879f7b053aSSeongJae Park } 889f7b053aSSeongJae Park 899f7b053aSSeongJae Park /** 909f7b053aSSeongJae Park * damon_select_ops() - Select a monitoring operations to use with the context. 919f7b053aSSeongJae Park * @ctx: monitoring context to use the operations. 929f7b053aSSeongJae Park * @id: id of the registered monitoring operations to select. 939f7b053aSSeongJae Park * 949f7b053aSSeongJae Park * This function finds registered monitoring operations set of @id and make 959f7b053aSSeongJae Park * @ctx to use it. 969f7b053aSSeongJae Park * 979f7b053aSSeongJae Park * Return: 0 on success, negative error code otherwise. 989f7b053aSSeongJae Park */ 999f7b053aSSeongJae Park int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id) 1009f7b053aSSeongJae Park { 1019f7b053aSSeongJae Park int err = 0; 1029f7b053aSSeongJae Park 1039f7b053aSSeongJae Park if (id >= NR_DAMON_OPS) 1049f7b053aSSeongJae Park return -EINVAL; 1059f7b053aSSeongJae Park 1069f7b053aSSeongJae Park mutex_lock(&damon_ops_lock); 107152e5617SSeongJae Park if (!__damon_is_registered_ops(id)) 1089f7b053aSSeongJae Park err = -EINVAL; 1099f7b053aSSeongJae Park else 1109f7b053aSSeongJae Park ctx->ops = damon_registered_ops[id]; 1119f7b053aSSeongJae Park mutex_unlock(&damon_ops_lock); 1129f7b053aSSeongJae Park return err; 1139f7b053aSSeongJae Park } 1149f7b053aSSeongJae Park 115f23b8eeeSSeongJae Park /* 116f23b8eeeSSeongJae Park * Construct a damon_region struct 117f23b8eeeSSeongJae Park * 118f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 119f23b8eeeSSeongJae Park */ 120f23b8eeeSSeongJae Park struct damon_region *damon_new_region(unsigned long start, unsigned long end) 121f23b8eeeSSeongJae Park { 122f23b8eeeSSeongJae Park struct damon_region *region; 123f23b8eeeSSeongJae Park 124a1870944SDawei Li region = kmem_cache_alloc(damon_region_cache, GFP_KERNEL); 125f23b8eeeSSeongJae Park if (!region) 126f23b8eeeSSeongJae Park return NULL; 127f23b8eeeSSeongJae Park 128f23b8eeeSSeongJae Park region->ar.start = start; 129f23b8eeeSSeongJae Park region->ar.end = end; 130f23b8eeeSSeongJae Park region->nr_accesses = 0; 131f23b8eeeSSeongJae Park INIT_LIST_HEAD(®ion->list); 132f23b8eeeSSeongJae Park 133fda504faSSeongJae Park region->age = 0; 134fda504faSSeongJae Park region->last_nr_accesses = 0; 135fda504faSSeongJae Park 136f23b8eeeSSeongJae Park return region; 137f23b8eeeSSeongJae Park } 138f23b8eeeSSeongJae Park 139f23b8eeeSSeongJae Park void damon_add_region(struct damon_region *r, struct damon_target *t) 140f23b8eeeSSeongJae Park { 141f23b8eeeSSeongJae Park list_add_tail(&r->list, &t->regions_list); 142b9a6ac4eSSeongJae Park t->nr_regions++; 143f23b8eeeSSeongJae Park } 144f23b8eeeSSeongJae Park 145b9a6ac4eSSeongJae Park static void damon_del_region(struct damon_region *r, struct damon_target *t) 146f23b8eeeSSeongJae Park { 147f23b8eeeSSeongJae Park list_del(&r->list); 148b9a6ac4eSSeongJae Park t->nr_regions--; 149f23b8eeeSSeongJae Park } 150f23b8eeeSSeongJae Park 151f23b8eeeSSeongJae Park static void damon_free_region(struct damon_region *r) 152f23b8eeeSSeongJae Park { 153a1870944SDawei Li kmem_cache_free(damon_region_cache, r); 154f23b8eeeSSeongJae Park } 155f23b8eeeSSeongJae Park 156b9a6ac4eSSeongJae Park void damon_destroy_region(struct damon_region *r, struct damon_target *t) 157f23b8eeeSSeongJae Park { 158b9a6ac4eSSeongJae Park damon_del_region(r, t); 159f23b8eeeSSeongJae Park damon_free_region(r); 160f23b8eeeSSeongJae Park } 161f23b8eeeSSeongJae Park 162d0723bc0SSeongJae Park /* 163d0723bc0SSeongJae Park * Check whether a region is intersecting an address range 164d0723bc0SSeongJae Park * 165d0723bc0SSeongJae Park * Returns true if it is. 166d0723bc0SSeongJae Park */ 167d0723bc0SSeongJae Park static bool damon_intersect(struct damon_region *r, 168d0723bc0SSeongJae Park struct damon_addr_range *re) 169d0723bc0SSeongJae Park { 170d0723bc0SSeongJae Park return !(r->ar.end <= re->start || re->end <= r->ar.start); 171d0723bc0SSeongJae Park } 172d0723bc0SSeongJae Park 173d0723bc0SSeongJae Park /* 1749c950c22SSeongJae Park * Fill holes in regions with new regions. 1759c950c22SSeongJae Park */ 1769c950c22SSeongJae Park static int damon_fill_regions_holes(struct damon_region *first, 1779c950c22SSeongJae Park struct damon_region *last, struct damon_target *t) 1789c950c22SSeongJae Park { 1799c950c22SSeongJae Park struct damon_region *r = first; 1809c950c22SSeongJae Park 1819c950c22SSeongJae Park damon_for_each_region_from(r, t) { 1829c950c22SSeongJae Park struct damon_region *next, *newr; 1839c950c22SSeongJae Park 1849c950c22SSeongJae Park if (r == last) 1859c950c22SSeongJae Park break; 1869c950c22SSeongJae Park next = damon_next_region(r); 1879c950c22SSeongJae Park if (r->ar.end != next->ar.start) { 1889c950c22SSeongJae Park newr = damon_new_region(r->ar.end, next->ar.start); 1899c950c22SSeongJae Park if (!newr) 1909c950c22SSeongJae Park return -ENOMEM; 1919c950c22SSeongJae Park damon_insert_region(newr, r, next, t); 1929c950c22SSeongJae Park } 1939c950c22SSeongJae Park } 1949c950c22SSeongJae Park return 0; 1959c950c22SSeongJae Park } 1969c950c22SSeongJae Park 1979c950c22SSeongJae Park /* 198d0723bc0SSeongJae Park * damon_set_regions() - Set regions of a target for given address ranges. 199d0723bc0SSeongJae Park * @t: the given target. 200d0723bc0SSeongJae Park * @ranges: array of new monitoring target ranges. 201d0723bc0SSeongJae Park * @nr_ranges: length of @ranges. 202d0723bc0SSeongJae Park * 203d0723bc0SSeongJae Park * This function adds new regions to, or modify existing regions of a 204d0723bc0SSeongJae Park * monitoring target to fit in specific ranges. 205d0723bc0SSeongJae Park * 206d0723bc0SSeongJae Park * Return: 0 if success, or negative error code otherwise. 207d0723bc0SSeongJae Park */ 208d0723bc0SSeongJae Park int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, 209d0723bc0SSeongJae Park unsigned int nr_ranges) 210d0723bc0SSeongJae Park { 211d0723bc0SSeongJae Park struct damon_region *r, *next; 212d0723bc0SSeongJae Park unsigned int i; 2139c950c22SSeongJae Park int err; 214d0723bc0SSeongJae Park 215d0723bc0SSeongJae Park /* Remove regions which are not in the new ranges */ 216d0723bc0SSeongJae Park damon_for_each_region_safe(r, next, t) { 217d0723bc0SSeongJae Park for (i = 0; i < nr_ranges; i++) { 218d0723bc0SSeongJae Park if (damon_intersect(r, &ranges[i])) 219d0723bc0SSeongJae Park break; 220d0723bc0SSeongJae Park } 221d0723bc0SSeongJae Park if (i == nr_ranges) 222d0723bc0SSeongJae Park damon_destroy_region(r, t); 223d0723bc0SSeongJae Park } 224d0723bc0SSeongJae Park 22536001cbaSKaixu Xia r = damon_first_region(t); 226d0723bc0SSeongJae Park /* Add new regions or resize existing regions to fit in the ranges */ 227d0723bc0SSeongJae Park for (i = 0; i < nr_ranges; i++) { 228d0723bc0SSeongJae Park struct damon_region *first = NULL, *last, *newr; 229d0723bc0SSeongJae Park struct damon_addr_range *range; 230d0723bc0SSeongJae Park 231d0723bc0SSeongJae Park range = &ranges[i]; 232d0723bc0SSeongJae Park /* Get the first/last regions intersecting with the range */ 23336001cbaSKaixu Xia damon_for_each_region_from(r, t) { 234d0723bc0SSeongJae Park if (damon_intersect(r, range)) { 235d0723bc0SSeongJae Park if (!first) 236d0723bc0SSeongJae Park first = r; 237d0723bc0SSeongJae Park last = r; 238d0723bc0SSeongJae Park } 239d0723bc0SSeongJae Park if (r->ar.start >= range->end) 240d0723bc0SSeongJae Park break; 241d0723bc0SSeongJae Park } 242d0723bc0SSeongJae Park if (!first) { 243d0723bc0SSeongJae Park /* no region intersects with this range */ 244d0723bc0SSeongJae Park newr = damon_new_region( 245d0723bc0SSeongJae Park ALIGN_DOWN(range->start, 246d0723bc0SSeongJae Park DAMON_MIN_REGION), 247d0723bc0SSeongJae Park ALIGN(range->end, DAMON_MIN_REGION)); 248d0723bc0SSeongJae Park if (!newr) 249d0723bc0SSeongJae Park return -ENOMEM; 250d0723bc0SSeongJae Park damon_insert_region(newr, damon_prev_region(r), r, t); 251d0723bc0SSeongJae Park } else { 252d0723bc0SSeongJae Park /* resize intersecting regions to fit in this range */ 253d0723bc0SSeongJae Park first->ar.start = ALIGN_DOWN(range->start, 254d0723bc0SSeongJae Park DAMON_MIN_REGION); 255d0723bc0SSeongJae Park last->ar.end = ALIGN(range->end, DAMON_MIN_REGION); 2569c950c22SSeongJae Park 2579c950c22SSeongJae Park /* fill possible holes in the range */ 2589c950c22SSeongJae Park err = damon_fill_regions_holes(first, last, t); 2599c950c22SSeongJae Park if (err) 2609c950c22SSeongJae Park return err; 261d0723bc0SSeongJae Park } 262d0723bc0SSeongJae Park } 263d0723bc0SSeongJae Park return 0; 264d0723bc0SSeongJae Park } 265d0723bc0SSeongJae Park 26698def236SSeongJae Park struct damos_filter *damos_new_filter(enum damos_filter_type type, 26798def236SSeongJae Park bool matching) 26898def236SSeongJae Park { 26998def236SSeongJae Park struct damos_filter *filter; 27098def236SSeongJae Park 27198def236SSeongJae Park filter = kmalloc(sizeof(*filter), GFP_KERNEL); 27298def236SSeongJae Park if (!filter) 27398def236SSeongJae Park return NULL; 27498def236SSeongJae Park filter->type = type; 27598def236SSeongJae Park filter->matching = matching; 2765f1fc67fSSeongJae Park INIT_LIST_HEAD(&filter->list); 27798def236SSeongJae Park return filter; 27898def236SSeongJae Park } 27998def236SSeongJae Park 28098def236SSeongJae Park void damos_add_filter(struct damos *s, struct damos_filter *f) 28198def236SSeongJae Park { 28298def236SSeongJae Park list_add_tail(&f->list, &s->filters); 28398def236SSeongJae Park } 28498def236SSeongJae Park 28598def236SSeongJae Park static void damos_del_filter(struct damos_filter *f) 28698def236SSeongJae Park { 28798def236SSeongJae Park list_del(&f->list); 28898def236SSeongJae Park } 28998def236SSeongJae Park 29098def236SSeongJae Park static void damos_free_filter(struct damos_filter *f) 29198def236SSeongJae Park { 29298def236SSeongJae Park kfree(f); 29398def236SSeongJae Park } 29498def236SSeongJae Park 29598def236SSeongJae Park void damos_destroy_filter(struct damos_filter *f) 29698def236SSeongJae Park { 29798def236SSeongJae Park damos_del_filter(f); 29898def236SSeongJae Park damos_free_filter(f); 29998def236SSeongJae Park } 30098def236SSeongJae Park 30170e0c1d1SSeongJae Park /* initialize private fields of damos_quota and return the pointer */ 30270e0c1d1SSeongJae Park static struct damos_quota *damos_quota_init_priv(struct damos_quota *quota) 30370e0c1d1SSeongJae Park { 30470e0c1d1SSeongJae Park quota->total_charged_sz = 0; 30570e0c1d1SSeongJae Park quota->total_charged_ns = 0; 30670e0c1d1SSeongJae Park quota->esz = 0; 30770e0c1d1SSeongJae Park quota->charged_sz = 0; 30870e0c1d1SSeongJae Park quota->charged_from = 0; 30970e0c1d1SSeongJae Park quota->charge_target_from = NULL; 31070e0c1d1SSeongJae Park quota->charge_addr_from = 0; 31170e0c1d1SSeongJae Park return quota; 31270e0c1d1SSeongJae Park } 31370e0c1d1SSeongJae Park 314f5a79d7cSYajun Deng struct damos *damon_new_scheme(struct damos_access_pattern *pattern, 315ee801b7dSSeongJae Park enum damos_action action, struct damos_quota *quota, 316ee801b7dSSeongJae Park struct damos_watermarks *wmarks) 3171f366e42SSeongJae Park { 3181f366e42SSeongJae Park struct damos *scheme; 3191f366e42SSeongJae Park 3201f366e42SSeongJae Park scheme = kmalloc(sizeof(*scheme), GFP_KERNEL); 3211f366e42SSeongJae Park if (!scheme) 3221f366e42SSeongJae Park return NULL; 32302f17037SSeongJae Park scheme->pattern = *pattern; 3241f366e42SSeongJae Park scheme->action = action; 32598def236SSeongJae Park INIT_LIST_HEAD(&scheme->filters); 3260e92c2eeSSeongJae Park scheme->stat = (struct damos_stat){}; 3271f366e42SSeongJae Park INIT_LIST_HEAD(&scheme->list); 3281f366e42SSeongJae Park 32970e0c1d1SSeongJae Park scheme->quota = *(damos_quota_init_priv(quota)); 3302b8a248dSSeongJae Park 33102f17037SSeongJae Park scheme->wmarks = *wmarks; 332ee801b7dSSeongJae Park scheme->wmarks.activated = true; 333ee801b7dSSeongJae Park 3341f366e42SSeongJae Park return scheme; 3351f366e42SSeongJae Park } 3361f366e42SSeongJae Park 3371f366e42SSeongJae Park void damon_add_scheme(struct damon_ctx *ctx, struct damos *s) 3381f366e42SSeongJae Park { 3391f366e42SSeongJae Park list_add_tail(&s->list, &ctx->schemes); 3401f366e42SSeongJae Park } 3411f366e42SSeongJae Park 3421f366e42SSeongJae Park static void damon_del_scheme(struct damos *s) 3431f366e42SSeongJae Park { 3441f366e42SSeongJae Park list_del(&s->list); 3451f366e42SSeongJae Park } 3461f366e42SSeongJae Park 3471f366e42SSeongJae Park static void damon_free_scheme(struct damos *s) 3481f366e42SSeongJae Park { 3491f366e42SSeongJae Park kfree(s); 3501f366e42SSeongJae Park } 3511f366e42SSeongJae Park 3521f366e42SSeongJae Park void damon_destroy_scheme(struct damos *s) 3531f366e42SSeongJae Park { 35498def236SSeongJae Park struct damos_filter *f, *next; 35598def236SSeongJae Park 35698def236SSeongJae Park damos_for_each_filter_safe(f, next, s) 35798def236SSeongJae Park damos_destroy_filter(f); 3581f366e42SSeongJae Park damon_del_scheme(s); 3591f366e42SSeongJae Park damon_free_scheme(s); 3601f366e42SSeongJae Park } 3611f366e42SSeongJae Park 362f23b8eeeSSeongJae Park /* 363f23b8eeeSSeongJae Park * Construct a damon_target struct 364f23b8eeeSSeongJae Park * 365f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 366f23b8eeeSSeongJae Park */ 3671971bd63SSeongJae Park struct damon_target *damon_new_target(void) 368f23b8eeeSSeongJae Park { 369f23b8eeeSSeongJae Park struct damon_target *t; 370f23b8eeeSSeongJae Park 371f23b8eeeSSeongJae Park t = kmalloc(sizeof(*t), GFP_KERNEL); 372f23b8eeeSSeongJae Park if (!t) 373f23b8eeeSSeongJae Park return NULL; 374f23b8eeeSSeongJae Park 3751971bd63SSeongJae Park t->pid = NULL; 376b9a6ac4eSSeongJae Park t->nr_regions = 0; 377f23b8eeeSSeongJae Park INIT_LIST_HEAD(&t->regions_list); 378b1f44cdaSSeongJae Park INIT_LIST_HEAD(&t->list); 379f23b8eeeSSeongJae Park 380f23b8eeeSSeongJae Park return t; 381f23b8eeeSSeongJae Park } 382f23b8eeeSSeongJae Park 383f23b8eeeSSeongJae Park void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) 384f23b8eeeSSeongJae Park { 385b9a6ac4eSSeongJae Park list_add_tail(&t->list, &ctx->adaptive_targets); 386f23b8eeeSSeongJae Park } 387f23b8eeeSSeongJae Park 388b5ca3e83SXin Hao bool damon_targets_empty(struct damon_ctx *ctx) 389b5ca3e83SXin Hao { 390b5ca3e83SXin Hao return list_empty(&ctx->adaptive_targets); 391b5ca3e83SXin Hao } 392b5ca3e83SXin Hao 393f23b8eeeSSeongJae Park static void damon_del_target(struct damon_target *t) 394f23b8eeeSSeongJae Park { 395f23b8eeeSSeongJae Park list_del(&t->list); 396f23b8eeeSSeongJae Park } 397f23b8eeeSSeongJae Park 398f23b8eeeSSeongJae Park void damon_free_target(struct damon_target *t) 399f23b8eeeSSeongJae Park { 400f23b8eeeSSeongJae Park struct damon_region *r, *next; 401f23b8eeeSSeongJae Park 402f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 403f23b8eeeSSeongJae Park damon_free_region(r); 404f23b8eeeSSeongJae Park kfree(t); 405f23b8eeeSSeongJae Park } 406f23b8eeeSSeongJae Park 407f23b8eeeSSeongJae Park void damon_destroy_target(struct damon_target *t) 408f23b8eeeSSeongJae Park { 409f23b8eeeSSeongJae Park damon_del_target(t); 410f23b8eeeSSeongJae Park damon_free_target(t); 411f23b8eeeSSeongJae Park } 412f23b8eeeSSeongJae Park 413b9a6ac4eSSeongJae Park unsigned int damon_nr_regions(struct damon_target *t) 414b9a6ac4eSSeongJae Park { 415b9a6ac4eSSeongJae Park return t->nr_regions; 416b9a6ac4eSSeongJae Park } 417b9a6ac4eSSeongJae Park 4182224d848SSeongJae Park struct damon_ctx *damon_new_ctx(void) 4192224d848SSeongJae Park { 4202224d848SSeongJae Park struct damon_ctx *ctx; 4212224d848SSeongJae Park 4222224d848SSeongJae Park ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 4232224d848SSeongJae Park if (!ctx) 4242224d848SSeongJae Park return NULL; 4252224d848SSeongJae Park 426e93bcaebSSeongJae Park init_completion(&ctx->kdamond_started); 427e93bcaebSSeongJae Park 428cbeaa77bSSeongJae Park ctx->attrs.sample_interval = 5 * 1000; 429cbeaa77bSSeongJae Park ctx->attrs.aggr_interval = 100 * 1000; 430cbeaa77bSSeongJae Park ctx->attrs.ops_update_interval = 60 * 1000 * 1000; 4312224d848SSeongJae Park 432c708a5e5SSeongJae Park ctx->passed_sample_intervals = 0; 433c708a5e5SSeongJae Park /* These will be set from kdamond_init_intervals_sis() */ 434c708a5e5SSeongJae Park ctx->next_aggregation_sis = 0; 435c708a5e5SSeongJae Park ctx->next_ops_update_sis = 0; 4362224d848SSeongJae Park 4372224d848SSeongJae Park mutex_init(&ctx->kdamond_lock); 4382224d848SSeongJae Park 439cbeaa77bSSeongJae Park ctx->attrs.min_nr_regions = 10; 440cbeaa77bSSeongJae Park ctx->attrs.max_nr_regions = 1000; 441b9a6ac4eSSeongJae Park 442b9a6ac4eSSeongJae Park INIT_LIST_HEAD(&ctx->adaptive_targets); 4431f366e42SSeongJae Park INIT_LIST_HEAD(&ctx->schemes); 4442224d848SSeongJae Park 4452224d848SSeongJae Park return ctx; 4462224d848SSeongJae Park } 4472224d848SSeongJae Park 448f23b8eeeSSeongJae Park static void damon_destroy_targets(struct damon_ctx *ctx) 449f23b8eeeSSeongJae Park { 450f23b8eeeSSeongJae Park struct damon_target *t, *next_t; 451f23b8eeeSSeongJae Park 452f7d911c3SSeongJae Park if (ctx->ops.cleanup) { 453f7d911c3SSeongJae Park ctx->ops.cleanup(ctx); 454f23b8eeeSSeongJae Park return; 455f23b8eeeSSeongJae Park } 456f23b8eeeSSeongJae Park 457f23b8eeeSSeongJae Park damon_for_each_target_safe(t, next_t, ctx) 458f23b8eeeSSeongJae Park damon_destroy_target(t); 459f23b8eeeSSeongJae Park } 460f23b8eeeSSeongJae Park 4612224d848SSeongJae Park void damon_destroy_ctx(struct damon_ctx *ctx) 4622224d848SSeongJae Park { 4631f366e42SSeongJae Park struct damos *s, *next_s; 4641f366e42SSeongJae Park 465f23b8eeeSSeongJae Park damon_destroy_targets(ctx); 4661f366e42SSeongJae Park 4671f366e42SSeongJae Park damon_for_each_scheme_safe(s, next_s, ctx) 4681f366e42SSeongJae Park damon_destroy_scheme(s); 4691f366e42SSeongJae Park 4702224d848SSeongJae Park kfree(ctx); 4712224d848SSeongJae Park } 4722224d848SSeongJae Park 4732f5bef5aSSeongJae Park static unsigned int damon_age_for_new_attrs(unsigned int age, 4742f5bef5aSSeongJae Park struct damon_attrs *old_attrs, struct damon_attrs *new_attrs) 4752f5bef5aSSeongJae Park { 4762f5bef5aSSeongJae Park return age * old_attrs->aggr_interval / new_attrs->aggr_interval; 4772f5bef5aSSeongJae Park } 4782f5bef5aSSeongJae Park 4792f5bef5aSSeongJae Park /* convert access ratio in bp (per 10,000) to nr_accesses */ 4802f5bef5aSSeongJae Park static unsigned int damon_accesses_bp_to_nr_accesses( 4812f5bef5aSSeongJae Park unsigned int accesses_bp, struct damon_attrs *attrs) 4822f5bef5aSSeongJae Park { 48374ed1097SSeongJae Park return accesses_bp * damon_max_nr_accesses(attrs) / 10000; 4842f5bef5aSSeongJae Park } 4852f5bef5aSSeongJae Park 4862f5bef5aSSeongJae Park /* convert nr_accesses to access ratio in bp (per 10,000) */ 4872f5bef5aSSeongJae Park static unsigned int damon_nr_accesses_to_accesses_bp( 4882f5bef5aSSeongJae Park unsigned int nr_accesses, struct damon_attrs *attrs) 4892f5bef5aSSeongJae Park { 49074ed1097SSeongJae Park return nr_accesses * 10000 / damon_max_nr_accesses(attrs); 4912f5bef5aSSeongJae Park } 4922f5bef5aSSeongJae Park 4932f5bef5aSSeongJae Park static unsigned int damon_nr_accesses_for_new_attrs(unsigned int nr_accesses, 4942f5bef5aSSeongJae Park struct damon_attrs *old_attrs, struct damon_attrs *new_attrs) 4952f5bef5aSSeongJae Park { 4962f5bef5aSSeongJae Park return damon_accesses_bp_to_nr_accesses( 4972f5bef5aSSeongJae Park damon_nr_accesses_to_accesses_bp( 4982f5bef5aSSeongJae Park nr_accesses, old_attrs), 4992f5bef5aSSeongJae Park new_attrs); 5002f5bef5aSSeongJae Park } 5012f5bef5aSSeongJae Park 5022f5bef5aSSeongJae Park static void damon_update_monitoring_result(struct damon_region *r, 5032f5bef5aSSeongJae Park struct damon_attrs *old_attrs, struct damon_attrs *new_attrs) 5042f5bef5aSSeongJae Park { 5052f5bef5aSSeongJae Park r->nr_accesses = damon_nr_accesses_for_new_attrs(r->nr_accesses, 5062f5bef5aSSeongJae Park old_attrs, new_attrs); 5072f5bef5aSSeongJae Park r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs); 5082f5bef5aSSeongJae Park } 5092f5bef5aSSeongJae Park 5102f5bef5aSSeongJae Park /* 5112f5bef5aSSeongJae Park * region->nr_accesses is the number of sampling intervals in the last 5122f5bef5aSSeongJae Park * aggregation interval that access to the region has found, and region->age is 5132f5bef5aSSeongJae Park * the number of aggregation intervals that its access pattern has maintained. 5142f5bef5aSSeongJae Park * For the reason, the real meaning of the two fields depend on current 5152f5bef5aSSeongJae Park * sampling interval and aggregation interval. This function updates 5162f5bef5aSSeongJae Park * ->nr_accesses and ->age of given damon_ctx's regions for new damon_attrs. 5172f5bef5aSSeongJae Park */ 5182f5bef5aSSeongJae Park static void damon_update_monitoring_results(struct damon_ctx *ctx, 5192f5bef5aSSeongJae Park struct damon_attrs *new_attrs) 5202f5bef5aSSeongJae Park { 5212f5bef5aSSeongJae Park struct damon_attrs *old_attrs = &ctx->attrs; 5222f5bef5aSSeongJae Park struct damon_target *t; 5232f5bef5aSSeongJae Park struct damon_region *r; 5242f5bef5aSSeongJae Park 5252f5bef5aSSeongJae Park /* if any interval is zero, simply forgive conversion */ 5262f5bef5aSSeongJae Park if (!old_attrs->sample_interval || !old_attrs->aggr_interval || 5272f5bef5aSSeongJae Park !new_attrs->sample_interval || 5282f5bef5aSSeongJae Park !new_attrs->aggr_interval) 5292f5bef5aSSeongJae Park return; 5302f5bef5aSSeongJae Park 5312f5bef5aSSeongJae Park damon_for_each_target(t, ctx) 5322f5bef5aSSeongJae Park damon_for_each_region(r, t) 5332f5bef5aSSeongJae Park damon_update_monitoring_result( 5342f5bef5aSSeongJae Park r, old_attrs, new_attrs); 5352f5bef5aSSeongJae Park } 5362f5bef5aSSeongJae Park 5372224d848SSeongJae Park /** 5382224d848SSeongJae Park * damon_set_attrs() - Set attributes for the monitoring. 5392224d848SSeongJae Park * @ctx: monitoring context 540bead3b00SSeongJae Park * @attrs: monitoring attributes 5412224d848SSeongJae Park * 5422224d848SSeongJae Park * This function should not be called while the kdamond is running. 5432224d848SSeongJae Park * Every time interval is in micro-seconds. 5442224d848SSeongJae Park * 5452224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 5462224d848SSeongJae Park */ 547bead3b00SSeongJae Park int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs) 5482224d848SSeongJae Park { 549c708a5e5SSeongJae Park unsigned long sample_interval = attrs->sample_interval ? 550c708a5e5SSeongJae Park attrs->sample_interval : 1; 551c708a5e5SSeongJae Park 552bead3b00SSeongJae Park if (attrs->min_nr_regions < 3) 553b9a6ac4eSSeongJae Park return -EINVAL; 554bead3b00SSeongJae Park if (attrs->min_nr_regions > attrs->max_nr_regions) 555b9a6ac4eSSeongJae Park return -EINVAL; 5565ff6e2ffSKefeng Wang if (attrs->sample_interval > attrs->aggr_interval) 5575ff6e2ffSKefeng Wang return -EINVAL; 558b9a6ac4eSSeongJae Park 559c708a5e5SSeongJae Park ctx->next_aggregation_sis = ctx->passed_sample_intervals + 560c708a5e5SSeongJae Park attrs->aggr_interval / sample_interval; 561c708a5e5SSeongJae Park ctx->next_ops_update_sis = ctx->passed_sample_intervals + 562c708a5e5SSeongJae Park attrs->ops_update_interval / sample_interval; 563c708a5e5SSeongJae Park 5642f5bef5aSSeongJae Park damon_update_monitoring_results(ctx, attrs); 565bead3b00SSeongJae Park ctx->attrs = *attrs; 5662224d848SSeongJae Park return 0; 5672224d848SSeongJae Park } 5682224d848SSeongJae Park 5694bc05954SSeongJae Park /** 5701f366e42SSeongJae Park * damon_set_schemes() - Set data access monitoring based operation schemes. 5711f366e42SSeongJae Park * @ctx: monitoring context 5721f366e42SSeongJae Park * @schemes: array of the schemes 5731f366e42SSeongJae Park * @nr_schemes: number of entries in @schemes 5741f366e42SSeongJae Park * 5751f366e42SSeongJae Park * This function should not be called while the kdamond of the context is 5761f366e42SSeongJae Park * running. 5771f366e42SSeongJae Park */ 578cc713520SKaixu Xia void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, 5791f366e42SSeongJae Park ssize_t nr_schemes) 5801f366e42SSeongJae Park { 5811f366e42SSeongJae Park struct damos *s, *next; 5821f366e42SSeongJae Park ssize_t i; 5831f366e42SSeongJae Park 5841f366e42SSeongJae Park damon_for_each_scheme_safe(s, next, ctx) 5851f366e42SSeongJae Park damon_destroy_scheme(s); 5861f366e42SSeongJae Park for (i = 0; i < nr_schemes; i++) 5871f366e42SSeongJae Park damon_add_scheme(ctx, schemes[i]); 5881f366e42SSeongJae Park } 5891f366e42SSeongJae Park 5901f366e42SSeongJae Park /** 5914bc05954SSeongJae Park * damon_nr_running_ctxs() - Return number of currently running contexts. 5924bc05954SSeongJae Park */ 5934bc05954SSeongJae Park int damon_nr_running_ctxs(void) 5944bc05954SSeongJae Park { 5954bc05954SSeongJae Park int nr_ctxs; 5964bc05954SSeongJae Park 5974bc05954SSeongJae Park mutex_lock(&damon_lock); 5984bc05954SSeongJae Park nr_ctxs = nr_running_ctxs; 5994bc05954SSeongJae Park mutex_unlock(&damon_lock); 6004bc05954SSeongJae Park 6014bc05954SSeongJae Park return nr_ctxs; 6024bc05954SSeongJae Park } 6034bc05954SSeongJae Park 604b9a6ac4eSSeongJae Park /* Returns the size upper limit for each monitoring region */ 605b9a6ac4eSSeongJae Park static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) 606b9a6ac4eSSeongJae Park { 607b9a6ac4eSSeongJae Park struct damon_target *t; 608b9a6ac4eSSeongJae Park struct damon_region *r; 609b9a6ac4eSSeongJae Park unsigned long sz = 0; 610b9a6ac4eSSeongJae Park 611b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) { 612b9a6ac4eSSeongJae Park damon_for_each_region(r, t) 613ab63f63fSXin Hao sz += damon_sz_region(r); 614b9a6ac4eSSeongJae Park } 615b9a6ac4eSSeongJae Park 616cbeaa77bSSeongJae Park if (ctx->attrs.min_nr_regions) 617cbeaa77bSSeongJae Park sz /= ctx->attrs.min_nr_regions; 618b9a6ac4eSSeongJae Park if (sz < DAMON_MIN_REGION) 619b9a6ac4eSSeongJae Park sz = DAMON_MIN_REGION; 620b9a6ac4eSSeongJae Park 621b9a6ac4eSSeongJae Park return sz; 622b9a6ac4eSSeongJae Park } 623b9a6ac4eSSeongJae Park 6242224d848SSeongJae Park static int kdamond_fn(void *data); 6252224d848SSeongJae Park 6262224d848SSeongJae Park /* 6272224d848SSeongJae Park * __damon_start() - Starts monitoring with given context. 6282224d848SSeongJae Park * @ctx: monitoring context 6292224d848SSeongJae Park * 6302224d848SSeongJae Park * This function should be called while damon_lock is hold. 6312224d848SSeongJae Park * 6322224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 6332224d848SSeongJae Park */ 6342224d848SSeongJae Park static int __damon_start(struct damon_ctx *ctx) 6352224d848SSeongJae Park { 6362224d848SSeongJae Park int err = -EBUSY; 6372224d848SSeongJae Park 6382224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 6392224d848SSeongJae Park if (!ctx->kdamond) { 6402224d848SSeongJae Park err = 0; 641e93bcaebSSeongJae Park reinit_completion(&ctx->kdamond_started); 6422224d848SSeongJae Park ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d", 6432224d848SSeongJae Park nr_running_ctxs); 6442224d848SSeongJae Park if (IS_ERR(ctx->kdamond)) { 6452224d848SSeongJae Park err = PTR_ERR(ctx->kdamond); 6467ec1992bSColin Ian King ctx->kdamond = NULL; 647e93bcaebSSeongJae Park } else { 648e93bcaebSSeongJae Park wait_for_completion(&ctx->kdamond_started); 6492224d848SSeongJae Park } 6502224d848SSeongJae Park } 6512224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 6522224d848SSeongJae Park 6532224d848SSeongJae Park return err; 6542224d848SSeongJae Park } 6552224d848SSeongJae Park 6562224d848SSeongJae Park /** 6572224d848SSeongJae Park * damon_start() - Starts the monitorings for a given group of contexts. 6582224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to start monitoring 6592224d848SSeongJae Park * @nr_ctxs: size of @ctxs 6608b9b0d33SSeongJae Park * @exclusive: exclusiveness of this contexts group 6612224d848SSeongJae Park * 6622224d848SSeongJae Park * This function starts a group of monitoring threads for a group of monitoring 6632224d848SSeongJae Park * contexts. One thread per each context is created and run in parallel. The 6648b9b0d33SSeongJae Park * caller should handle synchronization between the threads by itself. If 6658b9b0d33SSeongJae Park * @exclusive is true and a group of threads that created by other 6668b9b0d33SSeongJae Park * 'damon_start()' call is currently running, this function does nothing but 6678b9b0d33SSeongJae Park * returns -EBUSY. 6682224d848SSeongJae Park * 6692224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 6702224d848SSeongJae Park */ 6718b9b0d33SSeongJae Park int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) 6722224d848SSeongJae Park { 6732224d848SSeongJae Park int i; 6742224d848SSeongJae Park int err = 0; 6752224d848SSeongJae Park 6762224d848SSeongJae Park mutex_lock(&damon_lock); 6778b9b0d33SSeongJae Park if ((exclusive && nr_running_ctxs) || 6788b9b0d33SSeongJae Park (!exclusive && running_exclusive_ctxs)) { 6792224d848SSeongJae Park mutex_unlock(&damon_lock); 6802224d848SSeongJae Park return -EBUSY; 6812224d848SSeongJae Park } 6822224d848SSeongJae Park 6832224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 6842224d848SSeongJae Park err = __damon_start(ctxs[i]); 6852224d848SSeongJae Park if (err) 6862224d848SSeongJae Park break; 6872224d848SSeongJae Park nr_running_ctxs++; 6882224d848SSeongJae Park } 6898b9b0d33SSeongJae Park if (exclusive && nr_running_ctxs) 6908b9b0d33SSeongJae Park running_exclusive_ctxs = true; 6912224d848SSeongJae Park mutex_unlock(&damon_lock); 6922224d848SSeongJae Park 6932224d848SSeongJae Park return err; 6942224d848SSeongJae Park } 6952224d848SSeongJae Park 6962224d848SSeongJae Park /* 6978b9b0d33SSeongJae Park * __damon_stop() - Stops monitoring of a given context. 6982224d848SSeongJae Park * @ctx: monitoring context 6992224d848SSeongJae Park * 7002224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 7012224d848SSeongJae Park */ 7022224d848SSeongJae Park static int __damon_stop(struct damon_ctx *ctx) 7032224d848SSeongJae Park { 7040f91d133SChangbin Du struct task_struct *tsk; 7050f91d133SChangbin Du 7062224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 7070f91d133SChangbin Du tsk = ctx->kdamond; 7080f91d133SChangbin Du if (tsk) { 7090f91d133SChangbin Du get_task_struct(tsk); 7102224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 711*a9da6ddaSAndreas Gruenbacher kthread_stop_put(tsk); 7122224d848SSeongJae Park return 0; 7132224d848SSeongJae Park } 7142224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 7152224d848SSeongJae Park 7162224d848SSeongJae Park return -EPERM; 7172224d848SSeongJae Park } 7182224d848SSeongJae Park 7192224d848SSeongJae Park /** 7202224d848SSeongJae Park * damon_stop() - Stops the monitorings for a given group of contexts. 7212224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to stop monitoring 7222224d848SSeongJae Park * @nr_ctxs: size of @ctxs 7232224d848SSeongJae Park * 7242224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 7252224d848SSeongJae Park */ 7262224d848SSeongJae Park int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) 7272224d848SSeongJae Park { 7282224d848SSeongJae Park int i, err = 0; 7292224d848SSeongJae Park 7302224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 7312224d848SSeongJae Park /* nr_running_ctxs is decremented in kdamond_fn */ 7322224d848SSeongJae Park err = __damon_stop(ctxs[i]); 7332224d848SSeongJae Park if (err) 7348b9b0d33SSeongJae Park break; 7352224d848SSeongJae Park } 7362224d848SSeongJae Park return err; 7372224d848SSeongJae Park } 7382224d848SSeongJae Park 7392224d848SSeongJae Park /* 740f23b8eeeSSeongJae Park * Reset the aggregated monitoring results ('nr_accesses' of each region). 741f23b8eeeSSeongJae Park */ 742f23b8eeeSSeongJae Park static void kdamond_reset_aggregated(struct damon_ctx *c) 743f23b8eeeSSeongJae Park { 744f23b8eeeSSeongJae Park struct damon_target *t; 74576fd0285SSeongJae Park unsigned int ti = 0; /* target's index */ 746f23b8eeeSSeongJae Park 747f23b8eeeSSeongJae Park damon_for_each_target(t, c) { 748f23b8eeeSSeongJae Park struct damon_region *r; 749f23b8eeeSSeongJae Park 7502fcb9362SSeongJae Park damon_for_each_region(r, t) { 75176fd0285SSeongJae Park trace_damon_aggregated(t, ti, r, damon_nr_regions(t)); 752fda504faSSeongJae Park r->last_nr_accesses = r->nr_accesses; 753f23b8eeeSSeongJae Park r->nr_accesses = 0; 754f23b8eeeSSeongJae Park } 75576fd0285SSeongJae Park ti++; 756f23b8eeeSSeongJae Park } 7572fcb9362SSeongJae Park } 758f23b8eeeSSeongJae Park 7594ed98243SKaixu Xia static void damon_split_region_at(struct damon_target *t, 7604ed98243SKaixu Xia struct damon_region *r, unsigned long sz_r); 7612b8a248dSSeongJae Park 76238683e00SSeongJae Park static bool __damos_valid_target(struct damon_region *r, struct damos *s) 76338683e00SSeongJae Park { 76438683e00SSeongJae Park unsigned long sz; 76538683e00SSeongJae Park 766ab63f63fSXin Hao sz = damon_sz_region(r); 767f5a79d7cSYajun Deng return s->pattern.min_sz_region <= sz && 768f5a79d7cSYajun Deng sz <= s->pattern.max_sz_region && 769f5a79d7cSYajun Deng s->pattern.min_nr_accesses <= r->nr_accesses && 770f5a79d7cSYajun Deng r->nr_accesses <= s->pattern.max_nr_accesses && 771f5a79d7cSYajun Deng s->pattern.min_age_region <= r->age && 772f5a79d7cSYajun Deng r->age <= s->pattern.max_age_region; 77338683e00SSeongJae Park } 77438683e00SSeongJae Park 77538683e00SSeongJae Park static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, 77638683e00SSeongJae Park struct damon_region *r, struct damos *s) 77738683e00SSeongJae Park { 77838683e00SSeongJae Park bool ret = __damos_valid_target(r, s); 77938683e00SSeongJae Park 780f7d911c3SSeongJae Park if (!ret || !s->quota.esz || !c->ops.get_scheme_score) 78138683e00SSeongJae Park return ret; 78238683e00SSeongJae Park 783f7d911c3SSeongJae Park return c->ops.get_scheme_score(c, t, r, s) >= s->quota.min_score; 78438683e00SSeongJae Park } 78538683e00SSeongJae Park 7862ea34989SSeongJae Park /* 7872ea34989SSeongJae Park * damos_skip_charged_region() - Check if the given region or starting part of 7882ea34989SSeongJae Park * it is already charged for the DAMOS quota. 7892ea34989SSeongJae Park * @t: The target of the region. 7902ea34989SSeongJae Park * @rp: The pointer to the region. 7912ea34989SSeongJae Park * @s: The scheme to be applied. 7922ea34989SSeongJae Park * 7932ea34989SSeongJae Park * If a quota of a scheme has exceeded in a quota charge window, the scheme's 7942ea34989SSeongJae Park * action would applied to only a part of the target access pattern fulfilling 7952ea34989SSeongJae Park * regions. To avoid applying the scheme action to only already applied 7962ea34989SSeongJae Park * regions, DAMON skips applying the scheme action to the regions that charged 7972ea34989SSeongJae Park * in the previous charge window. 7982ea34989SSeongJae Park * 7992ea34989SSeongJae Park * This function checks if a given region should be skipped or not for the 8002ea34989SSeongJae Park * reason. If only the starting part of the region has previously charged, 8012ea34989SSeongJae Park * this function splits the region into two so that the second one covers the 8022ea34989SSeongJae Park * area that not charged in the previous charge widnow and saves the second 8032ea34989SSeongJae Park * region in *rp and returns false, so that the caller can apply DAMON action 8042ea34989SSeongJae Park * to the second one. 8052ea34989SSeongJae Park * 8062ea34989SSeongJae Park * Return: true if the region should be entirely skipped, false otherwise. 8072ea34989SSeongJae Park */ 8082ea34989SSeongJae Park static bool damos_skip_charged_region(struct damon_target *t, 8092ea34989SSeongJae Park struct damon_region **rp, struct damos *s) 8102ea34989SSeongJae Park { 8112ea34989SSeongJae Park struct damon_region *r = *rp; 8122ea34989SSeongJae Park struct damos_quota *quota = &s->quota; 8132ea34989SSeongJae Park unsigned long sz_to_skip; 8142ea34989SSeongJae Park 8152ea34989SSeongJae Park /* Skip previously charged regions */ 8162ea34989SSeongJae Park if (quota->charge_target_from) { 8172ea34989SSeongJae Park if (t != quota->charge_target_from) 8182ea34989SSeongJae Park return true; 8192ea34989SSeongJae Park if (r == damon_last_region(t)) { 8202ea34989SSeongJae Park quota->charge_target_from = NULL; 8212ea34989SSeongJae Park quota->charge_addr_from = 0; 8222ea34989SSeongJae Park return true; 8232ea34989SSeongJae Park } 8242ea34989SSeongJae Park if (quota->charge_addr_from && 8252ea34989SSeongJae Park r->ar.end <= quota->charge_addr_from) 8262ea34989SSeongJae Park return true; 8272ea34989SSeongJae Park 8282ea34989SSeongJae Park if (quota->charge_addr_from && r->ar.start < 8292ea34989SSeongJae Park quota->charge_addr_from) { 8302ea34989SSeongJae Park sz_to_skip = ALIGN_DOWN(quota->charge_addr_from - 8312ea34989SSeongJae Park r->ar.start, DAMON_MIN_REGION); 8322ea34989SSeongJae Park if (!sz_to_skip) { 8332ea34989SSeongJae Park if (damon_sz_region(r) <= DAMON_MIN_REGION) 8342ea34989SSeongJae Park return true; 8352ea34989SSeongJae Park sz_to_skip = DAMON_MIN_REGION; 8362ea34989SSeongJae Park } 8372ea34989SSeongJae Park damon_split_region_at(t, r, sz_to_skip); 8382ea34989SSeongJae Park r = damon_next_region(r); 8392ea34989SSeongJae Park *rp = r; 8402ea34989SSeongJae Park } 8412ea34989SSeongJae Park quota->charge_target_from = NULL; 8422ea34989SSeongJae Park quota->charge_addr_from = 0; 8432ea34989SSeongJae Park } 8442ea34989SSeongJae Park return false; 8452ea34989SSeongJae Park } 8462ea34989SSeongJae Park 847d1cbbf62SSeongJae Park static void damos_update_stat(struct damos *s, 848d1cbbf62SSeongJae Park unsigned long sz_tried, unsigned long sz_applied) 849d1cbbf62SSeongJae Park { 850d1cbbf62SSeongJae Park s->stat.nr_tried++; 851d1cbbf62SSeongJae Park s->stat.sz_tried += sz_tried; 852d1cbbf62SSeongJae Park if (sz_applied) 853d1cbbf62SSeongJae Park s->stat.nr_applied++; 854d1cbbf62SSeongJae Park s->stat.sz_applied += sz_applied; 855d1cbbf62SSeongJae Park } 856d1cbbf62SSeongJae Park 85717e7c724SSeongJae Park static bool __damos_filter_out(struct damon_ctx *ctx, struct damon_target *t, 85817e7c724SSeongJae Park struct damon_region *r, struct damos_filter *filter) 859ab9bda00SSeongJae Park { 860ab9bda00SSeongJae Park bool matched = false; 86117e7c724SSeongJae Park struct damon_target *ti; 86217e7c724SSeongJae Park int target_idx = 0; 863ab9bda00SSeongJae Park unsigned long start, end; 864ab9bda00SSeongJae Park 865ab9bda00SSeongJae Park switch (filter->type) { 86617e7c724SSeongJae Park case DAMOS_FILTER_TYPE_TARGET: 86717e7c724SSeongJae Park damon_for_each_target(ti, ctx) { 86817e7c724SSeongJae Park if (ti == t) 86917e7c724SSeongJae Park break; 87017e7c724SSeongJae Park target_idx++; 87117e7c724SSeongJae Park } 87217e7c724SSeongJae Park matched = target_idx == filter->target_idx; 87317e7c724SSeongJae Park break; 874ab9bda00SSeongJae Park case DAMOS_FILTER_TYPE_ADDR: 875ab9bda00SSeongJae Park start = ALIGN_DOWN(filter->addr_range.start, DAMON_MIN_REGION); 876ab9bda00SSeongJae Park end = ALIGN_DOWN(filter->addr_range.end, DAMON_MIN_REGION); 877ab9bda00SSeongJae Park 878ab9bda00SSeongJae Park /* inside the range */ 879ab9bda00SSeongJae Park if (start <= r->ar.start && r->ar.end <= end) { 880ab9bda00SSeongJae Park matched = true; 881ab9bda00SSeongJae Park break; 882ab9bda00SSeongJae Park } 883ab9bda00SSeongJae Park /* outside of the range */ 884ab9bda00SSeongJae Park if (r->ar.end <= start || end <= r->ar.start) { 885ab9bda00SSeongJae Park matched = false; 886ab9bda00SSeongJae Park break; 887ab9bda00SSeongJae Park } 888ab9bda00SSeongJae Park /* start before the range and overlap */ 889ab9bda00SSeongJae Park if (r->ar.start < start) { 890ab9bda00SSeongJae Park damon_split_region_at(t, r, start - r->ar.start); 891ab9bda00SSeongJae Park matched = false; 892ab9bda00SSeongJae Park break; 893ab9bda00SSeongJae Park } 894ab9bda00SSeongJae Park /* start inside the range */ 895ab9bda00SSeongJae Park damon_split_region_at(t, r, end - r->ar.start); 896ab9bda00SSeongJae Park matched = true; 897ab9bda00SSeongJae Park break; 898ab9bda00SSeongJae Park default: 8991777bb5cSHyeongtak Ji return false; 900ab9bda00SSeongJae Park } 901ab9bda00SSeongJae Park 902ab9bda00SSeongJae Park return matched == filter->matching; 903ab9bda00SSeongJae Park } 904ab9bda00SSeongJae Park 90517e7c724SSeongJae Park static bool damos_filter_out(struct damon_ctx *ctx, struct damon_target *t, 90617e7c724SSeongJae Park struct damon_region *r, struct damos *s) 907ab9bda00SSeongJae Park { 908ab9bda00SSeongJae Park struct damos_filter *filter; 909ab9bda00SSeongJae Park 910ab9bda00SSeongJae Park damos_for_each_filter(filter, s) { 91117e7c724SSeongJae Park if (__damos_filter_out(ctx, t, r, filter)) 912ab9bda00SSeongJae Park return true; 913ab9bda00SSeongJae Park } 914ab9bda00SSeongJae Park return false; 915ab9bda00SSeongJae Park } 916ab9bda00SSeongJae Park 917e63a30c5SSeongJae Park static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t, 918e63a30c5SSeongJae Park struct damon_region *r, struct damos *s) 9191f366e42SSeongJae Park { 9202b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 921e63a30c5SSeongJae Park unsigned long sz = damon_sz_region(r); 9221cd24303SSeongJae Park struct timespec64 begin, end; 9230e92c2eeSSeongJae Park unsigned long sz_applied = 0; 92444467bbbSSeongJae Park int err = 0; 9252b8a248dSSeongJae Park 926f7d911c3SSeongJae Park if (c->ops.apply_scheme) { 927e63a30c5SSeongJae Park if (quota->esz && quota->charged_sz + sz > quota->esz) { 9281cd24303SSeongJae Park sz = ALIGN_DOWN(quota->esz - quota->charged_sz, 9292b8a248dSSeongJae Park DAMON_MIN_REGION); 9302b8a248dSSeongJae Park if (!sz) 9312b8a248dSSeongJae Park goto update_stat; 9324ed98243SKaixu Xia damon_split_region_at(t, r, sz); 9332b8a248dSSeongJae Park } 93417e7c724SSeongJae Park if (damos_filter_out(c, t, r, s)) 935ab9bda00SSeongJae Park return; 9361cd24303SSeongJae Park ktime_get_coarse_ts64(&begin); 93744467bbbSSeongJae Park if (c->callback.before_damos_apply) 93844467bbbSSeongJae Park err = c->callback.before_damos_apply(c, t, r, s); 93944467bbbSSeongJae Park if (!err) 940f7d911c3SSeongJae Park sz_applied = c->ops.apply_scheme(c, t, r, s); 9411cd24303SSeongJae Park ktime_get_coarse_ts64(&end); 9421cd24303SSeongJae Park quota->total_charged_ns += timespec64_to_ns(&end) - 9431cd24303SSeongJae Park timespec64_to_ns(&begin); 9442b8a248dSSeongJae Park quota->charged_sz += sz; 9451cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) { 94650585192SSeongJae Park quota->charge_target_from = t; 94750585192SSeongJae Park quota->charge_addr_from = r->ar.end + 1; 94850585192SSeongJae Park } 9492b8a248dSSeongJae Park } 9502f0b548cSSeongJae Park if (s->action != DAMOS_STAT) 9511f366e42SSeongJae Park r->age = 0; 9522b8a248dSSeongJae Park 9532b8a248dSSeongJae Park update_stat: 954d1cbbf62SSeongJae Park damos_update_stat(s, sz, sz_applied); 9551f366e42SSeongJae Park } 956e63a30c5SSeongJae Park 957e63a30c5SSeongJae Park static void damon_do_apply_schemes(struct damon_ctx *c, 958e63a30c5SSeongJae Park struct damon_target *t, 959e63a30c5SSeongJae Park struct damon_region *r) 960e63a30c5SSeongJae Park { 961e63a30c5SSeongJae Park struct damos *s; 962e63a30c5SSeongJae Park 963e63a30c5SSeongJae Park damon_for_each_scheme(s, c) { 964e63a30c5SSeongJae Park struct damos_quota *quota = &s->quota; 965e63a30c5SSeongJae Park 966e63a30c5SSeongJae Park if (!s->wmarks.activated) 967e63a30c5SSeongJae Park continue; 968e63a30c5SSeongJae Park 969e63a30c5SSeongJae Park /* Check the quota */ 970e63a30c5SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 971e63a30c5SSeongJae Park continue; 972e63a30c5SSeongJae Park 973e63a30c5SSeongJae Park if (damos_skip_charged_region(t, &r, s)) 974e63a30c5SSeongJae Park continue; 975e63a30c5SSeongJae Park 976e63a30c5SSeongJae Park if (!damos_valid_target(c, t, r, s)) 977e63a30c5SSeongJae Park continue; 978e63a30c5SSeongJae Park 979e63a30c5SSeongJae Park damos_apply_scheme(c, t, r, s); 980e63a30c5SSeongJae Park } 9811f366e42SSeongJae Park } 9821f366e42SSeongJae Park 9831cd24303SSeongJae Park /* Shouldn't be called if quota->ms and quota->sz are zero */ 9841cd24303SSeongJae Park static void damos_set_effective_quota(struct damos_quota *quota) 9851cd24303SSeongJae Park { 9861cd24303SSeongJae Park unsigned long throughput; 9871cd24303SSeongJae Park unsigned long esz; 9881cd24303SSeongJae Park 9891cd24303SSeongJae Park if (!quota->ms) { 9901cd24303SSeongJae Park quota->esz = quota->sz; 9911cd24303SSeongJae Park return; 9921cd24303SSeongJae Park } 9931cd24303SSeongJae Park 9941cd24303SSeongJae Park if (quota->total_charged_ns) 9951cd24303SSeongJae Park throughput = quota->total_charged_sz * 1000000 / 9961cd24303SSeongJae Park quota->total_charged_ns; 9971cd24303SSeongJae Park else 9981cd24303SSeongJae Park throughput = PAGE_SIZE * 1024; 9991cd24303SSeongJae Park esz = throughput * quota->ms; 10001cd24303SSeongJae Park 10011cd24303SSeongJae Park if (quota->sz && quota->sz < esz) 10021cd24303SSeongJae Park esz = quota->sz; 10031cd24303SSeongJae Park quota->esz = esz; 10041cd24303SSeongJae Park } 10051cd24303SSeongJae Park 1006898810e5SSeongJae Park static void damos_adjust_quota(struct damon_ctx *c, struct damos *s) 10071f366e42SSeongJae Park { 10082b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 1009898810e5SSeongJae Park struct damon_target *t; 1010898810e5SSeongJae Park struct damon_region *r; 101138683e00SSeongJae Park unsigned long cumulated_sz; 101238683e00SSeongJae Park unsigned int score, max_score = 0; 10132b8a248dSSeongJae Park 10141cd24303SSeongJae Park if (!quota->ms && !quota->sz) 1015898810e5SSeongJae Park return; 10162b8a248dSSeongJae Park 10172b8a248dSSeongJae Park /* New charge window starts */ 10182b8a248dSSeongJae Park if (time_after_eq(jiffies, quota->charged_from + 1019898810e5SSeongJae Park msecs_to_jiffies(quota->reset_interval))) { 10206268eac3SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 10216268eac3SSeongJae Park s->stat.qt_exceeds++; 10221cd24303SSeongJae Park quota->total_charged_sz += quota->charged_sz; 10232b8a248dSSeongJae Park quota->charged_from = jiffies; 10242b8a248dSSeongJae Park quota->charged_sz = 0; 10251cd24303SSeongJae Park damos_set_effective_quota(quota); 10262b8a248dSSeongJae Park } 102738683e00SSeongJae Park 1028f7d911c3SSeongJae Park if (!c->ops.get_scheme_score) 1029898810e5SSeongJae Park return; 103038683e00SSeongJae Park 103138683e00SSeongJae Park /* Fill up the score histogram */ 103238683e00SSeongJae Park memset(quota->histogram, 0, sizeof(quota->histogram)); 103338683e00SSeongJae Park damon_for_each_target(t, c) { 103438683e00SSeongJae Park damon_for_each_region(r, t) { 103538683e00SSeongJae Park if (!__damos_valid_target(r, s)) 103638683e00SSeongJae Park continue; 1037898810e5SSeongJae Park score = c->ops.get_scheme_score(c, t, r, s); 1038ab63f63fSXin Hao quota->histogram[score] += damon_sz_region(r); 103938683e00SSeongJae Park if (score > max_score) 104038683e00SSeongJae Park max_score = score; 104138683e00SSeongJae Park } 104238683e00SSeongJae Park } 104338683e00SSeongJae Park 104438683e00SSeongJae Park /* Set the min score limit */ 104538683e00SSeongJae Park for (cumulated_sz = 0, score = max_score; ; score--) { 104638683e00SSeongJae Park cumulated_sz += quota->histogram[score]; 104738683e00SSeongJae Park if (cumulated_sz >= quota->esz || !score) 104838683e00SSeongJae Park break; 104938683e00SSeongJae Park } 105038683e00SSeongJae Park quota->min_score = score; 10512b8a248dSSeongJae Park } 10521f366e42SSeongJae Park 1053898810e5SSeongJae Park static void kdamond_apply_schemes(struct damon_ctx *c) 1054898810e5SSeongJae Park { 1055898810e5SSeongJae Park struct damon_target *t; 1056898810e5SSeongJae Park struct damon_region *r, *next_r; 1057898810e5SSeongJae Park struct damos *s; 1058898810e5SSeongJae Park 1059898810e5SSeongJae Park damon_for_each_scheme(s, c) { 1060898810e5SSeongJae Park if (!s->wmarks.activated) 1061898810e5SSeongJae Park continue; 1062898810e5SSeongJae Park 1063898810e5SSeongJae Park damos_adjust_quota(c, s); 1064898810e5SSeongJae Park } 1065898810e5SSeongJae Park 10661f366e42SSeongJae Park damon_for_each_target(t, c) { 10672b8a248dSSeongJae Park damon_for_each_region_safe(r, next_r, t) 10681f366e42SSeongJae Park damon_do_apply_schemes(c, t, r); 10691f366e42SSeongJae Park } 10701f366e42SSeongJae Park } 10711f366e42SSeongJae Park 1072b9a6ac4eSSeongJae Park /* 1073b9a6ac4eSSeongJae Park * Merge two adjacent regions into one region 1074b9a6ac4eSSeongJae Park */ 1075b9a6ac4eSSeongJae Park static void damon_merge_two_regions(struct damon_target *t, 1076b9a6ac4eSSeongJae Park struct damon_region *l, struct damon_region *r) 1077b9a6ac4eSSeongJae Park { 1078652e0446SXin Hao unsigned long sz_l = damon_sz_region(l), sz_r = damon_sz_region(r); 1079b9a6ac4eSSeongJae Park 1080b9a6ac4eSSeongJae Park l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / 1081b9a6ac4eSSeongJae Park (sz_l + sz_r); 1082fda504faSSeongJae Park l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); 1083b9a6ac4eSSeongJae Park l->ar.end = r->ar.end; 1084b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 1085b9a6ac4eSSeongJae Park } 1086b9a6ac4eSSeongJae Park 1087b9a6ac4eSSeongJae Park /* 1088b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 1089b9a6ac4eSSeongJae Park * 1090b9a6ac4eSSeongJae Park * t target affected by this merge operation 1091b9a6ac4eSSeongJae Park * thres '->nr_accesses' diff threshold for the merge 1092b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 1093b9a6ac4eSSeongJae Park */ 1094b9a6ac4eSSeongJae Park static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, 1095b9a6ac4eSSeongJae Park unsigned long sz_limit) 1096b9a6ac4eSSeongJae Park { 1097b9a6ac4eSSeongJae Park struct damon_region *r, *prev = NULL, *next; 1098b9a6ac4eSSeongJae Park 1099b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 1100d720bbbdSXin Hao if (abs(r->nr_accesses - r->last_nr_accesses) > thres) 1101fda504faSSeongJae Park r->age = 0; 1102fda504faSSeongJae Park else 1103fda504faSSeongJae Park r->age++; 1104fda504faSSeongJae Park 1105b9a6ac4eSSeongJae Park if (prev && prev->ar.end == r->ar.start && 1106d720bbbdSXin Hao abs(prev->nr_accesses - r->nr_accesses) <= thres && 1107652e0446SXin Hao damon_sz_region(prev) + damon_sz_region(r) <= sz_limit) 1108b9a6ac4eSSeongJae Park damon_merge_two_regions(t, prev, r); 1109b9a6ac4eSSeongJae Park else 1110b9a6ac4eSSeongJae Park prev = r; 1111b9a6ac4eSSeongJae Park } 1112b9a6ac4eSSeongJae Park } 1113b9a6ac4eSSeongJae Park 1114b9a6ac4eSSeongJae Park /* 1115b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 1116b9a6ac4eSSeongJae Park * 1117b9a6ac4eSSeongJae Park * threshold '->nr_accesses' diff threshold for the merge 1118b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 1119b9a6ac4eSSeongJae Park * 1120b9a6ac4eSSeongJae Park * This function merges monitoring target regions which are adjacent and their 1121b9a6ac4eSSeongJae Park * access frequencies are similar. This is for minimizing the monitoring 1122b9a6ac4eSSeongJae Park * overhead under the dynamically changeable access pattern. If a merge was 1123b9a6ac4eSSeongJae Park * unnecessarily made, later 'kdamond_split_regions()' will revert it. 1124b9a6ac4eSSeongJae Park */ 1125b9a6ac4eSSeongJae Park static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, 1126b9a6ac4eSSeongJae Park unsigned long sz_limit) 1127b9a6ac4eSSeongJae Park { 1128b9a6ac4eSSeongJae Park struct damon_target *t; 1129b9a6ac4eSSeongJae Park 1130b9a6ac4eSSeongJae Park damon_for_each_target(t, c) 1131b9a6ac4eSSeongJae Park damon_merge_regions_of(t, threshold, sz_limit); 1132b9a6ac4eSSeongJae Park } 1133b9a6ac4eSSeongJae Park 1134b9a6ac4eSSeongJae Park /* 1135b9a6ac4eSSeongJae Park * Split a region in two 1136b9a6ac4eSSeongJae Park * 1137b9a6ac4eSSeongJae Park * r the region to be split 1138b9a6ac4eSSeongJae Park * sz_r size of the first sub-region that will be made 1139b9a6ac4eSSeongJae Park */ 11404ed98243SKaixu Xia static void damon_split_region_at(struct damon_target *t, 11414ed98243SKaixu Xia struct damon_region *r, unsigned long sz_r) 1142b9a6ac4eSSeongJae Park { 1143b9a6ac4eSSeongJae Park struct damon_region *new; 1144b9a6ac4eSSeongJae Park 1145b9a6ac4eSSeongJae Park new = damon_new_region(r->ar.start + sz_r, r->ar.end); 1146b9a6ac4eSSeongJae Park if (!new) 1147b9a6ac4eSSeongJae Park return; 1148b9a6ac4eSSeongJae Park 1149b9a6ac4eSSeongJae Park r->ar.end = new->ar.start; 1150b9a6ac4eSSeongJae Park 1151fda504faSSeongJae Park new->age = r->age; 1152fda504faSSeongJae Park new->last_nr_accesses = r->last_nr_accesses; 1153fda504faSSeongJae Park 1154b9a6ac4eSSeongJae Park damon_insert_region(new, r, damon_next_region(r), t); 1155b9a6ac4eSSeongJae Park } 1156b9a6ac4eSSeongJae Park 1157b9a6ac4eSSeongJae Park /* Split every region in the given target into 'nr_subs' regions */ 11584ed98243SKaixu Xia static void damon_split_regions_of(struct damon_target *t, int nr_subs) 1159b9a6ac4eSSeongJae Park { 1160b9a6ac4eSSeongJae Park struct damon_region *r, *next; 1161b9a6ac4eSSeongJae Park unsigned long sz_region, sz_sub = 0; 1162b9a6ac4eSSeongJae Park int i; 1163b9a6ac4eSSeongJae Park 1164b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 1165ab63f63fSXin Hao sz_region = damon_sz_region(r); 1166b9a6ac4eSSeongJae Park 1167b9a6ac4eSSeongJae Park for (i = 0; i < nr_subs - 1 && 1168b9a6ac4eSSeongJae Park sz_region > 2 * DAMON_MIN_REGION; i++) { 1169b9a6ac4eSSeongJae Park /* 1170b9a6ac4eSSeongJae Park * Randomly select size of left sub-region to be at 1171b9a6ac4eSSeongJae Park * least 10 percent and at most 90% of original region 1172b9a6ac4eSSeongJae Park */ 1173b9a6ac4eSSeongJae Park sz_sub = ALIGN_DOWN(damon_rand(1, 10) * 1174b9a6ac4eSSeongJae Park sz_region / 10, DAMON_MIN_REGION); 1175b9a6ac4eSSeongJae Park /* Do not allow blank region */ 1176b9a6ac4eSSeongJae Park if (sz_sub == 0 || sz_sub >= sz_region) 1177b9a6ac4eSSeongJae Park continue; 1178b9a6ac4eSSeongJae Park 11794ed98243SKaixu Xia damon_split_region_at(t, r, sz_sub); 1180b9a6ac4eSSeongJae Park sz_region = sz_sub; 1181b9a6ac4eSSeongJae Park } 1182b9a6ac4eSSeongJae Park } 1183b9a6ac4eSSeongJae Park } 1184b9a6ac4eSSeongJae Park 1185b9a6ac4eSSeongJae Park /* 1186b9a6ac4eSSeongJae Park * Split every target region into randomly-sized small regions 1187b9a6ac4eSSeongJae Park * 1188b9a6ac4eSSeongJae Park * This function splits every target region into random-sized small regions if 1189b9a6ac4eSSeongJae Park * current total number of the regions is equal or smaller than half of the 1190b9a6ac4eSSeongJae Park * user-specified maximum number of regions. This is for maximizing the 1191b9a6ac4eSSeongJae Park * monitoring accuracy under the dynamically changeable access patterns. If a 1192b9a6ac4eSSeongJae Park * split was unnecessarily made, later 'kdamond_merge_regions()' will revert 1193b9a6ac4eSSeongJae Park * it. 1194b9a6ac4eSSeongJae Park */ 1195b9a6ac4eSSeongJae Park static void kdamond_split_regions(struct damon_ctx *ctx) 1196b9a6ac4eSSeongJae Park { 1197b9a6ac4eSSeongJae Park struct damon_target *t; 1198b9a6ac4eSSeongJae Park unsigned int nr_regions = 0; 1199b9a6ac4eSSeongJae Park static unsigned int last_nr_regions; 1200b9a6ac4eSSeongJae Park int nr_subregions = 2; 1201b9a6ac4eSSeongJae Park 1202b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 1203b9a6ac4eSSeongJae Park nr_regions += damon_nr_regions(t); 1204b9a6ac4eSSeongJae Park 1205cbeaa77bSSeongJae Park if (nr_regions > ctx->attrs.max_nr_regions / 2) 1206b9a6ac4eSSeongJae Park return; 1207b9a6ac4eSSeongJae Park 1208b9a6ac4eSSeongJae Park /* Maybe the middle of the region has different access frequency */ 1209b9a6ac4eSSeongJae Park if (last_nr_regions == nr_regions && 1210cbeaa77bSSeongJae Park nr_regions < ctx->attrs.max_nr_regions / 3) 1211b9a6ac4eSSeongJae Park nr_subregions = 3; 1212b9a6ac4eSSeongJae Park 1213b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 12144ed98243SKaixu Xia damon_split_regions_of(t, nr_subregions); 1215b9a6ac4eSSeongJae Park 1216b9a6ac4eSSeongJae Park last_nr_regions = nr_regions; 1217b9a6ac4eSSeongJae Park } 1218b9a6ac4eSSeongJae Park 1219f23b8eeeSSeongJae Park /* 12202224d848SSeongJae Park * Check whether current monitoring should be stopped 12212224d848SSeongJae Park * 12222224d848SSeongJae Park * The monitoring is stopped when either the user requested to stop, or all 12232224d848SSeongJae Park * monitoring targets are invalid. 12242224d848SSeongJae Park * 12252224d848SSeongJae Park * Returns true if need to stop current monitoring. 12262224d848SSeongJae Park */ 12272224d848SSeongJae Park static bool kdamond_need_stop(struct damon_ctx *ctx) 12282224d848SSeongJae Park { 1229f23b8eeeSSeongJae Park struct damon_target *t; 12302224d848SSeongJae Park 12310f91d133SChangbin Du if (kthread_should_stop()) 12322224d848SSeongJae Park return true; 12332224d848SSeongJae Park 1234f7d911c3SSeongJae Park if (!ctx->ops.target_valid) 12352224d848SSeongJae Park return false; 12362224d848SSeongJae Park 1237f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 1238f7d911c3SSeongJae Park if (ctx->ops.target_valid(t)) 1239f23b8eeeSSeongJae Park return false; 1240f23b8eeeSSeongJae Park } 1241f23b8eeeSSeongJae Park 1242f23b8eeeSSeongJae Park return true; 12432224d848SSeongJae Park } 12442224d848SSeongJae Park 1245ee801b7dSSeongJae Park static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric) 1246ee801b7dSSeongJae Park { 1247ee801b7dSSeongJae Park struct sysinfo i; 1248ee801b7dSSeongJae Park 1249ee801b7dSSeongJae Park switch (metric) { 1250ee801b7dSSeongJae Park case DAMOS_WMARK_FREE_MEM_RATE: 1251ee801b7dSSeongJae Park si_meminfo(&i); 1252ee801b7dSSeongJae Park return i.freeram * 1000 / i.totalram; 1253ee801b7dSSeongJae Park default: 1254ee801b7dSSeongJae Park break; 1255ee801b7dSSeongJae Park } 1256ee801b7dSSeongJae Park return -EINVAL; 1257ee801b7dSSeongJae Park } 1258ee801b7dSSeongJae Park 1259ee801b7dSSeongJae Park /* 1260ee801b7dSSeongJae Park * Returns zero if the scheme is active. Else, returns time to wait for next 1261ee801b7dSSeongJae Park * watermark check in micro-seconds. 1262ee801b7dSSeongJae Park */ 1263ee801b7dSSeongJae Park static unsigned long damos_wmark_wait_us(struct damos *scheme) 1264ee801b7dSSeongJae Park { 1265ee801b7dSSeongJae Park unsigned long metric; 1266ee801b7dSSeongJae Park 1267ee801b7dSSeongJae Park if (scheme->wmarks.metric == DAMOS_WMARK_NONE) 1268ee801b7dSSeongJae Park return 0; 1269ee801b7dSSeongJae Park 1270ee801b7dSSeongJae Park metric = damos_wmark_metric_value(scheme->wmarks.metric); 1271ee801b7dSSeongJae Park /* higher than high watermark or lower than low watermark */ 1272ee801b7dSSeongJae Park if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { 1273ee801b7dSSeongJae Park if (scheme->wmarks.activated) 127401078655SColin Ian King pr_debug("deactivate a scheme (%d) for %s wmark\n", 1275ee801b7dSSeongJae Park scheme->action, 1276ee801b7dSSeongJae Park metric > scheme->wmarks.high ? 1277ee801b7dSSeongJae Park "high" : "low"); 1278ee801b7dSSeongJae Park scheme->wmarks.activated = false; 1279ee801b7dSSeongJae Park return scheme->wmarks.interval; 1280ee801b7dSSeongJae Park } 1281ee801b7dSSeongJae Park 1282ee801b7dSSeongJae Park /* inactive and higher than middle watermark */ 1283ee801b7dSSeongJae Park if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) && 1284ee801b7dSSeongJae Park !scheme->wmarks.activated) 1285ee801b7dSSeongJae Park return scheme->wmarks.interval; 1286ee801b7dSSeongJae Park 1287ee801b7dSSeongJae Park if (!scheme->wmarks.activated) 1288ee801b7dSSeongJae Park pr_debug("activate a scheme (%d)\n", scheme->action); 1289ee801b7dSSeongJae Park scheme->wmarks.activated = true; 1290ee801b7dSSeongJae Park return 0; 1291ee801b7dSSeongJae Park } 1292ee801b7dSSeongJae Park 1293ee801b7dSSeongJae Park static void kdamond_usleep(unsigned long usecs) 1294ee801b7dSSeongJae Park { 12954de46a30SSeongJae Park /* See Documentation/timers/timers-howto.rst for the thresholds */ 12964de46a30SSeongJae Park if (usecs > 20 * USEC_PER_MSEC) 129770e92748SSeongJae Park schedule_timeout_idle(usecs_to_jiffies(usecs)); 1298ee801b7dSSeongJae Park else 129970e92748SSeongJae Park usleep_idle_range(usecs, usecs + 1); 1300ee801b7dSSeongJae Park } 1301ee801b7dSSeongJae Park 1302ee801b7dSSeongJae Park /* Returns negative error code if it's not activated but should return */ 1303ee801b7dSSeongJae Park static int kdamond_wait_activation(struct damon_ctx *ctx) 1304ee801b7dSSeongJae Park { 1305ee801b7dSSeongJae Park struct damos *s; 1306ee801b7dSSeongJae Park unsigned long wait_time; 1307ee801b7dSSeongJae Park unsigned long min_wait_time = 0; 130878049e94SJonghyeon Kim bool init_wait_time = false; 1309ee801b7dSSeongJae Park 1310ee801b7dSSeongJae Park while (!kdamond_need_stop(ctx)) { 1311ee801b7dSSeongJae Park damon_for_each_scheme(s, ctx) { 1312ee801b7dSSeongJae Park wait_time = damos_wmark_wait_us(s); 131378049e94SJonghyeon Kim if (!init_wait_time || wait_time < min_wait_time) { 131478049e94SJonghyeon Kim init_wait_time = true; 1315ee801b7dSSeongJae Park min_wait_time = wait_time; 1316ee801b7dSSeongJae Park } 131778049e94SJonghyeon Kim } 1318ee801b7dSSeongJae Park if (!min_wait_time) 1319ee801b7dSSeongJae Park return 0; 1320ee801b7dSSeongJae Park 1321ee801b7dSSeongJae Park kdamond_usleep(min_wait_time); 13226e74d2bfSSeongJae Park 13236e74d2bfSSeongJae Park if (ctx->callback.after_wmarks_check && 13246e74d2bfSSeongJae Park ctx->callback.after_wmarks_check(ctx)) 13256e74d2bfSSeongJae Park break; 1326ee801b7dSSeongJae Park } 1327ee801b7dSSeongJae Park return -EBUSY; 1328ee801b7dSSeongJae Park } 1329ee801b7dSSeongJae Park 1330c708a5e5SSeongJae Park static void kdamond_init_intervals_sis(struct damon_ctx *ctx) 1331c708a5e5SSeongJae Park { 1332c708a5e5SSeongJae Park unsigned long sample_interval = ctx->attrs.sample_interval ? 1333c708a5e5SSeongJae Park ctx->attrs.sample_interval : 1; 1334c708a5e5SSeongJae Park 1335c708a5e5SSeongJae Park ctx->passed_sample_intervals = 0; 1336c708a5e5SSeongJae Park ctx->next_aggregation_sis = ctx->attrs.aggr_interval / sample_interval; 1337c708a5e5SSeongJae Park ctx->next_ops_update_sis = ctx->attrs.ops_update_interval / 1338c708a5e5SSeongJae Park sample_interval; 1339c708a5e5SSeongJae Park } 1340c708a5e5SSeongJae Park 13412224d848SSeongJae Park /* 13422224d848SSeongJae Park * The monitoring daemon that runs as a kernel thread 13432224d848SSeongJae Park */ 13442224d848SSeongJae Park static int kdamond_fn(void *data) 13452224d848SSeongJae Park { 1346cef4493fSYu Zhe struct damon_ctx *ctx = data; 1347f23b8eeeSSeongJae Park struct damon_target *t; 1348f23b8eeeSSeongJae Park struct damon_region *r, *next; 1349b9a6ac4eSSeongJae Park unsigned int max_nr_accesses = 0; 1350b9a6ac4eSSeongJae Park unsigned long sz_limit = 0; 13512224d848SSeongJae Park 135242e4cef5SChangbin Du pr_debug("kdamond (%d) starts\n", current->pid); 13532224d848SSeongJae Park 1354e93bcaebSSeongJae Park complete(&ctx->kdamond_started); 1355c708a5e5SSeongJae Park kdamond_init_intervals_sis(ctx); 1356c708a5e5SSeongJae Park 1357f7d911c3SSeongJae Park if (ctx->ops.init) 1358f7d911c3SSeongJae Park ctx->ops.init(ctx); 13592224d848SSeongJae Park if (ctx->callback.before_start && ctx->callback.before_start(ctx)) 136029454cf6SKaixu Xia goto done; 13612224d848SSeongJae Park 1362b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 1363b9a6ac4eSSeongJae Park 136429454cf6SKaixu Xia while (!kdamond_need_stop(ctx)) { 1365c708a5e5SSeongJae Park /* 1366c708a5e5SSeongJae Park * ctx->attrs and ctx->next_{aggregation,ops_update}_sis could 1367c708a5e5SSeongJae Park * be changed from after_wmarks_check() or after_aggregation() 1368c708a5e5SSeongJae Park * callbacks. Read the values here, and use those for this 1369c708a5e5SSeongJae Park * iteration. That is, damon_set_attrs() updated new values 1370c708a5e5SSeongJae Park * are respected from next iteration. 1371c708a5e5SSeongJae Park */ 1372c708a5e5SSeongJae Park unsigned long next_aggregation_sis = ctx->next_aggregation_sis; 1373c708a5e5SSeongJae Park unsigned long next_ops_update_sis = ctx->next_ops_update_sis; 1374c708a5e5SSeongJae Park unsigned long sample_interval = ctx->attrs.sample_interval; 1375c708a5e5SSeongJae Park 137629454cf6SKaixu Xia if (kdamond_wait_activation(ctx)) 137729454cf6SKaixu Xia break; 1378ee801b7dSSeongJae Park 1379f7d911c3SSeongJae Park if (ctx->ops.prepare_access_checks) 1380f7d911c3SSeongJae Park ctx->ops.prepare_access_checks(ctx); 13812224d848SSeongJae Park if (ctx->callback.after_sampling && 138229454cf6SKaixu Xia ctx->callback.after_sampling(ctx)) 138329454cf6SKaixu Xia break; 13842224d848SSeongJae Park 1385c708a5e5SSeongJae Park kdamond_usleep(sample_interval); 1386c708a5e5SSeongJae Park ctx->passed_sample_intervals++; 13872224d848SSeongJae Park 1388f7d911c3SSeongJae Park if (ctx->ops.check_accesses) 1389f7d911c3SSeongJae Park max_nr_accesses = ctx->ops.check_accesses(ctx); 13902224d848SSeongJae Park 1391c708a5e5SSeongJae Park sample_interval = ctx->attrs.sample_interval ? 1392c708a5e5SSeongJae Park ctx->attrs.sample_interval : 1; 1393c708a5e5SSeongJae Park if (ctx->passed_sample_intervals == next_aggregation_sis) { 1394c708a5e5SSeongJae Park ctx->next_aggregation_sis = next_aggregation_sis + 1395c708a5e5SSeongJae Park ctx->attrs.aggr_interval / sample_interval; 1396b9a6ac4eSSeongJae Park kdamond_merge_regions(ctx, 1397b9a6ac4eSSeongJae Park max_nr_accesses / 10, 1398b9a6ac4eSSeongJae Park sz_limit); 13992224d848SSeongJae Park if (ctx->callback.after_aggregation && 140029454cf6SKaixu Xia ctx->callback.after_aggregation(ctx)) 140129454cf6SKaixu Xia break; 140264517d6eSHuaisheng Ye if (!list_empty(&ctx->schemes)) 14031f366e42SSeongJae Park kdamond_apply_schemes(ctx); 1404f23b8eeeSSeongJae Park kdamond_reset_aggregated(ctx); 1405b9a6ac4eSSeongJae Park kdamond_split_regions(ctx); 1406f7d911c3SSeongJae Park if (ctx->ops.reset_aggregated) 1407f7d911c3SSeongJae Park ctx->ops.reset_aggregated(ctx); 14082224d848SSeongJae Park } 14092224d848SSeongJae Park 1410c708a5e5SSeongJae Park if (ctx->passed_sample_intervals == next_ops_update_sis) { 1411c708a5e5SSeongJae Park ctx->next_ops_update_sis = next_ops_update_sis + 1412c708a5e5SSeongJae Park ctx->attrs.ops_update_interval / 1413c708a5e5SSeongJae Park sample_interval; 1414f7d911c3SSeongJae Park if (ctx->ops.update) 1415f7d911c3SSeongJae Park ctx->ops.update(ctx); 1416b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 14172224d848SSeongJae Park } 14182224d848SSeongJae Park } 141929454cf6SKaixu Xia done: 1420f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 1421f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 1422b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 1423f23b8eeeSSeongJae Park } 14242224d848SSeongJae Park 14250f91d133SChangbin Du if (ctx->callback.before_terminate) 14260f91d133SChangbin Du ctx->callback.before_terminate(ctx); 1427f7d911c3SSeongJae Park if (ctx->ops.cleanup) 1428f7d911c3SSeongJae Park ctx->ops.cleanup(ctx); 14292224d848SSeongJae Park 143042e4cef5SChangbin Du pr_debug("kdamond (%d) finishes\n", current->pid); 14312224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 14322224d848SSeongJae Park ctx->kdamond = NULL; 14332224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 14342224d848SSeongJae Park 14352224d848SSeongJae Park mutex_lock(&damon_lock); 14362224d848SSeongJae Park nr_running_ctxs--; 14378b9b0d33SSeongJae Park if (!nr_running_ctxs && running_exclusive_ctxs) 14388b9b0d33SSeongJae Park running_exclusive_ctxs = false; 14392224d848SSeongJae Park mutex_unlock(&damon_lock); 14402224d848SSeongJae Park 14415f7fe2b9SChangbin Du return 0; 14422224d848SSeongJae Park } 144317ccae8bSSeongJae Park 14440d83b2d8SXin Hao /* 14450d83b2d8SXin Hao * struct damon_system_ram_region - System RAM resource address region of 14460d83b2d8SXin Hao * [@start, @end). 14470d83b2d8SXin Hao * @start: Start address of the region (inclusive). 14480d83b2d8SXin Hao * @end: End address of the region (exclusive). 14490d83b2d8SXin Hao */ 14500d83b2d8SXin Hao struct damon_system_ram_region { 14510d83b2d8SXin Hao unsigned long start; 14520d83b2d8SXin Hao unsigned long end; 14530d83b2d8SXin Hao }; 14540d83b2d8SXin Hao 14550d83b2d8SXin Hao static int walk_system_ram(struct resource *res, void *arg) 14560d83b2d8SXin Hao { 14570d83b2d8SXin Hao struct damon_system_ram_region *a = arg; 14580d83b2d8SXin Hao 14590d83b2d8SXin Hao if (a->end - a->start < resource_size(res)) { 14600d83b2d8SXin Hao a->start = res->start; 14610d83b2d8SXin Hao a->end = res->end; 14620d83b2d8SXin Hao } 14630d83b2d8SXin Hao return 0; 14640d83b2d8SXin Hao } 14650d83b2d8SXin Hao 14660d83b2d8SXin Hao /* 14670d83b2d8SXin Hao * Find biggest 'System RAM' resource and store its start and end address in 14680d83b2d8SXin Hao * @start and @end, respectively. If no System RAM is found, returns false. 14690d83b2d8SXin Hao */ 1470233f0b31SKaixu Xia static bool damon_find_biggest_system_ram(unsigned long *start, 1471233f0b31SKaixu Xia unsigned long *end) 14720d83b2d8SXin Hao 14730d83b2d8SXin Hao { 14740d83b2d8SXin Hao struct damon_system_ram_region arg = {}; 14750d83b2d8SXin Hao 14760d83b2d8SXin Hao walk_system_ram_res(0, ULONG_MAX, &arg, walk_system_ram); 14770d83b2d8SXin Hao if (arg.end <= arg.start) 14780d83b2d8SXin Hao return false; 14790d83b2d8SXin Hao 14800d83b2d8SXin Hao *start = arg.start; 14810d83b2d8SXin Hao *end = arg.end; 14820d83b2d8SXin Hao return true; 14830d83b2d8SXin Hao } 14840d83b2d8SXin Hao 1485233f0b31SKaixu Xia /** 1486233f0b31SKaixu Xia * damon_set_region_biggest_system_ram_default() - Set the region of the given 1487233f0b31SKaixu Xia * monitoring target as requested, or biggest 'System RAM'. 1488233f0b31SKaixu Xia * @t: The monitoring target to set the region. 1489233f0b31SKaixu Xia * @start: The pointer to the start address of the region. 1490233f0b31SKaixu Xia * @end: The pointer to the end address of the region. 1491233f0b31SKaixu Xia * 1492233f0b31SKaixu Xia * This function sets the region of @t as requested by @start and @end. If the 1493233f0b31SKaixu Xia * values of @start and @end are zero, however, this function finds the biggest 1494233f0b31SKaixu Xia * 'System RAM' resource and sets the region to cover the resource. In the 1495233f0b31SKaixu Xia * latter case, this function saves the start and end addresses of the resource 1496233f0b31SKaixu Xia * in @start and @end, respectively. 1497233f0b31SKaixu Xia * 1498233f0b31SKaixu Xia * Return: 0 on success, negative error code otherwise. 1499233f0b31SKaixu Xia */ 1500233f0b31SKaixu Xia int damon_set_region_biggest_system_ram_default(struct damon_target *t, 1501233f0b31SKaixu Xia unsigned long *start, unsigned long *end) 1502233f0b31SKaixu Xia { 1503233f0b31SKaixu Xia struct damon_addr_range addr_range; 1504233f0b31SKaixu Xia 1505233f0b31SKaixu Xia if (*start > *end) 1506233f0b31SKaixu Xia return -EINVAL; 1507233f0b31SKaixu Xia 1508233f0b31SKaixu Xia if (!*start && !*end && 1509233f0b31SKaixu Xia !damon_find_biggest_system_ram(start, end)) 1510233f0b31SKaixu Xia return -EINVAL; 1511233f0b31SKaixu Xia 1512233f0b31SKaixu Xia addr_range.start = *start; 1513233f0b31SKaixu Xia addr_range.end = *end; 1514233f0b31SKaixu Xia return damon_set_regions(t, &addr_range, 1); 1515233f0b31SKaixu Xia } 1516233f0b31SKaixu Xia 1517a1870944SDawei Li static int __init damon_init(void) 1518a1870944SDawei Li { 1519a1870944SDawei Li damon_region_cache = KMEM_CACHE(damon_region, 0); 1520a1870944SDawei Li if (unlikely(!damon_region_cache)) { 1521a1870944SDawei Li pr_err("creating damon_region_cache fails\n"); 1522a1870944SDawei Li return -ENOMEM; 1523a1870944SDawei Li } 1524a1870944SDawei Li 1525a1870944SDawei Li return 0; 1526a1870944SDawei Li } 1527a1870944SDawei Li 1528a1870944SDawei Li subsys_initcall(damon_init); 1529a1870944SDawei Li 153017ccae8bSSeongJae Park #include "core-test.h" 1531