12224d848SSeongJae Park // SPDX-License-Identifier: GPL-2.0 22224d848SSeongJae Park /* 32224d848SSeongJae Park * Data Access Monitor 42224d848SSeongJae Park * 52224d848SSeongJae Park * Author: SeongJae Park <sjpark@amazon.de> 62224d848SSeongJae Park */ 72224d848SSeongJae Park 82224d848SSeongJae Park #define pr_fmt(fmt) "damon: " fmt 92224d848SSeongJae Park 102224d848SSeongJae Park #include <linux/damon.h> 112224d848SSeongJae Park #include <linux/delay.h> 122224d848SSeongJae Park #include <linux/kthread.h> 13ee801b7dSSeongJae Park #include <linux/mm.h> 142224d848SSeongJae Park #include <linux/slab.h> 1538683e00SSeongJae Park #include <linux/string.h> 162224d848SSeongJae Park 172fcb9362SSeongJae Park #define CREATE_TRACE_POINTS 182fcb9362SSeongJae Park #include <trace/events/damon.h> 192fcb9362SSeongJae Park 2017ccae8bSSeongJae Park #ifdef CONFIG_DAMON_KUNIT_TEST 2117ccae8bSSeongJae Park #undef DAMON_MIN_REGION 2217ccae8bSSeongJae Park #define DAMON_MIN_REGION 1 2317ccae8bSSeongJae Park #endif 2417ccae8bSSeongJae Park 252224d848SSeongJae Park static DEFINE_MUTEX(damon_lock); 262224d848SSeongJae Park static int nr_running_ctxs; 278b9b0d33SSeongJae Park static bool running_exclusive_ctxs; 282224d848SSeongJae Park 299f7b053aSSeongJae Park static DEFINE_MUTEX(damon_ops_lock); 309f7b053aSSeongJae Park static struct damon_operations damon_registered_ops[NR_DAMON_OPS]; 319f7b053aSSeongJae Park 329f7b053aSSeongJae Park /* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */ 33152e5617SSeongJae Park static bool __damon_is_registered_ops(enum damon_ops_id id) 349f7b053aSSeongJae Park { 359f7b053aSSeongJae Park struct damon_operations empty_ops = {}; 369f7b053aSSeongJae Park 379f7b053aSSeongJae Park if (!memcmp(&empty_ops, &damon_registered_ops[id], sizeof(empty_ops))) 389f7b053aSSeongJae Park return false; 399f7b053aSSeongJae Park return true; 409f7b053aSSeongJae Park } 419f7b053aSSeongJae Park 429f7b053aSSeongJae Park /** 43152e5617SSeongJae Park * damon_is_registered_ops() - Check if a given damon_operations is registered. 44152e5617SSeongJae Park * @id: Id of the damon_operations to check if registered. 45152e5617SSeongJae Park * 46152e5617SSeongJae Park * Return: true if the ops is set, false otherwise. 47152e5617SSeongJae Park */ 48152e5617SSeongJae Park bool damon_is_registered_ops(enum damon_ops_id id) 49152e5617SSeongJae Park { 50152e5617SSeongJae Park bool registered; 51152e5617SSeongJae Park 52152e5617SSeongJae Park if (id >= NR_DAMON_OPS) 53152e5617SSeongJae Park return false; 54152e5617SSeongJae Park mutex_lock(&damon_ops_lock); 55152e5617SSeongJae Park registered = __damon_is_registered_ops(id); 56152e5617SSeongJae Park mutex_unlock(&damon_ops_lock); 57152e5617SSeongJae Park return registered; 58152e5617SSeongJae Park } 59152e5617SSeongJae Park 60152e5617SSeongJae Park /** 619f7b053aSSeongJae Park * damon_register_ops() - Register a monitoring operations set to DAMON. 629f7b053aSSeongJae Park * @ops: monitoring operations set to register. 639f7b053aSSeongJae Park * 649f7b053aSSeongJae Park * This function registers a monitoring operations set of valid &struct 659f7b053aSSeongJae Park * damon_operations->id so that others can find and use them later. 669f7b053aSSeongJae Park * 679f7b053aSSeongJae Park * Return: 0 on success, negative error code otherwise. 689f7b053aSSeongJae Park */ 699f7b053aSSeongJae Park int damon_register_ops(struct damon_operations *ops) 709f7b053aSSeongJae Park { 719f7b053aSSeongJae Park int err = 0; 729f7b053aSSeongJae Park 739f7b053aSSeongJae Park if (ops->id >= NR_DAMON_OPS) 749f7b053aSSeongJae Park return -EINVAL; 759f7b053aSSeongJae Park mutex_lock(&damon_ops_lock); 769f7b053aSSeongJae Park /* Fail for already registered ops */ 77152e5617SSeongJae Park if (__damon_is_registered_ops(ops->id)) { 789f7b053aSSeongJae Park err = -EINVAL; 799f7b053aSSeongJae Park goto out; 809f7b053aSSeongJae Park } 819f7b053aSSeongJae Park damon_registered_ops[ops->id] = *ops; 829f7b053aSSeongJae Park out: 839f7b053aSSeongJae Park mutex_unlock(&damon_ops_lock); 849f7b053aSSeongJae Park return err; 859f7b053aSSeongJae Park } 869f7b053aSSeongJae Park 879f7b053aSSeongJae Park /** 889f7b053aSSeongJae Park * damon_select_ops() - Select a monitoring operations to use with the context. 899f7b053aSSeongJae Park * @ctx: monitoring context to use the operations. 909f7b053aSSeongJae Park * @id: id of the registered monitoring operations to select. 919f7b053aSSeongJae Park * 929f7b053aSSeongJae Park * This function finds registered monitoring operations set of @id and make 939f7b053aSSeongJae Park * @ctx to use it. 949f7b053aSSeongJae Park * 959f7b053aSSeongJae Park * Return: 0 on success, negative error code otherwise. 969f7b053aSSeongJae Park */ 979f7b053aSSeongJae Park int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id) 989f7b053aSSeongJae Park { 999f7b053aSSeongJae Park int err = 0; 1009f7b053aSSeongJae Park 1019f7b053aSSeongJae Park if (id >= NR_DAMON_OPS) 1029f7b053aSSeongJae Park return -EINVAL; 1039f7b053aSSeongJae Park 1049f7b053aSSeongJae Park mutex_lock(&damon_ops_lock); 105152e5617SSeongJae Park if (!__damon_is_registered_ops(id)) 1069f7b053aSSeongJae Park err = -EINVAL; 1079f7b053aSSeongJae Park else 1089f7b053aSSeongJae Park ctx->ops = damon_registered_ops[id]; 1099f7b053aSSeongJae Park mutex_unlock(&damon_ops_lock); 1109f7b053aSSeongJae Park return err; 1119f7b053aSSeongJae Park } 1129f7b053aSSeongJae Park 113f23b8eeeSSeongJae Park /* 114f23b8eeeSSeongJae Park * Construct a damon_region struct 115f23b8eeeSSeongJae Park * 116f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 117f23b8eeeSSeongJae Park */ 118f23b8eeeSSeongJae Park struct damon_region *damon_new_region(unsigned long start, unsigned long end) 119f23b8eeeSSeongJae Park { 120f23b8eeeSSeongJae Park struct damon_region *region; 121f23b8eeeSSeongJae Park 122f23b8eeeSSeongJae Park region = kmalloc(sizeof(*region), GFP_KERNEL); 123f23b8eeeSSeongJae Park if (!region) 124f23b8eeeSSeongJae Park return NULL; 125f23b8eeeSSeongJae Park 126f23b8eeeSSeongJae Park region->ar.start = start; 127f23b8eeeSSeongJae Park region->ar.end = end; 128f23b8eeeSSeongJae Park region->nr_accesses = 0; 129f23b8eeeSSeongJae Park INIT_LIST_HEAD(®ion->list); 130f23b8eeeSSeongJae Park 131fda504faSSeongJae Park region->age = 0; 132fda504faSSeongJae Park region->last_nr_accesses = 0; 133fda504faSSeongJae Park 134f23b8eeeSSeongJae Park return region; 135f23b8eeeSSeongJae Park } 136f23b8eeeSSeongJae Park 137f23b8eeeSSeongJae Park void damon_add_region(struct damon_region *r, struct damon_target *t) 138f23b8eeeSSeongJae Park { 139f23b8eeeSSeongJae Park list_add_tail(&r->list, &t->regions_list); 140b9a6ac4eSSeongJae Park t->nr_regions++; 141f23b8eeeSSeongJae Park } 142f23b8eeeSSeongJae Park 143b9a6ac4eSSeongJae Park static void damon_del_region(struct damon_region *r, struct damon_target *t) 144f23b8eeeSSeongJae Park { 145f23b8eeeSSeongJae Park list_del(&r->list); 146b9a6ac4eSSeongJae Park t->nr_regions--; 147f23b8eeeSSeongJae Park } 148f23b8eeeSSeongJae Park 149f23b8eeeSSeongJae Park static void damon_free_region(struct damon_region *r) 150f23b8eeeSSeongJae Park { 151f23b8eeeSSeongJae Park kfree(r); 152f23b8eeeSSeongJae Park } 153f23b8eeeSSeongJae Park 154b9a6ac4eSSeongJae Park void damon_destroy_region(struct damon_region *r, struct damon_target *t) 155f23b8eeeSSeongJae Park { 156b9a6ac4eSSeongJae Park damon_del_region(r, t); 157f23b8eeeSSeongJae Park damon_free_region(r); 158f23b8eeeSSeongJae Park } 159f23b8eeeSSeongJae Park 160d0723bc0SSeongJae Park /* 161d0723bc0SSeongJae Park * Check whether a region is intersecting an address range 162d0723bc0SSeongJae Park * 163d0723bc0SSeongJae Park * Returns true if it is. 164d0723bc0SSeongJae Park */ 165d0723bc0SSeongJae Park static bool damon_intersect(struct damon_region *r, 166d0723bc0SSeongJae Park struct damon_addr_range *re) 167d0723bc0SSeongJae Park { 168d0723bc0SSeongJae Park return !(r->ar.end <= re->start || re->end <= r->ar.start); 169d0723bc0SSeongJae Park } 170d0723bc0SSeongJae Park 171d0723bc0SSeongJae Park /* 172d0723bc0SSeongJae Park * damon_set_regions() - Set regions of a target for given address ranges. 173d0723bc0SSeongJae Park * @t: the given target. 174d0723bc0SSeongJae Park * @ranges: array of new monitoring target ranges. 175d0723bc0SSeongJae Park * @nr_ranges: length of @ranges. 176d0723bc0SSeongJae Park * 177d0723bc0SSeongJae Park * This function adds new regions to, or modify existing regions of a 178d0723bc0SSeongJae Park * monitoring target to fit in specific ranges. 179d0723bc0SSeongJae Park * 180d0723bc0SSeongJae Park * Return: 0 if success, or negative error code otherwise. 181d0723bc0SSeongJae Park */ 182d0723bc0SSeongJae Park int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, 183d0723bc0SSeongJae Park unsigned int nr_ranges) 184d0723bc0SSeongJae Park { 185d0723bc0SSeongJae Park struct damon_region *r, *next; 186d0723bc0SSeongJae Park unsigned int i; 187d0723bc0SSeongJae Park 188d0723bc0SSeongJae Park /* Remove regions which are not in the new ranges */ 189d0723bc0SSeongJae Park damon_for_each_region_safe(r, next, t) { 190d0723bc0SSeongJae Park for (i = 0; i < nr_ranges; i++) { 191d0723bc0SSeongJae Park if (damon_intersect(r, &ranges[i])) 192d0723bc0SSeongJae Park break; 193d0723bc0SSeongJae Park } 194d0723bc0SSeongJae Park if (i == nr_ranges) 195d0723bc0SSeongJae Park damon_destroy_region(r, t); 196d0723bc0SSeongJae Park } 197d0723bc0SSeongJae Park 198d0723bc0SSeongJae Park /* Add new regions or resize existing regions to fit in the ranges */ 199d0723bc0SSeongJae Park for (i = 0; i < nr_ranges; i++) { 200d0723bc0SSeongJae Park struct damon_region *first = NULL, *last, *newr; 201d0723bc0SSeongJae Park struct damon_addr_range *range; 202d0723bc0SSeongJae Park 203d0723bc0SSeongJae Park range = &ranges[i]; 204d0723bc0SSeongJae Park /* Get the first/last regions intersecting with the range */ 205d0723bc0SSeongJae Park damon_for_each_region(r, t) { 206d0723bc0SSeongJae Park if (damon_intersect(r, range)) { 207d0723bc0SSeongJae Park if (!first) 208d0723bc0SSeongJae Park first = r; 209d0723bc0SSeongJae Park last = r; 210d0723bc0SSeongJae Park } 211d0723bc0SSeongJae Park if (r->ar.start >= range->end) 212d0723bc0SSeongJae Park break; 213d0723bc0SSeongJae Park } 214d0723bc0SSeongJae Park if (!first) { 215d0723bc0SSeongJae Park /* no region intersects with this range */ 216d0723bc0SSeongJae Park newr = damon_new_region( 217d0723bc0SSeongJae Park ALIGN_DOWN(range->start, 218d0723bc0SSeongJae Park DAMON_MIN_REGION), 219d0723bc0SSeongJae Park ALIGN(range->end, DAMON_MIN_REGION)); 220d0723bc0SSeongJae Park if (!newr) 221d0723bc0SSeongJae Park return -ENOMEM; 222d0723bc0SSeongJae Park damon_insert_region(newr, damon_prev_region(r), r, t); 223d0723bc0SSeongJae Park } else { 224d0723bc0SSeongJae Park /* resize intersecting regions to fit in this range */ 225d0723bc0SSeongJae Park first->ar.start = ALIGN_DOWN(range->start, 226d0723bc0SSeongJae Park DAMON_MIN_REGION); 227d0723bc0SSeongJae Park last->ar.end = ALIGN(range->end, DAMON_MIN_REGION); 228d0723bc0SSeongJae Park } 229d0723bc0SSeongJae Park } 230d0723bc0SSeongJae Park return 0; 231d0723bc0SSeongJae Park } 232d0723bc0SSeongJae Park 2331f366e42SSeongJae Park struct damos *damon_new_scheme( 2341f366e42SSeongJae Park unsigned long min_sz_region, unsigned long max_sz_region, 2351f366e42SSeongJae Park unsigned int min_nr_accesses, unsigned int max_nr_accesses, 2361f366e42SSeongJae Park unsigned int min_age_region, unsigned int max_age_region, 237ee801b7dSSeongJae Park enum damos_action action, struct damos_quota *quota, 238ee801b7dSSeongJae Park struct damos_watermarks *wmarks) 2391f366e42SSeongJae Park { 2401f366e42SSeongJae Park struct damos *scheme; 2411f366e42SSeongJae Park 2421f366e42SSeongJae Park scheme = kmalloc(sizeof(*scheme), GFP_KERNEL); 2431f366e42SSeongJae Park if (!scheme) 2441f366e42SSeongJae Park return NULL; 2451f366e42SSeongJae Park scheme->min_sz_region = min_sz_region; 2461f366e42SSeongJae Park scheme->max_sz_region = max_sz_region; 2471f366e42SSeongJae Park scheme->min_nr_accesses = min_nr_accesses; 2481f366e42SSeongJae Park scheme->max_nr_accesses = max_nr_accesses; 2491f366e42SSeongJae Park scheme->min_age_region = min_age_region; 2501f366e42SSeongJae Park scheme->max_age_region = max_age_region; 2511f366e42SSeongJae Park scheme->action = action; 2520e92c2eeSSeongJae Park scheme->stat = (struct damos_stat){}; 2531f366e42SSeongJae Park INIT_LIST_HEAD(&scheme->list); 2541f366e42SSeongJae Park 2551cd24303SSeongJae Park scheme->quota.ms = quota->ms; 2562b8a248dSSeongJae Park scheme->quota.sz = quota->sz; 2572b8a248dSSeongJae Park scheme->quota.reset_interval = quota->reset_interval; 25838683e00SSeongJae Park scheme->quota.weight_sz = quota->weight_sz; 25938683e00SSeongJae Park scheme->quota.weight_nr_accesses = quota->weight_nr_accesses; 26038683e00SSeongJae Park scheme->quota.weight_age = quota->weight_age; 2611cd24303SSeongJae Park scheme->quota.total_charged_sz = 0; 2621cd24303SSeongJae Park scheme->quota.total_charged_ns = 0; 2631cd24303SSeongJae Park scheme->quota.esz = 0; 2642b8a248dSSeongJae Park scheme->quota.charged_sz = 0; 2652b8a248dSSeongJae Park scheme->quota.charged_from = 0; 26650585192SSeongJae Park scheme->quota.charge_target_from = NULL; 26750585192SSeongJae Park scheme->quota.charge_addr_from = 0; 2682b8a248dSSeongJae Park 269ee801b7dSSeongJae Park scheme->wmarks.metric = wmarks->metric; 270ee801b7dSSeongJae Park scheme->wmarks.interval = wmarks->interval; 271ee801b7dSSeongJae Park scheme->wmarks.high = wmarks->high; 272ee801b7dSSeongJae Park scheme->wmarks.mid = wmarks->mid; 273ee801b7dSSeongJae Park scheme->wmarks.low = wmarks->low; 274ee801b7dSSeongJae Park scheme->wmarks.activated = true; 275ee801b7dSSeongJae Park 2761f366e42SSeongJae Park return scheme; 2771f366e42SSeongJae Park } 2781f366e42SSeongJae Park 2791f366e42SSeongJae Park void damon_add_scheme(struct damon_ctx *ctx, struct damos *s) 2801f366e42SSeongJae Park { 2811f366e42SSeongJae Park list_add_tail(&s->list, &ctx->schemes); 2821f366e42SSeongJae Park } 2831f366e42SSeongJae Park 2841f366e42SSeongJae Park static void damon_del_scheme(struct damos *s) 2851f366e42SSeongJae Park { 2861f366e42SSeongJae Park list_del(&s->list); 2871f366e42SSeongJae Park } 2881f366e42SSeongJae Park 2891f366e42SSeongJae Park static void damon_free_scheme(struct damos *s) 2901f366e42SSeongJae Park { 2911f366e42SSeongJae Park kfree(s); 2921f366e42SSeongJae Park } 2931f366e42SSeongJae Park 2941f366e42SSeongJae Park void damon_destroy_scheme(struct damos *s) 2951f366e42SSeongJae Park { 2961f366e42SSeongJae Park damon_del_scheme(s); 2971f366e42SSeongJae Park damon_free_scheme(s); 2981f366e42SSeongJae Park } 2991f366e42SSeongJae Park 300f23b8eeeSSeongJae Park /* 301f23b8eeeSSeongJae Park * Construct a damon_target struct 302f23b8eeeSSeongJae Park * 303f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 304f23b8eeeSSeongJae Park */ 3051971bd63SSeongJae Park struct damon_target *damon_new_target(void) 306f23b8eeeSSeongJae Park { 307f23b8eeeSSeongJae Park struct damon_target *t; 308f23b8eeeSSeongJae Park 309f23b8eeeSSeongJae Park t = kmalloc(sizeof(*t), GFP_KERNEL); 310f23b8eeeSSeongJae Park if (!t) 311f23b8eeeSSeongJae Park return NULL; 312f23b8eeeSSeongJae Park 3131971bd63SSeongJae Park t->pid = NULL; 314b9a6ac4eSSeongJae Park t->nr_regions = 0; 315f23b8eeeSSeongJae Park INIT_LIST_HEAD(&t->regions_list); 316*b1f44cdaSSeongJae Park INIT_LIST_HEAD(&t->list); 317f23b8eeeSSeongJae Park 318f23b8eeeSSeongJae Park return t; 319f23b8eeeSSeongJae Park } 320f23b8eeeSSeongJae Park 321f23b8eeeSSeongJae Park void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) 322f23b8eeeSSeongJae Park { 323b9a6ac4eSSeongJae Park list_add_tail(&t->list, &ctx->adaptive_targets); 324f23b8eeeSSeongJae Park } 325f23b8eeeSSeongJae Park 326b5ca3e83SXin Hao bool damon_targets_empty(struct damon_ctx *ctx) 327b5ca3e83SXin Hao { 328b5ca3e83SXin Hao return list_empty(&ctx->adaptive_targets); 329b5ca3e83SXin Hao } 330b5ca3e83SXin Hao 331f23b8eeeSSeongJae Park static void damon_del_target(struct damon_target *t) 332f23b8eeeSSeongJae Park { 333f23b8eeeSSeongJae Park list_del(&t->list); 334f23b8eeeSSeongJae Park } 335f23b8eeeSSeongJae Park 336f23b8eeeSSeongJae Park void damon_free_target(struct damon_target *t) 337f23b8eeeSSeongJae Park { 338f23b8eeeSSeongJae Park struct damon_region *r, *next; 339f23b8eeeSSeongJae Park 340f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 341f23b8eeeSSeongJae Park damon_free_region(r); 342f23b8eeeSSeongJae Park kfree(t); 343f23b8eeeSSeongJae Park } 344f23b8eeeSSeongJae Park 345f23b8eeeSSeongJae Park void damon_destroy_target(struct damon_target *t) 346f23b8eeeSSeongJae Park { 347f23b8eeeSSeongJae Park damon_del_target(t); 348f23b8eeeSSeongJae Park damon_free_target(t); 349f23b8eeeSSeongJae Park } 350f23b8eeeSSeongJae Park 351b9a6ac4eSSeongJae Park unsigned int damon_nr_regions(struct damon_target *t) 352b9a6ac4eSSeongJae Park { 353b9a6ac4eSSeongJae Park return t->nr_regions; 354b9a6ac4eSSeongJae Park } 355b9a6ac4eSSeongJae Park 3562224d848SSeongJae Park struct damon_ctx *damon_new_ctx(void) 3572224d848SSeongJae Park { 3582224d848SSeongJae Park struct damon_ctx *ctx; 3592224d848SSeongJae Park 3602224d848SSeongJae Park ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 3612224d848SSeongJae Park if (!ctx) 3622224d848SSeongJae Park return NULL; 3632224d848SSeongJae Park 3642224d848SSeongJae Park ctx->sample_interval = 5 * 1000; 3652224d848SSeongJae Park ctx->aggr_interval = 100 * 1000; 366f7d911c3SSeongJae Park ctx->ops_update_interval = 60 * 1000 * 1000; 3672224d848SSeongJae Park 3682224d848SSeongJae Park ktime_get_coarse_ts64(&ctx->last_aggregation); 369f7d911c3SSeongJae Park ctx->last_ops_update = ctx->last_aggregation; 3702224d848SSeongJae Park 3712224d848SSeongJae Park mutex_init(&ctx->kdamond_lock); 3722224d848SSeongJae Park 373b9a6ac4eSSeongJae Park ctx->min_nr_regions = 10; 374b9a6ac4eSSeongJae Park ctx->max_nr_regions = 1000; 375b9a6ac4eSSeongJae Park 376b9a6ac4eSSeongJae Park INIT_LIST_HEAD(&ctx->adaptive_targets); 3771f366e42SSeongJae Park INIT_LIST_HEAD(&ctx->schemes); 3782224d848SSeongJae Park 3792224d848SSeongJae Park return ctx; 3802224d848SSeongJae Park } 3812224d848SSeongJae Park 382f23b8eeeSSeongJae Park static void damon_destroy_targets(struct damon_ctx *ctx) 383f23b8eeeSSeongJae Park { 384f23b8eeeSSeongJae Park struct damon_target *t, *next_t; 385f23b8eeeSSeongJae Park 386f7d911c3SSeongJae Park if (ctx->ops.cleanup) { 387f7d911c3SSeongJae Park ctx->ops.cleanup(ctx); 388f23b8eeeSSeongJae Park return; 389f23b8eeeSSeongJae Park } 390f23b8eeeSSeongJae Park 391f23b8eeeSSeongJae Park damon_for_each_target_safe(t, next_t, ctx) 392f23b8eeeSSeongJae Park damon_destroy_target(t); 393f23b8eeeSSeongJae Park } 394f23b8eeeSSeongJae Park 3952224d848SSeongJae Park void damon_destroy_ctx(struct damon_ctx *ctx) 3962224d848SSeongJae Park { 3971f366e42SSeongJae Park struct damos *s, *next_s; 3981f366e42SSeongJae Park 399f23b8eeeSSeongJae Park damon_destroy_targets(ctx); 4001f366e42SSeongJae Park 4011f366e42SSeongJae Park damon_for_each_scheme_safe(s, next_s, ctx) 4021f366e42SSeongJae Park damon_destroy_scheme(s); 4031f366e42SSeongJae Park 4042224d848SSeongJae Park kfree(ctx); 4052224d848SSeongJae Park } 4062224d848SSeongJae Park 4072224d848SSeongJae Park /** 4082224d848SSeongJae Park * damon_set_attrs() - Set attributes for the monitoring. 4092224d848SSeongJae Park * @ctx: monitoring context 4102224d848SSeongJae Park * @sample_int: time interval between samplings 4112224d848SSeongJae Park * @aggr_int: time interval between aggregations 412f7d911c3SSeongJae Park * @ops_upd_int: time interval between monitoring operations updates 413b9a6ac4eSSeongJae Park * @min_nr_reg: minimal number of regions 414b9a6ac4eSSeongJae Park * @max_nr_reg: maximum number of regions 4152224d848SSeongJae Park * 4162224d848SSeongJae Park * This function should not be called while the kdamond is running. 4172224d848SSeongJae Park * Every time interval is in micro-seconds. 4182224d848SSeongJae Park * 4192224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4202224d848SSeongJae Park */ 4212224d848SSeongJae Park int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, 422f7d911c3SSeongJae Park unsigned long aggr_int, unsigned long ops_upd_int, 423b9a6ac4eSSeongJae Park unsigned long min_nr_reg, unsigned long max_nr_reg) 4242224d848SSeongJae Park { 4251afaf5cbSSeongJae Park if (min_nr_reg < 3) 426b9a6ac4eSSeongJae Park return -EINVAL; 4271afaf5cbSSeongJae Park if (min_nr_reg > max_nr_reg) 428b9a6ac4eSSeongJae Park return -EINVAL; 429b9a6ac4eSSeongJae Park 4302224d848SSeongJae Park ctx->sample_interval = sample_int; 4312224d848SSeongJae Park ctx->aggr_interval = aggr_int; 432f7d911c3SSeongJae Park ctx->ops_update_interval = ops_upd_int; 433b9a6ac4eSSeongJae Park ctx->min_nr_regions = min_nr_reg; 434b9a6ac4eSSeongJae Park ctx->max_nr_regions = max_nr_reg; 4352224d848SSeongJae Park 4362224d848SSeongJae Park return 0; 4372224d848SSeongJae Park } 4382224d848SSeongJae Park 4394bc05954SSeongJae Park /** 4401f366e42SSeongJae Park * damon_set_schemes() - Set data access monitoring based operation schemes. 4411f366e42SSeongJae Park * @ctx: monitoring context 4421f366e42SSeongJae Park * @schemes: array of the schemes 4431f366e42SSeongJae Park * @nr_schemes: number of entries in @schemes 4441f366e42SSeongJae Park * 4451f366e42SSeongJae Park * This function should not be called while the kdamond of the context is 4461f366e42SSeongJae Park * running. 4471f366e42SSeongJae Park * 4481f366e42SSeongJae Park * Return: 0 if success, or negative error code otherwise. 4491f366e42SSeongJae Park */ 4501f366e42SSeongJae Park int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, 4511f366e42SSeongJae Park ssize_t nr_schemes) 4521f366e42SSeongJae Park { 4531f366e42SSeongJae Park struct damos *s, *next; 4541f366e42SSeongJae Park ssize_t i; 4551f366e42SSeongJae Park 4561f366e42SSeongJae Park damon_for_each_scheme_safe(s, next, ctx) 4571f366e42SSeongJae Park damon_destroy_scheme(s); 4581f366e42SSeongJae Park for (i = 0; i < nr_schemes; i++) 4591f366e42SSeongJae Park damon_add_scheme(ctx, schemes[i]); 4601f366e42SSeongJae Park return 0; 4611f366e42SSeongJae Park } 4621f366e42SSeongJae Park 4631f366e42SSeongJae Park /** 4644bc05954SSeongJae Park * damon_nr_running_ctxs() - Return number of currently running contexts. 4654bc05954SSeongJae Park */ 4664bc05954SSeongJae Park int damon_nr_running_ctxs(void) 4674bc05954SSeongJae Park { 4684bc05954SSeongJae Park int nr_ctxs; 4694bc05954SSeongJae Park 4704bc05954SSeongJae Park mutex_lock(&damon_lock); 4714bc05954SSeongJae Park nr_ctxs = nr_running_ctxs; 4724bc05954SSeongJae Park mutex_unlock(&damon_lock); 4734bc05954SSeongJae Park 4744bc05954SSeongJae Park return nr_ctxs; 4754bc05954SSeongJae Park } 4764bc05954SSeongJae Park 477b9a6ac4eSSeongJae Park /* Returns the size upper limit for each monitoring region */ 478b9a6ac4eSSeongJae Park static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) 479b9a6ac4eSSeongJae Park { 480b9a6ac4eSSeongJae Park struct damon_target *t; 481b9a6ac4eSSeongJae Park struct damon_region *r; 482b9a6ac4eSSeongJae Park unsigned long sz = 0; 483b9a6ac4eSSeongJae Park 484b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) { 485b9a6ac4eSSeongJae Park damon_for_each_region(r, t) 486b9a6ac4eSSeongJae Park sz += r->ar.end - r->ar.start; 487b9a6ac4eSSeongJae Park } 488b9a6ac4eSSeongJae Park 489b9a6ac4eSSeongJae Park if (ctx->min_nr_regions) 490b9a6ac4eSSeongJae Park sz /= ctx->min_nr_regions; 491b9a6ac4eSSeongJae Park if (sz < DAMON_MIN_REGION) 492b9a6ac4eSSeongJae Park sz = DAMON_MIN_REGION; 493b9a6ac4eSSeongJae Park 494b9a6ac4eSSeongJae Park return sz; 495b9a6ac4eSSeongJae Park } 496b9a6ac4eSSeongJae Park 4972224d848SSeongJae Park static int kdamond_fn(void *data); 4982224d848SSeongJae Park 4992224d848SSeongJae Park /* 5002224d848SSeongJae Park * __damon_start() - Starts monitoring with given context. 5012224d848SSeongJae Park * @ctx: monitoring context 5022224d848SSeongJae Park * 5032224d848SSeongJae Park * This function should be called while damon_lock is hold. 5042224d848SSeongJae Park * 5052224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 5062224d848SSeongJae Park */ 5072224d848SSeongJae Park static int __damon_start(struct damon_ctx *ctx) 5082224d848SSeongJae Park { 5092224d848SSeongJae Park int err = -EBUSY; 5102224d848SSeongJae Park 5112224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 5122224d848SSeongJae Park if (!ctx->kdamond) { 5132224d848SSeongJae Park err = 0; 5142224d848SSeongJae Park ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d", 5152224d848SSeongJae Park nr_running_ctxs); 5162224d848SSeongJae Park if (IS_ERR(ctx->kdamond)) { 5172224d848SSeongJae Park err = PTR_ERR(ctx->kdamond); 5187ec1992bSColin Ian King ctx->kdamond = NULL; 5192224d848SSeongJae Park } 5202224d848SSeongJae Park } 5212224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 5222224d848SSeongJae Park 5232224d848SSeongJae Park return err; 5242224d848SSeongJae Park } 5252224d848SSeongJae Park 5262224d848SSeongJae Park /** 5272224d848SSeongJae Park * damon_start() - Starts the monitorings for a given group of contexts. 5282224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to start monitoring 5292224d848SSeongJae Park * @nr_ctxs: size of @ctxs 5308b9b0d33SSeongJae Park * @exclusive: exclusiveness of this contexts group 5312224d848SSeongJae Park * 5322224d848SSeongJae Park * This function starts a group of monitoring threads for a group of monitoring 5332224d848SSeongJae Park * contexts. One thread per each context is created and run in parallel. The 5348b9b0d33SSeongJae Park * caller should handle synchronization between the threads by itself. If 5358b9b0d33SSeongJae Park * @exclusive is true and a group of threads that created by other 5368b9b0d33SSeongJae Park * 'damon_start()' call is currently running, this function does nothing but 5378b9b0d33SSeongJae Park * returns -EBUSY. 5382224d848SSeongJae Park * 5392224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 5402224d848SSeongJae Park */ 5418b9b0d33SSeongJae Park int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) 5422224d848SSeongJae Park { 5432224d848SSeongJae Park int i; 5442224d848SSeongJae Park int err = 0; 5452224d848SSeongJae Park 5462224d848SSeongJae Park mutex_lock(&damon_lock); 5478b9b0d33SSeongJae Park if ((exclusive && nr_running_ctxs) || 5488b9b0d33SSeongJae Park (!exclusive && running_exclusive_ctxs)) { 5492224d848SSeongJae Park mutex_unlock(&damon_lock); 5502224d848SSeongJae Park return -EBUSY; 5512224d848SSeongJae Park } 5522224d848SSeongJae Park 5532224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 5542224d848SSeongJae Park err = __damon_start(ctxs[i]); 5552224d848SSeongJae Park if (err) 5562224d848SSeongJae Park break; 5572224d848SSeongJae Park nr_running_ctxs++; 5582224d848SSeongJae Park } 5598b9b0d33SSeongJae Park if (exclusive && nr_running_ctxs) 5608b9b0d33SSeongJae Park running_exclusive_ctxs = true; 5612224d848SSeongJae Park mutex_unlock(&damon_lock); 5622224d848SSeongJae Park 5632224d848SSeongJae Park return err; 5642224d848SSeongJae Park } 5652224d848SSeongJae Park 5662224d848SSeongJae Park /* 5678b9b0d33SSeongJae Park * __damon_stop() - Stops monitoring of a given context. 5682224d848SSeongJae Park * @ctx: monitoring context 5692224d848SSeongJae Park * 5702224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 5712224d848SSeongJae Park */ 5722224d848SSeongJae Park static int __damon_stop(struct damon_ctx *ctx) 5732224d848SSeongJae Park { 5740f91d133SChangbin Du struct task_struct *tsk; 5750f91d133SChangbin Du 5762224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 5770f91d133SChangbin Du tsk = ctx->kdamond; 5780f91d133SChangbin Du if (tsk) { 5790f91d133SChangbin Du get_task_struct(tsk); 5802224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 5810f91d133SChangbin Du kthread_stop(tsk); 5820f91d133SChangbin Du put_task_struct(tsk); 5832224d848SSeongJae Park return 0; 5842224d848SSeongJae Park } 5852224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 5862224d848SSeongJae Park 5872224d848SSeongJae Park return -EPERM; 5882224d848SSeongJae Park } 5892224d848SSeongJae Park 5902224d848SSeongJae Park /** 5912224d848SSeongJae Park * damon_stop() - Stops the monitorings for a given group of contexts. 5922224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to stop monitoring 5932224d848SSeongJae Park * @nr_ctxs: size of @ctxs 5942224d848SSeongJae Park * 5952224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 5962224d848SSeongJae Park */ 5972224d848SSeongJae Park int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) 5982224d848SSeongJae Park { 5992224d848SSeongJae Park int i, err = 0; 6002224d848SSeongJae Park 6012224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 6022224d848SSeongJae Park /* nr_running_ctxs is decremented in kdamond_fn */ 6032224d848SSeongJae Park err = __damon_stop(ctxs[i]); 6042224d848SSeongJae Park if (err) 6058b9b0d33SSeongJae Park break; 6062224d848SSeongJae Park } 6072224d848SSeongJae Park return err; 6082224d848SSeongJae Park } 6092224d848SSeongJae Park 6102224d848SSeongJae Park /* 6112224d848SSeongJae Park * damon_check_reset_time_interval() - Check if a time interval is elapsed. 6122224d848SSeongJae Park * @baseline: the time to check whether the interval has elapsed since 6132224d848SSeongJae Park * @interval: the time interval (microseconds) 6142224d848SSeongJae Park * 6152224d848SSeongJae Park * See whether the given time interval has passed since the given baseline 6162224d848SSeongJae Park * time. If so, it also updates the baseline to current time for next check. 6172224d848SSeongJae Park * 6182224d848SSeongJae Park * Return: true if the time interval has passed, or false otherwise. 6192224d848SSeongJae Park */ 6202224d848SSeongJae Park static bool damon_check_reset_time_interval(struct timespec64 *baseline, 6212224d848SSeongJae Park unsigned long interval) 6222224d848SSeongJae Park { 6232224d848SSeongJae Park struct timespec64 now; 6242224d848SSeongJae Park 6252224d848SSeongJae Park ktime_get_coarse_ts64(&now); 6262224d848SSeongJae Park if ((timespec64_to_ns(&now) - timespec64_to_ns(baseline)) < 6272224d848SSeongJae Park interval * 1000) 6282224d848SSeongJae Park return false; 6292224d848SSeongJae Park *baseline = now; 6302224d848SSeongJae Park return true; 6312224d848SSeongJae Park } 6322224d848SSeongJae Park 6332224d848SSeongJae Park /* 6342224d848SSeongJae Park * Check whether it is time to flush the aggregated information 6352224d848SSeongJae Park */ 6362224d848SSeongJae Park static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) 6372224d848SSeongJae Park { 6382224d848SSeongJae Park return damon_check_reset_time_interval(&ctx->last_aggregation, 6392224d848SSeongJae Park ctx->aggr_interval); 6402224d848SSeongJae Park } 6412224d848SSeongJae Park 6422224d848SSeongJae Park /* 643f23b8eeeSSeongJae Park * Reset the aggregated monitoring results ('nr_accesses' of each region). 644f23b8eeeSSeongJae Park */ 645f23b8eeeSSeongJae Park static void kdamond_reset_aggregated(struct damon_ctx *c) 646f23b8eeeSSeongJae Park { 647f23b8eeeSSeongJae Park struct damon_target *t; 64876fd0285SSeongJae Park unsigned int ti = 0; /* target's index */ 649f23b8eeeSSeongJae Park 650f23b8eeeSSeongJae Park damon_for_each_target(t, c) { 651f23b8eeeSSeongJae Park struct damon_region *r; 652f23b8eeeSSeongJae Park 6532fcb9362SSeongJae Park damon_for_each_region(r, t) { 65476fd0285SSeongJae Park trace_damon_aggregated(t, ti, r, damon_nr_regions(t)); 655fda504faSSeongJae Park r->last_nr_accesses = r->nr_accesses; 656f23b8eeeSSeongJae Park r->nr_accesses = 0; 657f23b8eeeSSeongJae Park } 65876fd0285SSeongJae Park ti++; 659f23b8eeeSSeongJae Park } 6602fcb9362SSeongJae Park } 661f23b8eeeSSeongJae Park 6622b8a248dSSeongJae Park static void damon_split_region_at(struct damon_ctx *ctx, 6632b8a248dSSeongJae Park struct damon_target *t, struct damon_region *r, 6642b8a248dSSeongJae Park unsigned long sz_r); 6652b8a248dSSeongJae Park 66638683e00SSeongJae Park static bool __damos_valid_target(struct damon_region *r, struct damos *s) 66738683e00SSeongJae Park { 66838683e00SSeongJae Park unsigned long sz; 66938683e00SSeongJae Park 67038683e00SSeongJae Park sz = r->ar.end - r->ar.start; 67138683e00SSeongJae Park return s->min_sz_region <= sz && sz <= s->max_sz_region && 67238683e00SSeongJae Park s->min_nr_accesses <= r->nr_accesses && 67338683e00SSeongJae Park r->nr_accesses <= s->max_nr_accesses && 67438683e00SSeongJae Park s->min_age_region <= r->age && r->age <= s->max_age_region; 67538683e00SSeongJae Park } 67638683e00SSeongJae Park 67738683e00SSeongJae Park static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, 67838683e00SSeongJae Park struct damon_region *r, struct damos *s) 67938683e00SSeongJae Park { 68038683e00SSeongJae Park bool ret = __damos_valid_target(r, s); 68138683e00SSeongJae Park 682f7d911c3SSeongJae Park if (!ret || !s->quota.esz || !c->ops.get_scheme_score) 68338683e00SSeongJae Park return ret; 68438683e00SSeongJae Park 685f7d911c3SSeongJae Park return c->ops.get_scheme_score(c, t, r, s) >= s->quota.min_score; 68638683e00SSeongJae Park } 68738683e00SSeongJae Park 6881f366e42SSeongJae Park static void damon_do_apply_schemes(struct damon_ctx *c, 6891f366e42SSeongJae Park struct damon_target *t, 6901f366e42SSeongJae Park struct damon_region *r) 6911f366e42SSeongJae Park { 6921f366e42SSeongJae Park struct damos *s; 6931f366e42SSeongJae Park 6941f366e42SSeongJae Park damon_for_each_scheme(s, c) { 6952b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 6962b8a248dSSeongJae Park unsigned long sz = r->ar.end - r->ar.start; 6971cd24303SSeongJae Park struct timespec64 begin, end; 6980e92c2eeSSeongJae Park unsigned long sz_applied = 0; 6992b8a248dSSeongJae Park 700ee801b7dSSeongJae Park if (!s->wmarks.activated) 701ee801b7dSSeongJae Park continue; 702ee801b7dSSeongJae Park 7032b8a248dSSeongJae Park /* Check the quota */ 7041cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 7052b8a248dSSeongJae Park continue; 7062b8a248dSSeongJae Park 70750585192SSeongJae Park /* Skip previously charged regions */ 70850585192SSeongJae Park if (quota->charge_target_from) { 70950585192SSeongJae Park if (t != quota->charge_target_from) 71050585192SSeongJae Park continue; 71150585192SSeongJae Park if (r == damon_last_region(t)) { 71250585192SSeongJae Park quota->charge_target_from = NULL; 71350585192SSeongJae Park quota->charge_addr_from = 0; 71450585192SSeongJae Park continue; 71550585192SSeongJae Park } 71650585192SSeongJae Park if (quota->charge_addr_from && 71750585192SSeongJae Park r->ar.end <= quota->charge_addr_from) 71850585192SSeongJae Park continue; 71950585192SSeongJae Park 72050585192SSeongJae Park if (quota->charge_addr_from && r->ar.start < 72150585192SSeongJae Park quota->charge_addr_from) { 72250585192SSeongJae Park sz = ALIGN_DOWN(quota->charge_addr_from - 72350585192SSeongJae Park r->ar.start, DAMON_MIN_REGION); 72450585192SSeongJae Park if (!sz) { 72550585192SSeongJae Park if (r->ar.end - r->ar.start <= 72650585192SSeongJae Park DAMON_MIN_REGION) 72750585192SSeongJae Park continue; 72850585192SSeongJae Park sz = DAMON_MIN_REGION; 72950585192SSeongJae Park } 73050585192SSeongJae Park damon_split_region_at(c, t, r, sz); 73150585192SSeongJae Park r = damon_next_region(r); 73250585192SSeongJae Park sz = r->ar.end - r->ar.start; 73350585192SSeongJae Park } 73450585192SSeongJae Park quota->charge_target_from = NULL; 73550585192SSeongJae Park quota->charge_addr_from = 0; 73650585192SSeongJae Park } 73750585192SSeongJae Park 73838683e00SSeongJae Park if (!damos_valid_target(c, t, r, s)) 7391f366e42SSeongJae Park continue; 7402b8a248dSSeongJae Park 7412b8a248dSSeongJae Park /* Apply the scheme */ 742f7d911c3SSeongJae Park if (c->ops.apply_scheme) { 7431cd24303SSeongJae Park if (quota->esz && 7441cd24303SSeongJae Park quota->charged_sz + sz > quota->esz) { 7451cd24303SSeongJae Park sz = ALIGN_DOWN(quota->esz - quota->charged_sz, 7462b8a248dSSeongJae Park DAMON_MIN_REGION); 7472b8a248dSSeongJae Park if (!sz) 7482b8a248dSSeongJae Park goto update_stat; 7492b8a248dSSeongJae Park damon_split_region_at(c, t, r, sz); 7502b8a248dSSeongJae Park } 7511cd24303SSeongJae Park ktime_get_coarse_ts64(&begin); 752f7d911c3SSeongJae Park sz_applied = c->ops.apply_scheme(c, t, r, s); 7531cd24303SSeongJae Park ktime_get_coarse_ts64(&end); 7541cd24303SSeongJae Park quota->total_charged_ns += timespec64_to_ns(&end) - 7551cd24303SSeongJae Park timespec64_to_ns(&begin); 7562b8a248dSSeongJae Park quota->charged_sz += sz; 7571cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) { 75850585192SSeongJae Park quota->charge_target_from = t; 75950585192SSeongJae Park quota->charge_addr_from = r->ar.end + 1; 76050585192SSeongJae Park } 7612b8a248dSSeongJae Park } 7622f0b548cSSeongJae Park if (s->action != DAMOS_STAT) 7631f366e42SSeongJae Park r->age = 0; 7642b8a248dSSeongJae Park 7652b8a248dSSeongJae Park update_stat: 7660e92c2eeSSeongJae Park s->stat.nr_tried++; 7670e92c2eeSSeongJae Park s->stat.sz_tried += sz; 7680e92c2eeSSeongJae Park if (sz_applied) 7690e92c2eeSSeongJae Park s->stat.nr_applied++; 7700e92c2eeSSeongJae Park s->stat.sz_applied += sz_applied; 7711f366e42SSeongJae Park } 7721f366e42SSeongJae Park } 7731f366e42SSeongJae Park 7741cd24303SSeongJae Park /* Shouldn't be called if quota->ms and quota->sz are zero */ 7751cd24303SSeongJae Park static void damos_set_effective_quota(struct damos_quota *quota) 7761cd24303SSeongJae Park { 7771cd24303SSeongJae Park unsigned long throughput; 7781cd24303SSeongJae Park unsigned long esz; 7791cd24303SSeongJae Park 7801cd24303SSeongJae Park if (!quota->ms) { 7811cd24303SSeongJae Park quota->esz = quota->sz; 7821cd24303SSeongJae Park return; 7831cd24303SSeongJae Park } 7841cd24303SSeongJae Park 7851cd24303SSeongJae Park if (quota->total_charged_ns) 7861cd24303SSeongJae Park throughput = quota->total_charged_sz * 1000000 / 7871cd24303SSeongJae Park quota->total_charged_ns; 7881cd24303SSeongJae Park else 7891cd24303SSeongJae Park throughput = PAGE_SIZE * 1024; 7901cd24303SSeongJae Park esz = throughput * quota->ms; 7911cd24303SSeongJae Park 7921cd24303SSeongJae Park if (quota->sz && quota->sz < esz) 7931cd24303SSeongJae Park esz = quota->sz; 7941cd24303SSeongJae Park quota->esz = esz; 7951cd24303SSeongJae Park } 7961cd24303SSeongJae Park 7971f366e42SSeongJae Park static void kdamond_apply_schemes(struct damon_ctx *c) 7981f366e42SSeongJae Park { 7991f366e42SSeongJae Park struct damon_target *t; 8002b8a248dSSeongJae Park struct damon_region *r, *next_r; 8012b8a248dSSeongJae Park struct damos *s; 8022b8a248dSSeongJae Park 8032b8a248dSSeongJae Park damon_for_each_scheme(s, c) { 8042b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 80538683e00SSeongJae Park unsigned long cumulated_sz; 80638683e00SSeongJae Park unsigned int score, max_score = 0; 8072b8a248dSSeongJae Park 808ee801b7dSSeongJae Park if (!s->wmarks.activated) 809ee801b7dSSeongJae Park continue; 810ee801b7dSSeongJae Park 8111cd24303SSeongJae Park if (!quota->ms && !quota->sz) 8122b8a248dSSeongJae Park continue; 8132b8a248dSSeongJae Park 8142b8a248dSSeongJae Park /* New charge window starts */ 8152b8a248dSSeongJae Park if (time_after_eq(jiffies, quota->charged_from + 8162b8a248dSSeongJae Park msecs_to_jiffies( 8172b8a248dSSeongJae Park quota->reset_interval))) { 8186268eac3SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 8196268eac3SSeongJae Park s->stat.qt_exceeds++; 8201cd24303SSeongJae Park quota->total_charged_sz += quota->charged_sz; 8212b8a248dSSeongJae Park quota->charged_from = jiffies; 8222b8a248dSSeongJae Park quota->charged_sz = 0; 8231cd24303SSeongJae Park damos_set_effective_quota(quota); 8242b8a248dSSeongJae Park } 82538683e00SSeongJae Park 826f7d911c3SSeongJae Park if (!c->ops.get_scheme_score) 82738683e00SSeongJae Park continue; 82838683e00SSeongJae Park 82938683e00SSeongJae Park /* Fill up the score histogram */ 83038683e00SSeongJae Park memset(quota->histogram, 0, sizeof(quota->histogram)); 83138683e00SSeongJae Park damon_for_each_target(t, c) { 83238683e00SSeongJae Park damon_for_each_region(r, t) { 83338683e00SSeongJae Park if (!__damos_valid_target(r, s)) 83438683e00SSeongJae Park continue; 835f7d911c3SSeongJae Park score = c->ops.get_scheme_score( 83638683e00SSeongJae Park c, t, r, s); 83738683e00SSeongJae Park quota->histogram[score] += 83838683e00SSeongJae Park r->ar.end - r->ar.start; 83938683e00SSeongJae Park if (score > max_score) 84038683e00SSeongJae Park max_score = score; 84138683e00SSeongJae Park } 84238683e00SSeongJae Park } 84338683e00SSeongJae Park 84438683e00SSeongJae Park /* Set the min score limit */ 84538683e00SSeongJae Park for (cumulated_sz = 0, score = max_score; ; score--) { 84638683e00SSeongJae Park cumulated_sz += quota->histogram[score]; 84738683e00SSeongJae Park if (cumulated_sz >= quota->esz || !score) 84838683e00SSeongJae Park break; 84938683e00SSeongJae Park } 85038683e00SSeongJae Park quota->min_score = score; 8512b8a248dSSeongJae Park } 8521f366e42SSeongJae Park 8531f366e42SSeongJae Park damon_for_each_target(t, c) { 8542b8a248dSSeongJae Park damon_for_each_region_safe(r, next_r, t) 8551f366e42SSeongJae Park damon_do_apply_schemes(c, t, r); 8561f366e42SSeongJae Park } 8571f366e42SSeongJae Park } 8581f366e42SSeongJae Park 85988f86dcfSSeongJae Park static inline unsigned long sz_damon_region(struct damon_region *r) 86088f86dcfSSeongJae Park { 86188f86dcfSSeongJae Park return r->ar.end - r->ar.start; 86288f86dcfSSeongJae Park } 863b9a6ac4eSSeongJae Park 864b9a6ac4eSSeongJae Park /* 865b9a6ac4eSSeongJae Park * Merge two adjacent regions into one region 866b9a6ac4eSSeongJae Park */ 867b9a6ac4eSSeongJae Park static void damon_merge_two_regions(struct damon_target *t, 868b9a6ac4eSSeongJae Park struct damon_region *l, struct damon_region *r) 869b9a6ac4eSSeongJae Park { 870b9a6ac4eSSeongJae Park unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r); 871b9a6ac4eSSeongJae Park 872b9a6ac4eSSeongJae Park l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / 873b9a6ac4eSSeongJae Park (sz_l + sz_r); 874fda504faSSeongJae Park l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); 875b9a6ac4eSSeongJae Park l->ar.end = r->ar.end; 876b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 877b9a6ac4eSSeongJae Park } 878b9a6ac4eSSeongJae Park 879b9a6ac4eSSeongJae Park /* 880b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 881b9a6ac4eSSeongJae Park * 882b9a6ac4eSSeongJae Park * t target affected by this merge operation 883b9a6ac4eSSeongJae Park * thres '->nr_accesses' diff threshold for the merge 884b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 885b9a6ac4eSSeongJae Park */ 886b9a6ac4eSSeongJae Park static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, 887b9a6ac4eSSeongJae Park unsigned long sz_limit) 888b9a6ac4eSSeongJae Park { 889b9a6ac4eSSeongJae Park struct damon_region *r, *prev = NULL, *next; 890b9a6ac4eSSeongJae Park 891b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 892d720bbbdSXin Hao if (abs(r->nr_accesses - r->last_nr_accesses) > thres) 893fda504faSSeongJae Park r->age = 0; 894fda504faSSeongJae Park else 895fda504faSSeongJae Park r->age++; 896fda504faSSeongJae Park 897b9a6ac4eSSeongJae Park if (prev && prev->ar.end == r->ar.start && 898d720bbbdSXin Hao abs(prev->nr_accesses - r->nr_accesses) <= thres && 899b9a6ac4eSSeongJae Park sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) 900b9a6ac4eSSeongJae Park damon_merge_two_regions(t, prev, r); 901b9a6ac4eSSeongJae Park else 902b9a6ac4eSSeongJae Park prev = r; 903b9a6ac4eSSeongJae Park } 904b9a6ac4eSSeongJae Park } 905b9a6ac4eSSeongJae Park 906b9a6ac4eSSeongJae Park /* 907b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 908b9a6ac4eSSeongJae Park * 909b9a6ac4eSSeongJae Park * threshold '->nr_accesses' diff threshold for the merge 910b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 911b9a6ac4eSSeongJae Park * 912b9a6ac4eSSeongJae Park * This function merges monitoring target regions which are adjacent and their 913b9a6ac4eSSeongJae Park * access frequencies are similar. This is for minimizing the monitoring 914b9a6ac4eSSeongJae Park * overhead under the dynamically changeable access pattern. If a merge was 915b9a6ac4eSSeongJae Park * unnecessarily made, later 'kdamond_split_regions()' will revert it. 916b9a6ac4eSSeongJae Park */ 917b9a6ac4eSSeongJae Park static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, 918b9a6ac4eSSeongJae Park unsigned long sz_limit) 919b9a6ac4eSSeongJae Park { 920b9a6ac4eSSeongJae Park struct damon_target *t; 921b9a6ac4eSSeongJae Park 922b9a6ac4eSSeongJae Park damon_for_each_target(t, c) 923b9a6ac4eSSeongJae Park damon_merge_regions_of(t, threshold, sz_limit); 924b9a6ac4eSSeongJae Park } 925b9a6ac4eSSeongJae Park 926b9a6ac4eSSeongJae Park /* 927b9a6ac4eSSeongJae Park * Split a region in two 928b9a6ac4eSSeongJae Park * 929b9a6ac4eSSeongJae Park * r the region to be split 930b9a6ac4eSSeongJae Park * sz_r size of the first sub-region that will be made 931b9a6ac4eSSeongJae Park */ 932b9a6ac4eSSeongJae Park static void damon_split_region_at(struct damon_ctx *ctx, 933b9a6ac4eSSeongJae Park struct damon_target *t, struct damon_region *r, 934b9a6ac4eSSeongJae Park unsigned long sz_r) 935b9a6ac4eSSeongJae Park { 936b9a6ac4eSSeongJae Park struct damon_region *new; 937b9a6ac4eSSeongJae Park 938b9a6ac4eSSeongJae Park new = damon_new_region(r->ar.start + sz_r, r->ar.end); 939b9a6ac4eSSeongJae Park if (!new) 940b9a6ac4eSSeongJae Park return; 941b9a6ac4eSSeongJae Park 942b9a6ac4eSSeongJae Park r->ar.end = new->ar.start; 943b9a6ac4eSSeongJae Park 944fda504faSSeongJae Park new->age = r->age; 945fda504faSSeongJae Park new->last_nr_accesses = r->last_nr_accesses; 946fda504faSSeongJae Park 947b9a6ac4eSSeongJae Park damon_insert_region(new, r, damon_next_region(r), t); 948b9a6ac4eSSeongJae Park } 949b9a6ac4eSSeongJae Park 950b9a6ac4eSSeongJae Park /* Split every region in the given target into 'nr_subs' regions */ 951b9a6ac4eSSeongJae Park static void damon_split_regions_of(struct damon_ctx *ctx, 952b9a6ac4eSSeongJae Park struct damon_target *t, int nr_subs) 953b9a6ac4eSSeongJae Park { 954b9a6ac4eSSeongJae Park struct damon_region *r, *next; 955b9a6ac4eSSeongJae Park unsigned long sz_region, sz_sub = 0; 956b9a6ac4eSSeongJae Park int i; 957b9a6ac4eSSeongJae Park 958b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 959b9a6ac4eSSeongJae Park sz_region = r->ar.end - r->ar.start; 960b9a6ac4eSSeongJae Park 961b9a6ac4eSSeongJae Park for (i = 0; i < nr_subs - 1 && 962b9a6ac4eSSeongJae Park sz_region > 2 * DAMON_MIN_REGION; i++) { 963b9a6ac4eSSeongJae Park /* 964b9a6ac4eSSeongJae Park * Randomly select size of left sub-region to be at 965b9a6ac4eSSeongJae Park * least 10 percent and at most 90% of original region 966b9a6ac4eSSeongJae Park */ 967b9a6ac4eSSeongJae Park sz_sub = ALIGN_DOWN(damon_rand(1, 10) * 968b9a6ac4eSSeongJae Park sz_region / 10, DAMON_MIN_REGION); 969b9a6ac4eSSeongJae Park /* Do not allow blank region */ 970b9a6ac4eSSeongJae Park if (sz_sub == 0 || sz_sub >= sz_region) 971b9a6ac4eSSeongJae Park continue; 972b9a6ac4eSSeongJae Park 973b9a6ac4eSSeongJae Park damon_split_region_at(ctx, t, r, sz_sub); 974b9a6ac4eSSeongJae Park sz_region = sz_sub; 975b9a6ac4eSSeongJae Park } 976b9a6ac4eSSeongJae Park } 977b9a6ac4eSSeongJae Park } 978b9a6ac4eSSeongJae Park 979b9a6ac4eSSeongJae Park /* 980b9a6ac4eSSeongJae Park * Split every target region into randomly-sized small regions 981b9a6ac4eSSeongJae Park * 982b9a6ac4eSSeongJae Park * This function splits every target region into random-sized small regions if 983b9a6ac4eSSeongJae Park * current total number of the regions is equal or smaller than half of the 984b9a6ac4eSSeongJae Park * user-specified maximum number of regions. This is for maximizing the 985b9a6ac4eSSeongJae Park * monitoring accuracy under the dynamically changeable access patterns. If a 986b9a6ac4eSSeongJae Park * split was unnecessarily made, later 'kdamond_merge_regions()' will revert 987b9a6ac4eSSeongJae Park * it. 988b9a6ac4eSSeongJae Park */ 989b9a6ac4eSSeongJae Park static void kdamond_split_regions(struct damon_ctx *ctx) 990b9a6ac4eSSeongJae Park { 991b9a6ac4eSSeongJae Park struct damon_target *t; 992b9a6ac4eSSeongJae Park unsigned int nr_regions = 0; 993b9a6ac4eSSeongJae Park static unsigned int last_nr_regions; 994b9a6ac4eSSeongJae Park int nr_subregions = 2; 995b9a6ac4eSSeongJae Park 996b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 997b9a6ac4eSSeongJae Park nr_regions += damon_nr_regions(t); 998b9a6ac4eSSeongJae Park 999b9a6ac4eSSeongJae Park if (nr_regions > ctx->max_nr_regions / 2) 1000b9a6ac4eSSeongJae Park return; 1001b9a6ac4eSSeongJae Park 1002b9a6ac4eSSeongJae Park /* Maybe the middle of the region has different access frequency */ 1003b9a6ac4eSSeongJae Park if (last_nr_regions == nr_regions && 1004b9a6ac4eSSeongJae Park nr_regions < ctx->max_nr_regions / 3) 1005b9a6ac4eSSeongJae Park nr_subregions = 3; 1006b9a6ac4eSSeongJae Park 1007b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 1008b9a6ac4eSSeongJae Park damon_split_regions_of(ctx, t, nr_subregions); 1009b9a6ac4eSSeongJae Park 1010b9a6ac4eSSeongJae Park last_nr_regions = nr_regions; 1011b9a6ac4eSSeongJae Park } 1012b9a6ac4eSSeongJae Park 1013f23b8eeeSSeongJae Park /* 1014f7d911c3SSeongJae Park * Check whether it is time to check and apply the operations-related data 1015f7d911c3SSeongJae Park * structures. 10162224d848SSeongJae Park * 10172224d848SSeongJae Park * Returns true if it is. 10182224d848SSeongJae Park */ 1019f7d911c3SSeongJae Park static bool kdamond_need_update_operations(struct damon_ctx *ctx) 10202224d848SSeongJae Park { 1021f7d911c3SSeongJae Park return damon_check_reset_time_interval(&ctx->last_ops_update, 1022f7d911c3SSeongJae Park ctx->ops_update_interval); 10232224d848SSeongJae Park } 10242224d848SSeongJae Park 10252224d848SSeongJae Park /* 10262224d848SSeongJae Park * Check whether current monitoring should be stopped 10272224d848SSeongJae Park * 10282224d848SSeongJae Park * The monitoring is stopped when either the user requested to stop, or all 10292224d848SSeongJae Park * monitoring targets are invalid. 10302224d848SSeongJae Park * 10312224d848SSeongJae Park * Returns true if need to stop current monitoring. 10322224d848SSeongJae Park */ 10332224d848SSeongJae Park static bool kdamond_need_stop(struct damon_ctx *ctx) 10342224d848SSeongJae Park { 1035f23b8eeeSSeongJae Park struct damon_target *t; 10362224d848SSeongJae Park 10370f91d133SChangbin Du if (kthread_should_stop()) 10382224d848SSeongJae Park return true; 10392224d848SSeongJae Park 1040f7d911c3SSeongJae Park if (!ctx->ops.target_valid) 10412224d848SSeongJae Park return false; 10422224d848SSeongJae Park 1043f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 1044f7d911c3SSeongJae Park if (ctx->ops.target_valid(t)) 1045f23b8eeeSSeongJae Park return false; 1046f23b8eeeSSeongJae Park } 1047f23b8eeeSSeongJae Park 1048f23b8eeeSSeongJae Park return true; 10492224d848SSeongJae Park } 10502224d848SSeongJae Park 1051ee801b7dSSeongJae Park static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric) 1052ee801b7dSSeongJae Park { 1053ee801b7dSSeongJae Park struct sysinfo i; 1054ee801b7dSSeongJae Park 1055ee801b7dSSeongJae Park switch (metric) { 1056ee801b7dSSeongJae Park case DAMOS_WMARK_FREE_MEM_RATE: 1057ee801b7dSSeongJae Park si_meminfo(&i); 1058ee801b7dSSeongJae Park return i.freeram * 1000 / i.totalram; 1059ee801b7dSSeongJae Park default: 1060ee801b7dSSeongJae Park break; 1061ee801b7dSSeongJae Park } 1062ee801b7dSSeongJae Park return -EINVAL; 1063ee801b7dSSeongJae Park } 1064ee801b7dSSeongJae Park 1065ee801b7dSSeongJae Park /* 1066ee801b7dSSeongJae Park * Returns zero if the scheme is active. Else, returns time to wait for next 1067ee801b7dSSeongJae Park * watermark check in micro-seconds. 1068ee801b7dSSeongJae Park */ 1069ee801b7dSSeongJae Park static unsigned long damos_wmark_wait_us(struct damos *scheme) 1070ee801b7dSSeongJae Park { 1071ee801b7dSSeongJae Park unsigned long metric; 1072ee801b7dSSeongJae Park 1073ee801b7dSSeongJae Park if (scheme->wmarks.metric == DAMOS_WMARK_NONE) 1074ee801b7dSSeongJae Park return 0; 1075ee801b7dSSeongJae Park 1076ee801b7dSSeongJae Park metric = damos_wmark_metric_value(scheme->wmarks.metric); 1077ee801b7dSSeongJae Park /* higher than high watermark or lower than low watermark */ 1078ee801b7dSSeongJae Park if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { 1079ee801b7dSSeongJae Park if (scheme->wmarks.activated) 108001078655SColin Ian King pr_debug("deactivate a scheme (%d) for %s wmark\n", 1081ee801b7dSSeongJae Park scheme->action, 1082ee801b7dSSeongJae Park metric > scheme->wmarks.high ? 1083ee801b7dSSeongJae Park "high" : "low"); 1084ee801b7dSSeongJae Park scheme->wmarks.activated = false; 1085ee801b7dSSeongJae Park return scheme->wmarks.interval; 1086ee801b7dSSeongJae Park } 1087ee801b7dSSeongJae Park 1088ee801b7dSSeongJae Park /* inactive and higher than middle watermark */ 1089ee801b7dSSeongJae Park if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) && 1090ee801b7dSSeongJae Park !scheme->wmarks.activated) 1091ee801b7dSSeongJae Park return scheme->wmarks.interval; 1092ee801b7dSSeongJae Park 1093ee801b7dSSeongJae Park if (!scheme->wmarks.activated) 1094ee801b7dSSeongJae Park pr_debug("activate a scheme (%d)\n", scheme->action); 1095ee801b7dSSeongJae Park scheme->wmarks.activated = true; 1096ee801b7dSSeongJae Park return 0; 1097ee801b7dSSeongJae Park } 1098ee801b7dSSeongJae Park 1099ee801b7dSSeongJae Park static void kdamond_usleep(unsigned long usecs) 1100ee801b7dSSeongJae Park { 11014de46a30SSeongJae Park /* See Documentation/timers/timers-howto.rst for the thresholds */ 11024de46a30SSeongJae Park if (usecs > 20 * USEC_PER_MSEC) 110370e92748SSeongJae Park schedule_timeout_idle(usecs_to_jiffies(usecs)); 1104ee801b7dSSeongJae Park else 110570e92748SSeongJae Park usleep_idle_range(usecs, usecs + 1); 1106ee801b7dSSeongJae Park } 1107ee801b7dSSeongJae Park 1108ee801b7dSSeongJae Park /* Returns negative error code if it's not activated but should return */ 1109ee801b7dSSeongJae Park static int kdamond_wait_activation(struct damon_ctx *ctx) 1110ee801b7dSSeongJae Park { 1111ee801b7dSSeongJae Park struct damos *s; 1112ee801b7dSSeongJae Park unsigned long wait_time; 1113ee801b7dSSeongJae Park unsigned long min_wait_time = 0; 111478049e94SJonghyeon Kim bool init_wait_time = false; 1115ee801b7dSSeongJae Park 1116ee801b7dSSeongJae Park while (!kdamond_need_stop(ctx)) { 1117ee801b7dSSeongJae Park damon_for_each_scheme(s, ctx) { 1118ee801b7dSSeongJae Park wait_time = damos_wmark_wait_us(s); 111978049e94SJonghyeon Kim if (!init_wait_time || wait_time < min_wait_time) { 112078049e94SJonghyeon Kim init_wait_time = true; 1121ee801b7dSSeongJae Park min_wait_time = wait_time; 1122ee801b7dSSeongJae Park } 112378049e94SJonghyeon Kim } 1124ee801b7dSSeongJae Park if (!min_wait_time) 1125ee801b7dSSeongJae Park return 0; 1126ee801b7dSSeongJae Park 1127ee801b7dSSeongJae Park kdamond_usleep(min_wait_time); 11286e74d2bfSSeongJae Park 11296e74d2bfSSeongJae Park if (ctx->callback.after_wmarks_check && 11306e74d2bfSSeongJae Park ctx->callback.after_wmarks_check(ctx)) 11316e74d2bfSSeongJae Park break; 1132ee801b7dSSeongJae Park } 1133ee801b7dSSeongJae Park return -EBUSY; 1134ee801b7dSSeongJae Park } 1135ee801b7dSSeongJae Park 11362224d848SSeongJae Park /* 11372224d848SSeongJae Park * The monitoring daemon that runs as a kernel thread 11382224d848SSeongJae Park */ 11392224d848SSeongJae Park static int kdamond_fn(void *data) 11402224d848SSeongJae Park { 1141cef4493fSYu Zhe struct damon_ctx *ctx = data; 1142f23b8eeeSSeongJae Park struct damon_target *t; 1143f23b8eeeSSeongJae Park struct damon_region *r, *next; 1144b9a6ac4eSSeongJae Park unsigned int max_nr_accesses = 0; 1145b9a6ac4eSSeongJae Park unsigned long sz_limit = 0; 11460f91d133SChangbin Du bool done = false; 11472224d848SSeongJae Park 114842e4cef5SChangbin Du pr_debug("kdamond (%d) starts\n", current->pid); 11492224d848SSeongJae Park 1150f7d911c3SSeongJae Park if (ctx->ops.init) 1151f7d911c3SSeongJae Park ctx->ops.init(ctx); 11522224d848SSeongJae Park if (ctx->callback.before_start && ctx->callback.before_start(ctx)) 11530f91d133SChangbin Du done = true; 11542224d848SSeongJae Park 1155b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 1156b9a6ac4eSSeongJae Park 11570f91d133SChangbin Du while (!kdamond_need_stop(ctx) && !done) { 11586e74d2bfSSeongJae Park if (kdamond_wait_activation(ctx)) { 11596e74d2bfSSeongJae Park done = true; 1160ee801b7dSSeongJae Park continue; 11616e74d2bfSSeongJae Park } 1162ee801b7dSSeongJae Park 1163f7d911c3SSeongJae Park if (ctx->ops.prepare_access_checks) 1164f7d911c3SSeongJae Park ctx->ops.prepare_access_checks(ctx); 11652224d848SSeongJae Park if (ctx->callback.after_sampling && 1166abacd635SSeongJae Park ctx->callback.after_sampling(ctx)) { 11670f91d133SChangbin Du done = true; 1168abacd635SSeongJae Park continue; 1169abacd635SSeongJae Park } 11702224d848SSeongJae Park 117170e92748SSeongJae Park kdamond_usleep(ctx->sample_interval); 11722224d848SSeongJae Park 1173f7d911c3SSeongJae Park if (ctx->ops.check_accesses) 1174f7d911c3SSeongJae Park max_nr_accesses = ctx->ops.check_accesses(ctx); 11752224d848SSeongJae Park 11762224d848SSeongJae Park if (kdamond_aggregate_interval_passed(ctx)) { 1177b9a6ac4eSSeongJae Park kdamond_merge_regions(ctx, 1178b9a6ac4eSSeongJae Park max_nr_accesses / 10, 1179b9a6ac4eSSeongJae Park sz_limit); 11802224d848SSeongJae Park if (ctx->callback.after_aggregation && 1181abacd635SSeongJae Park ctx->callback.after_aggregation(ctx)) { 11820f91d133SChangbin Du done = true; 1183abacd635SSeongJae Park continue; 1184abacd635SSeongJae Park } 11851f366e42SSeongJae Park kdamond_apply_schemes(ctx); 1186f23b8eeeSSeongJae Park kdamond_reset_aggregated(ctx); 1187b9a6ac4eSSeongJae Park kdamond_split_regions(ctx); 1188f7d911c3SSeongJae Park if (ctx->ops.reset_aggregated) 1189f7d911c3SSeongJae Park ctx->ops.reset_aggregated(ctx); 11902224d848SSeongJae Park } 11912224d848SSeongJae Park 1192f7d911c3SSeongJae Park if (kdamond_need_update_operations(ctx)) { 1193f7d911c3SSeongJae Park if (ctx->ops.update) 1194f7d911c3SSeongJae Park ctx->ops.update(ctx); 1195b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 11962224d848SSeongJae Park } 11972224d848SSeongJae Park } 1198f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 1199f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 1200b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 1201f23b8eeeSSeongJae Park } 12022224d848SSeongJae Park 12030f91d133SChangbin Du if (ctx->callback.before_terminate) 12040f91d133SChangbin Du ctx->callback.before_terminate(ctx); 1205f7d911c3SSeongJae Park if (ctx->ops.cleanup) 1206f7d911c3SSeongJae Park ctx->ops.cleanup(ctx); 12072224d848SSeongJae Park 120842e4cef5SChangbin Du pr_debug("kdamond (%d) finishes\n", current->pid); 12092224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 12102224d848SSeongJae Park ctx->kdamond = NULL; 12112224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 12122224d848SSeongJae Park 12132224d848SSeongJae Park mutex_lock(&damon_lock); 12142224d848SSeongJae Park nr_running_ctxs--; 12158b9b0d33SSeongJae Park if (!nr_running_ctxs && running_exclusive_ctxs) 12168b9b0d33SSeongJae Park running_exclusive_ctxs = false; 12172224d848SSeongJae Park mutex_unlock(&damon_lock); 12182224d848SSeongJae Park 12195f7fe2b9SChangbin Du return 0; 12202224d848SSeongJae Park } 122117ccae8bSSeongJae Park 122217ccae8bSSeongJae Park #include "core-test.h" 1223