12224d848SSeongJae Park // SPDX-License-Identifier: GPL-2.0 22224d848SSeongJae Park /* 32224d848SSeongJae Park * Data Access Monitor 42224d848SSeongJae Park * 52224d848SSeongJae Park * Author: SeongJae Park <sjpark@amazon.de> 62224d848SSeongJae Park */ 72224d848SSeongJae Park 82224d848SSeongJae Park #define pr_fmt(fmt) "damon: " fmt 92224d848SSeongJae Park 102224d848SSeongJae Park #include <linux/damon.h> 112224d848SSeongJae Park #include <linux/delay.h> 122224d848SSeongJae Park #include <linux/kthread.h> 13ee801b7dSSeongJae Park #include <linux/mm.h> 142224d848SSeongJae Park #include <linux/slab.h> 1538683e00SSeongJae Park #include <linux/string.h> 162224d848SSeongJae Park 172fcb9362SSeongJae Park #define CREATE_TRACE_POINTS 182fcb9362SSeongJae Park #include <trace/events/damon.h> 192fcb9362SSeongJae Park 2017ccae8bSSeongJae Park #ifdef CONFIG_DAMON_KUNIT_TEST 2117ccae8bSSeongJae Park #undef DAMON_MIN_REGION 2217ccae8bSSeongJae Park #define DAMON_MIN_REGION 1 2317ccae8bSSeongJae Park #endif 2417ccae8bSSeongJae Park 252224d848SSeongJae Park static DEFINE_MUTEX(damon_lock); 262224d848SSeongJae Park static int nr_running_ctxs; 27*8b9b0d33SSeongJae Park static bool running_exclusive_ctxs; 282224d848SSeongJae Park 299f7b053aSSeongJae Park static DEFINE_MUTEX(damon_ops_lock); 309f7b053aSSeongJae Park static struct damon_operations damon_registered_ops[NR_DAMON_OPS]; 319f7b053aSSeongJae Park 329f7b053aSSeongJae Park /* Should be called under damon_ops_lock with id smaller than NR_DAMON_OPS */ 339f7b053aSSeongJae Park static bool damon_registered_ops_id(enum damon_ops_id id) 349f7b053aSSeongJae Park { 359f7b053aSSeongJae Park struct damon_operations empty_ops = {}; 369f7b053aSSeongJae Park 379f7b053aSSeongJae Park if (!memcmp(&empty_ops, &damon_registered_ops[id], sizeof(empty_ops))) 389f7b053aSSeongJae Park return false; 399f7b053aSSeongJae Park return true; 409f7b053aSSeongJae Park } 419f7b053aSSeongJae Park 429f7b053aSSeongJae Park /** 439f7b053aSSeongJae Park * damon_register_ops() - Register a monitoring operations set to DAMON. 449f7b053aSSeongJae Park * @ops: monitoring operations set to register. 459f7b053aSSeongJae Park * 469f7b053aSSeongJae Park * This function registers a monitoring operations set of valid &struct 479f7b053aSSeongJae Park * damon_operations->id so that others can find and use them later. 489f7b053aSSeongJae Park * 499f7b053aSSeongJae Park * Return: 0 on success, negative error code otherwise. 509f7b053aSSeongJae Park */ 519f7b053aSSeongJae Park int damon_register_ops(struct damon_operations *ops) 529f7b053aSSeongJae Park { 539f7b053aSSeongJae Park int err = 0; 549f7b053aSSeongJae Park 559f7b053aSSeongJae Park if (ops->id >= NR_DAMON_OPS) 569f7b053aSSeongJae Park return -EINVAL; 579f7b053aSSeongJae Park mutex_lock(&damon_ops_lock); 589f7b053aSSeongJae Park /* Fail for already registered ops */ 599f7b053aSSeongJae Park if (damon_registered_ops_id(ops->id)) { 609f7b053aSSeongJae Park err = -EINVAL; 619f7b053aSSeongJae Park goto out; 629f7b053aSSeongJae Park } 639f7b053aSSeongJae Park damon_registered_ops[ops->id] = *ops; 649f7b053aSSeongJae Park out: 659f7b053aSSeongJae Park mutex_unlock(&damon_ops_lock); 669f7b053aSSeongJae Park return err; 679f7b053aSSeongJae Park } 689f7b053aSSeongJae Park 699f7b053aSSeongJae Park /** 709f7b053aSSeongJae Park * damon_select_ops() - Select a monitoring operations to use with the context. 719f7b053aSSeongJae Park * @ctx: monitoring context to use the operations. 729f7b053aSSeongJae Park * @id: id of the registered monitoring operations to select. 739f7b053aSSeongJae Park * 749f7b053aSSeongJae Park * This function finds registered monitoring operations set of @id and make 759f7b053aSSeongJae Park * @ctx to use it. 769f7b053aSSeongJae Park * 779f7b053aSSeongJae Park * Return: 0 on success, negative error code otherwise. 789f7b053aSSeongJae Park */ 799f7b053aSSeongJae Park int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id) 809f7b053aSSeongJae Park { 819f7b053aSSeongJae Park int err = 0; 829f7b053aSSeongJae Park 839f7b053aSSeongJae Park if (id >= NR_DAMON_OPS) 849f7b053aSSeongJae Park return -EINVAL; 859f7b053aSSeongJae Park 869f7b053aSSeongJae Park mutex_lock(&damon_ops_lock); 879f7b053aSSeongJae Park if (!damon_registered_ops_id(id)) 889f7b053aSSeongJae Park err = -EINVAL; 899f7b053aSSeongJae Park else 909f7b053aSSeongJae Park ctx->ops = damon_registered_ops[id]; 919f7b053aSSeongJae Park mutex_unlock(&damon_ops_lock); 929f7b053aSSeongJae Park return err; 939f7b053aSSeongJae Park } 949f7b053aSSeongJae Park 95f23b8eeeSSeongJae Park /* 96f23b8eeeSSeongJae Park * Construct a damon_region struct 97f23b8eeeSSeongJae Park * 98f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 99f23b8eeeSSeongJae Park */ 100f23b8eeeSSeongJae Park struct damon_region *damon_new_region(unsigned long start, unsigned long end) 101f23b8eeeSSeongJae Park { 102f23b8eeeSSeongJae Park struct damon_region *region; 103f23b8eeeSSeongJae Park 104f23b8eeeSSeongJae Park region = kmalloc(sizeof(*region), GFP_KERNEL); 105f23b8eeeSSeongJae Park if (!region) 106f23b8eeeSSeongJae Park return NULL; 107f23b8eeeSSeongJae Park 108f23b8eeeSSeongJae Park region->ar.start = start; 109f23b8eeeSSeongJae Park region->ar.end = end; 110f23b8eeeSSeongJae Park region->nr_accesses = 0; 111f23b8eeeSSeongJae Park INIT_LIST_HEAD(®ion->list); 112f23b8eeeSSeongJae Park 113fda504faSSeongJae Park region->age = 0; 114fda504faSSeongJae Park region->last_nr_accesses = 0; 115fda504faSSeongJae Park 116f23b8eeeSSeongJae Park return region; 117f23b8eeeSSeongJae Park } 118f23b8eeeSSeongJae Park 119f23b8eeeSSeongJae Park void damon_add_region(struct damon_region *r, struct damon_target *t) 120f23b8eeeSSeongJae Park { 121f23b8eeeSSeongJae Park list_add_tail(&r->list, &t->regions_list); 122b9a6ac4eSSeongJae Park t->nr_regions++; 123f23b8eeeSSeongJae Park } 124f23b8eeeSSeongJae Park 125b9a6ac4eSSeongJae Park static void damon_del_region(struct damon_region *r, struct damon_target *t) 126f23b8eeeSSeongJae Park { 127f23b8eeeSSeongJae Park list_del(&r->list); 128b9a6ac4eSSeongJae Park t->nr_regions--; 129f23b8eeeSSeongJae Park } 130f23b8eeeSSeongJae Park 131f23b8eeeSSeongJae Park static void damon_free_region(struct damon_region *r) 132f23b8eeeSSeongJae Park { 133f23b8eeeSSeongJae Park kfree(r); 134f23b8eeeSSeongJae Park } 135f23b8eeeSSeongJae Park 136b9a6ac4eSSeongJae Park void damon_destroy_region(struct damon_region *r, struct damon_target *t) 137f23b8eeeSSeongJae Park { 138b9a6ac4eSSeongJae Park damon_del_region(r, t); 139f23b8eeeSSeongJae Park damon_free_region(r); 140f23b8eeeSSeongJae Park } 141f23b8eeeSSeongJae Park 1421f366e42SSeongJae Park struct damos *damon_new_scheme( 1431f366e42SSeongJae Park unsigned long min_sz_region, unsigned long max_sz_region, 1441f366e42SSeongJae Park unsigned int min_nr_accesses, unsigned int max_nr_accesses, 1451f366e42SSeongJae Park unsigned int min_age_region, unsigned int max_age_region, 146ee801b7dSSeongJae Park enum damos_action action, struct damos_quota *quota, 147ee801b7dSSeongJae Park struct damos_watermarks *wmarks) 1481f366e42SSeongJae Park { 1491f366e42SSeongJae Park struct damos *scheme; 1501f366e42SSeongJae Park 1511f366e42SSeongJae Park scheme = kmalloc(sizeof(*scheme), GFP_KERNEL); 1521f366e42SSeongJae Park if (!scheme) 1531f366e42SSeongJae Park return NULL; 1541f366e42SSeongJae Park scheme->min_sz_region = min_sz_region; 1551f366e42SSeongJae Park scheme->max_sz_region = max_sz_region; 1561f366e42SSeongJae Park scheme->min_nr_accesses = min_nr_accesses; 1571f366e42SSeongJae Park scheme->max_nr_accesses = max_nr_accesses; 1581f366e42SSeongJae Park scheme->min_age_region = min_age_region; 1591f366e42SSeongJae Park scheme->max_age_region = max_age_region; 1601f366e42SSeongJae Park scheme->action = action; 1610e92c2eeSSeongJae Park scheme->stat = (struct damos_stat){}; 1621f366e42SSeongJae Park INIT_LIST_HEAD(&scheme->list); 1631f366e42SSeongJae Park 1641cd24303SSeongJae Park scheme->quota.ms = quota->ms; 1652b8a248dSSeongJae Park scheme->quota.sz = quota->sz; 1662b8a248dSSeongJae Park scheme->quota.reset_interval = quota->reset_interval; 16738683e00SSeongJae Park scheme->quota.weight_sz = quota->weight_sz; 16838683e00SSeongJae Park scheme->quota.weight_nr_accesses = quota->weight_nr_accesses; 16938683e00SSeongJae Park scheme->quota.weight_age = quota->weight_age; 1701cd24303SSeongJae Park scheme->quota.total_charged_sz = 0; 1711cd24303SSeongJae Park scheme->quota.total_charged_ns = 0; 1721cd24303SSeongJae Park scheme->quota.esz = 0; 1732b8a248dSSeongJae Park scheme->quota.charged_sz = 0; 1742b8a248dSSeongJae Park scheme->quota.charged_from = 0; 17550585192SSeongJae Park scheme->quota.charge_target_from = NULL; 17650585192SSeongJae Park scheme->quota.charge_addr_from = 0; 1772b8a248dSSeongJae Park 178ee801b7dSSeongJae Park scheme->wmarks.metric = wmarks->metric; 179ee801b7dSSeongJae Park scheme->wmarks.interval = wmarks->interval; 180ee801b7dSSeongJae Park scheme->wmarks.high = wmarks->high; 181ee801b7dSSeongJae Park scheme->wmarks.mid = wmarks->mid; 182ee801b7dSSeongJae Park scheme->wmarks.low = wmarks->low; 183ee801b7dSSeongJae Park scheme->wmarks.activated = true; 184ee801b7dSSeongJae Park 1851f366e42SSeongJae Park return scheme; 1861f366e42SSeongJae Park } 1871f366e42SSeongJae Park 1881f366e42SSeongJae Park void damon_add_scheme(struct damon_ctx *ctx, struct damos *s) 1891f366e42SSeongJae Park { 1901f366e42SSeongJae Park list_add_tail(&s->list, &ctx->schemes); 1911f366e42SSeongJae Park } 1921f366e42SSeongJae Park 1931f366e42SSeongJae Park static void damon_del_scheme(struct damos *s) 1941f366e42SSeongJae Park { 1951f366e42SSeongJae Park list_del(&s->list); 1961f366e42SSeongJae Park } 1971f366e42SSeongJae Park 1981f366e42SSeongJae Park static void damon_free_scheme(struct damos *s) 1991f366e42SSeongJae Park { 2001f366e42SSeongJae Park kfree(s); 2011f366e42SSeongJae Park } 2021f366e42SSeongJae Park 2031f366e42SSeongJae Park void damon_destroy_scheme(struct damos *s) 2041f366e42SSeongJae Park { 2051f366e42SSeongJae Park damon_del_scheme(s); 2061f366e42SSeongJae Park damon_free_scheme(s); 2071f366e42SSeongJae Park } 2081f366e42SSeongJae Park 209f23b8eeeSSeongJae Park /* 210f23b8eeeSSeongJae Park * Construct a damon_target struct 211f23b8eeeSSeongJae Park * 212f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 213f23b8eeeSSeongJae Park */ 2141971bd63SSeongJae Park struct damon_target *damon_new_target(void) 215f23b8eeeSSeongJae Park { 216f23b8eeeSSeongJae Park struct damon_target *t; 217f23b8eeeSSeongJae Park 218f23b8eeeSSeongJae Park t = kmalloc(sizeof(*t), GFP_KERNEL); 219f23b8eeeSSeongJae Park if (!t) 220f23b8eeeSSeongJae Park return NULL; 221f23b8eeeSSeongJae Park 2221971bd63SSeongJae Park t->pid = NULL; 223b9a6ac4eSSeongJae Park t->nr_regions = 0; 224f23b8eeeSSeongJae Park INIT_LIST_HEAD(&t->regions_list); 225f23b8eeeSSeongJae Park 226f23b8eeeSSeongJae Park return t; 227f23b8eeeSSeongJae Park } 228f23b8eeeSSeongJae Park 229f23b8eeeSSeongJae Park void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) 230f23b8eeeSSeongJae Park { 231b9a6ac4eSSeongJae Park list_add_tail(&t->list, &ctx->adaptive_targets); 232f23b8eeeSSeongJae Park } 233f23b8eeeSSeongJae Park 234b5ca3e83SXin Hao bool damon_targets_empty(struct damon_ctx *ctx) 235b5ca3e83SXin Hao { 236b5ca3e83SXin Hao return list_empty(&ctx->adaptive_targets); 237b5ca3e83SXin Hao } 238b5ca3e83SXin Hao 239f23b8eeeSSeongJae Park static void damon_del_target(struct damon_target *t) 240f23b8eeeSSeongJae Park { 241f23b8eeeSSeongJae Park list_del(&t->list); 242f23b8eeeSSeongJae Park } 243f23b8eeeSSeongJae Park 244f23b8eeeSSeongJae Park void damon_free_target(struct damon_target *t) 245f23b8eeeSSeongJae Park { 246f23b8eeeSSeongJae Park struct damon_region *r, *next; 247f23b8eeeSSeongJae Park 248f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 249f23b8eeeSSeongJae Park damon_free_region(r); 250f23b8eeeSSeongJae Park kfree(t); 251f23b8eeeSSeongJae Park } 252f23b8eeeSSeongJae Park 253f23b8eeeSSeongJae Park void damon_destroy_target(struct damon_target *t) 254f23b8eeeSSeongJae Park { 255f23b8eeeSSeongJae Park damon_del_target(t); 256f23b8eeeSSeongJae Park damon_free_target(t); 257f23b8eeeSSeongJae Park } 258f23b8eeeSSeongJae Park 259b9a6ac4eSSeongJae Park unsigned int damon_nr_regions(struct damon_target *t) 260b9a6ac4eSSeongJae Park { 261b9a6ac4eSSeongJae Park return t->nr_regions; 262b9a6ac4eSSeongJae Park } 263b9a6ac4eSSeongJae Park 2642224d848SSeongJae Park struct damon_ctx *damon_new_ctx(void) 2652224d848SSeongJae Park { 2662224d848SSeongJae Park struct damon_ctx *ctx; 2672224d848SSeongJae Park 2682224d848SSeongJae Park ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 2692224d848SSeongJae Park if (!ctx) 2702224d848SSeongJae Park return NULL; 2712224d848SSeongJae Park 2722224d848SSeongJae Park ctx->sample_interval = 5 * 1000; 2732224d848SSeongJae Park ctx->aggr_interval = 100 * 1000; 274f7d911c3SSeongJae Park ctx->ops_update_interval = 60 * 1000 * 1000; 2752224d848SSeongJae Park 2762224d848SSeongJae Park ktime_get_coarse_ts64(&ctx->last_aggregation); 277f7d911c3SSeongJae Park ctx->last_ops_update = ctx->last_aggregation; 2782224d848SSeongJae Park 2792224d848SSeongJae Park mutex_init(&ctx->kdamond_lock); 2802224d848SSeongJae Park 281b9a6ac4eSSeongJae Park ctx->min_nr_regions = 10; 282b9a6ac4eSSeongJae Park ctx->max_nr_regions = 1000; 283b9a6ac4eSSeongJae Park 284b9a6ac4eSSeongJae Park INIT_LIST_HEAD(&ctx->adaptive_targets); 2851f366e42SSeongJae Park INIT_LIST_HEAD(&ctx->schemes); 2862224d848SSeongJae Park 2872224d848SSeongJae Park return ctx; 2882224d848SSeongJae Park } 2892224d848SSeongJae Park 290f23b8eeeSSeongJae Park static void damon_destroy_targets(struct damon_ctx *ctx) 291f23b8eeeSSeongJae Park { 292f23b8eeeSSeongJae Park struct damon_target *t, *next_t; 293f23b8eeeSSeongJae Park 294f7d911c3SSeongJae Park if (ctx->ops.cleanup) { 295f7d911c3SSeongJae Park ctx->ops.cleanup(ctx); 296f23b8eeeSSeongJae Park return; 297f23b8eeeSSeongJae Park } 298f23b8eeeSSeongJae Park 299f23b8eeeSSeongJae Park damon_for_each_target_safe(t, next_t, ctx) 300f23b8eeeSSeongJae Park damon_destroy_target(t); 301f23b8eeeSSeongJae Park } 302f23b8eeeSSeongJae Park 3032224d848SSeongJae Park void damon_destroy_ctx(struct damon_ctx *ctx) 3042224d848SSeongJae Park { 3051f366e42SSeongJae Park struct damos *s, *next_s; 3061f366e42SSeongJae Park 307f23b8eeeSSeongJae Park damon_destroy_targets(ctx); 3081f366e42SSeongJae Park 3091f366e42SSeongJae Park damon_for_each_scheme_safe(s, next_s, ctx) 3101f366e42SSeongJae Park damon_destroy_scheme(s); 3111f366e42SSeongJae Park 3122224d848SSeongJae Park kfree(ctx); 3132224d848SSeongJae Park } 3142224d848SSeongJae Park 3152224d848SSeongJae Park /** 3162224d848SSeongJae Park * damon_set_attrs() - Set attributes for the monitoring. 3172224d848SSeongJae Park * @ctx: monitoring context 3182224d848SSeongJae Park * @sample_int: time interval between samplings 3192224d848SSeongJae Park * @aggr_int: time interval between aggregations 320f7d911c3SSeongJae Park * @ops_upd_int: time interval between monitoring operations updates 321b9a6ac4eSSeongJae Park * @min_nr_reg: minimal number of regions 322b9a6ac4eSSeongJae Park * @max_nr_reg: maximum number of regions 3232224d848SSeongJae Park * 3242224d848SSeongJae Park * This function should not be called while the kdamond is running. 3252224d848SSeongJae Park * Every time interval is in micro-seconds. 3262224d848SSeongJae Park * 3272224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 3282224d848SSeongJae Park */ 3292224d848SSeongJae Park int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, 330f7d911c3SSeongJae Park unsigned long aggr_int, unsigned long ops_upd_int, 331b9a6ac4eSSeongJae Park unsigned long min_nr_reg, unsigned long max_nr_reg) 3322224d848SSeongJae Park { 3331afaf5cbSSeongJae Park if (min_nr_reg < 3) 334b9a6ac4eSSeongJae Park return -EINVAL; 3351afaf5cbSSeongJae Park if (min_nr_reg > max_nr_reg) 336b9a6ac4eSSeongJae Park return -EINVAL; 337b9a6ac4eSSeongJae Park 3382224d848SSeongJae Park ctx->sample_interval = sample_int; 3392224d848SSeongJae Park ctx->aggr_interval = aggr_int; 340f7d911c3SSeongJae Park ctx->ops_update_interval = ops_upd_int; 341b9a6ac4eSSeongJae Park ctx->min_nr_regions = min_nr_reg; 342b9a6ac4eSSeongJae Park ctx->max_nr_regions = max_nr_reg; 3432224d848SSeongJae Park 3442224d848SSeongJae Park return 0; 3452224d848SSeongJae Park } 3462224d848SSeongJae Park 3474bc05954SSeongJae Park /** 3481f366e42SSeongJae Park * damon_set_schemes() - Set data access monitoring based operation schemes. 3491f366e42SSeongJae Park * @ctx: monitoring context 3501f366e42SSeongJae Park * @schemes: array of the schemes 3511f366e42SSeongJae Park * @nr_schemes: number of entries in @schemes 3521f366e42SSeongJae Park * 3531f366e42SSeongJae Park * This function should not be called while the kdamond of the context is 3541f366e42SSeongJae Park * running. 3551f366e42SSeongJae Park * 3561f366e42SSeongJae Park * Return: 0 if success, or negative error code otherwise. 3571f366e42SSeongJae Park */ 3581f366e42SSeongJae Park int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, 3591f366e42SSeongJae Park ssize_t nr_schemes) 3601f366e42SSeongJae Park { 3611f366e42SSeongJae Park struct damos *s, *next; 3621f366e42SSeongJae Park ssize_t i; 3631f366e42SSeongJae Park 3641f366e42SSeongJae Park damon_for_each_scheme_safe(s, next, ctx) 3651f366e42SSeongJae Park damon_destroy_scheme(s); 3661f366e42SSeongJae Park for (i = 0; i < nr_schemes; i++) 3671f366e42SSeongJae Park damon_add_scheme(ctx, schemes[i]); 3681f366e42SSeongJae Park return 0; 3691f366e42SSeongJae Park } 3701f366e42SSeongJae Park 3711f366e42SSeongJae Park /** 3724bc05954SSeongJae Park * damon_nr_running_ctxs() - Return number of currently running contexts. 3734bc05954SSeongJae Park */ 3744bc05954SSeongJae Park int damon_nr_running_ctxs(void) 3754bc05954SSeongJae Park { 3764bc05954SSeongJae Park int nr_ctxs; 3774bc05954SSeongJae Park 3784bc05954SSeongJae Park mutex_lock(&damon_lock); 3794bc05954SSeongJae Park nr_ctxs = nr_running_ctxs; 3804bc05954SSeongJae Park mutex_unlock(&damon_lock); 3814bc05954SSeongJae Park 3824bc05954SSeongJae Park return nr_ctxs; 3834bc05954SSeongJae Park } 3844bc05954SSeongJae Park 385b9a6ac4eSSeongJae Park /* Returns the size upper limit for each monitoring region */ 386b9a6ac4eSSeongJae Park static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) 387b9a6ac4eSSeongJae Park { 388b9a6ac4eSSeongJae Park struct damon_target *t; 389b9a6ac4eSSeongJae Park struct damon_region *r; 390b9a6ac4eSSeongJae Park unsigned long sz = 0; 391b9a6ac4eSSeongJae Park 392b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) { 393b9a6ac4eSSeongJae Park damon_for_each_region(r, t) 394b9a6ac4eSSeongJae Park sz += r->ar.end - r->ar.start; 395b9a6ac4eSSeongJae Park } 396b9a6ac4eSSeongJae Park 397b9a6ac4eSSeongJae Park if (ctx->min_nr_regions) 398b9a6ac4eSSeongJae Park sz /= ctx->min_nr_regions; 399b9a6ac4eSSeongJae Park if (sz < DAMON_MIN_REGION) 400b9a6ac4eSSeongJae Park sz = DAMON_MIN_REGION; 401b9a6ac4eSSeongJae Park 402b9a6ac4eSSeongJae Park return sz; 403b9a6ac4eSSeongJae Park } 404b9a6ac4eSSeongJae Park 4052224d848SSeongJae Park static int kdamond_fn(void *data); 4062224d848SSeongJae Park 4072224d848SSeongJae Park /* 4082224d848SSeongJae Park * __damon_start() - Starts monitoring with given context. 4092224d848SSeongJae Park * @ctx: monitoring context 4102224d848SSeongJae Park * 4112224d848SSeongJae Park * This function should be called while damon_lock is hold. 4122224d848SSeongJae Park * 4132224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4142224d848SSeongJae Park */ 4152224d848SSeongJae Park static int __damon_start(struct damon_ctx *ctx) 4162224d848SSeongJae Park { 4172224d848SSeongJae Park int err = -EBUSY; 4182224d848SSeongJae Park 4192224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 4202224d848SSeongJae Park if (!ctx->kdamond) { 4212224d848SSeongJae Park err = 0; 4222224d848SSeongJae Park ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d", 4232224d848SSeongJae Park nr_running_ctxs); 4242224d848SSeongJae Park if (IS_ERR(ctx->kdamond)) { 4252224d848SSeongJae Park err = PTR_ERR(ctx->kdamond); 4267ec1992bSColin Ian King ctx->kdamond = NULL; 4272224d848SSeongJae Park } 4282224d848SSeongJae Park } 4292224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 4302224d848SSeongJae Park 4312224d848SSeongJae Park return err; 4322224d848SSeongJae Park } 4332224d848SSeongJae Park 4342224d848SSeongJae Park /** 4352224d848SSeongJae Park * damon_start() - Starts the monitorings for a given group of contexts. 4362224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to start monitoring 4372224d848SSeongJae Park * @nr_ctxs: size of @ctxs 438*8b9b0d33SSeongJae Park * @exclusive: exclusiveness of this contexts group 4392224d848SSeongJae Park * 4402224d848SSeongJae Park * This function starts a group of monitoring threads for a group of monitoring 4412224d848SSeongJae Park * contexts. One thread per each context is created and run in parallel. The 442*8b9b0d33SSeongJae Park * caller should handle synchronization between the threads by itself. If 443*8b9b0d33SSeongJae Park * @exclusive is true and a group of threads that created by other 444*8b9b0d33SSeongJae Park * 'damon_start()' call is currently running, this function does nothing but 445*8b9b0d33SSeongJae Park * returns -EBUSY. 4462224d848SSeongJae Park * 4472224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4482224d848SSeongJae Park */ 449*8b9b0d33SSeongJae Park int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive) 4502224d848SSeongJae Park { 4512224d848SSeongJae Park int i; 4522224d848SSeongJae Park int err = 0; 4532224d848SSeongJae Park 4542224d848SSeongJae Park mutex_lock(&damon_lock); 455*8b9b0d33SSeongJae Park if ((exclusive && nr_running_ctxs) || 456*8b9b0d33SSeongJae Park (!exclusive && running_exclusive_ctxs)) { 4572224d848SSeongJae Park mutex_unlock(&damon_lock); 4582224d848SSeongJae Park return -EBUSY; 4592224d848SSeongJae Park } 4602224d848SSeongJae Park 4612224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 4622224d848SSeongJae Park err = __damon_start(ctxs[i]); 4632224d848SSeongJae Park if (err) 4642224d848SSeongJae Park break; 4652224d848SSeongJae Park nr_running_ctxs++; 4662224d848SSeongJae Park } 467*8b9b0d33SSeongJae Park if (exclusive && nr_running_ctxs) 468*8b9b0d33SSeongJae Park running_exclusive_ctxs = true; 4692224d848SSeongJae Park mutex_unlock(&damon_lock); 4702224d848SSeongJae Park 4712224d848SSeongJae Park return err; 4722224d848SSeongJae Park } 4732224d848SSeongJae Park 4742224d848SSeongJae Park /* 475*8b9b0d33SSeongJae Park * __damon_stop() - Stops monitoring of a given context. 4762224d848SSeongJae Park * @ctx: monitoring context 4772224d848SSeongJae Park * 4782224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4792224d848SSeongJae Park */ 4802224d848SSeongJae Park static int __damon_stop(struct damon_ctx *ctx) 4812224d848SSeongJae Park { 4820f91d133SChangbin Du struct task_struct *tsk; 4830f91d133SChangbin Du 4842224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 4850f91d133SChangbin Du tsk = ctx->kdamond; 4860f91d133SChangbin Du if (tsk) { 4870f91d133SChangbin Du get_task_struct(tsk); 4882224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 4890f91d133SChangbin Du kthread_stop(tsk); 4900f91d133SChangbin Du put_task_struct(tsk); 4912224d848SSeongJae Park return 0; 4922224d848SSeongJae Park } 4932224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 4942224d848SSeongJae Park 4952224d848SSeongJae Park return -EPERM; 4962224d848SSeongJae Park } 4972224d848SSeongJae Park 4982224d848SSeongJae Park /** 4992224d848SSeongJae Park * damon_stop() - Stops the monitorings for a given group of contexts. 5002224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to stop monitoring 5012224d848SSeongJae Park * @nr_ctxs: size of @ctxs 5022224d848SSeongJae Park * 5032224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 5042224d848SSeongJae Park */ 5052224d848SSeongJae Park int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) 5062224d848SSeongJae Park { 5072224d848SSeongJae Park int i, err = 0; 5082224d848SSeongJae Park 5092224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 5102224d848SSeongJae Park /* nr_running_ctxs is decremented in kdamond_fn */ 5112224d848SSeongJae Park err = __damon_stop(ctxs[i]); 5122224d848SSeongJae Park if (err) 513*8b9b0d33SSeongJae Park break; 5142224d848SSeongJae Park } 5152224d848SSeongJae Park return err; 5162224d848SSeongJae Park } 5172224d848SSeongJae Park 5182224d848SSeongJae Park /* 5192224d848SSeongJae Park * damon_check_reset_time_interval() - Check if a time interval is elapsed. 5202224d848SSeongJae Park * @baseline: the time to check whether the interval has elapsed since 5212224d848SSeongJae Park * @interval: the time interval (microseconds) 5222224d848SSeongJae Park * 5232224d848SSeongJae Park * See whether the given time interval has passed since the given baseline 5242224d848SSeongJae Park * time. If so, it also updates the baseline to current time for next check. 5252224d848SSeongJae Park * 5262224d848SSeongJae Park * Return: true if the time interval has passed, or false otherwise. 5272224d848SSeongJae Park */ 5282224d848SSeongJae Park static bool damon_check_reset_time_interval(struct timespec64 *baseline, 5292224d848SSeongJae Park unsigned long interval) 5302224d848SSeongJae Park { 5312224d848SSeongJae Park struct timespec64 now; 5322224d848SSeongJae Park 5332224d848SSeongJae Park ktime_get_coarse_ts64(&now); 5342224d848SSeongJae Park if ((timespec64_to_ns(&now) - timespec64_to_ns(baseline)) < 5352224d848SSeongJae Park interval * 1000) 5362224d848SSeongJae Park return false; 5372224d848SSeongJae Park *baseline = now; 5382224d848SSeongJae Park return true; 5392224d848SSeongJae Park } 5402224d848SSeongJae Park 5412224d848SSeongJae Park /* 5422224d848SSeongJae Park * Check whether it is time to flush the aggregated information 5432224d848SSeongJae Park */ 5442224d848SSeongJae Park static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) 5452224d848SSeongJae Park { 5462224d848SSeongJae Park return damon_check_reset_time_interval(&ctx->last_aggregation, 5472224d848SSeongJae Park ctx->aggr_interval); 5482224d848SSeongJae Park } 5492224d848SSeongJae Park 5502224d848SSeongJae Park /* 551f23b8eeeSSeongJae Park * Reset the aggregated monitoring results ('nr_accesses' of each region). 552f23b8eeeSSeongJae Park */ 553f23b8eeeSSeongJae Park static void kdamond_reset_aggregated(struct damon_ctx *c) 554f23b8eeeSSeongJae Park { 555f23b8eeeSSeongJae Park struct damon_target *t; 55676fd0285SSeongJae Park unsigned int ti = 0; /* target's index */ 557f23b8eeeSSeongJae Park 558f23b8eeeSSeongJae Park damon_for_each_target(t, c) { 559f23b8eeeSSeongJae Park struct damon_region *r; 560f23b8eeeSSeongJae Park 5612fcb9362SSeongJae Park damon_for_each_region(r, t) { 56276fd0285SSeongJae Park trace_damon_aggregated(t, ti, r, damon_nr_regions(t)); 563fda504faSSeongJae Park r->last_nr_accesses = r->nr_accesses; 564f23b8eeeSSeongJae Park r->nr_accesses = 0; 565f23b8eeeSSeongJae Park } 56676fd0285SSeongJae Park ti++; 567f23b8eeeSSeongJae Park } 5682fcb9362SSeongJae Park } 569f23b8eeeSSeongJae Park 5702b8a248dSSeongJae Park static void damon_split_region_at(struct damon_ctx *ctx, 5712b8a248dSSeongJae Park struct damon_target *t, struct damon_region *r, 5722b8a248dSSeongJae Park unsigned long sz_r); 5732b8a248dSSeongJae Park 57438683e00SSeongJae Park static bool __damos_valid_target(struct damon_region *r, struct damos *s) 57538683e00SSeongJae Park { 57638683e00SSeongJae Park unsigned long sz; 57738683e00SSeongJae Park 57838683e00SSeongJae Park sz = r->ar.end - r->ar.start; 57938683e00SSeongJae Park return s->min_sz_region <= sz && sz <= s->max_sz_region && 58038683e00SSeongJae Park s->min_nr_accesses <= r->nr_accesses && 58138683e00SSeongJae Park r->nr_accesses <= s->max_nr_accesses && 58238683e00SSeongJae Park s->min_age_region <= r->age && r->age <= s->max_age_region; 58338683e00SSeongJae Park } 58438683e00SSeongJae Park 58538683e00SSeongJae Park static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, 58638683e00SSeongJae Park struct damon_region *r, struct damos *s) 58738683e00SSeongJae Park { 58838683e00SSeongJae Park bool ret = __damos_valid_target(r, s); 58938683e00SSeongJae Park 590f7d911c3SSeongJae Park if (!ret || !s->quota.esz || !c->ops.get_scheme_score) 59138683e00SSeongJae Park return ret; 59238683e00SSeongJae Park 593f7d911c3SSeongJae Park return c->ops.get_scheme_score(c, t, r, s) >= s->quota.min_score; 59438683e00SSeongJae Park } 59538683e00SSeongJae Park 5961f366e42SSeongJae Park static void damon_do_apply_schemes(struct damon_ctx *c, 5971f366e42SSeongJae Park struct damon_target *t, 5981f366e42SSeongJae Park struct damon_region *r) 5991f366e42SSeongJae Park { 6001f366e42SSeongJae Park struct damos *s; 6011f366e42SSeongJae Park 6021f366e42SSeongJae Park damon_for_each_scheme(s, c) { 6032b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 6042b8a248dSSeongJae Park unsigned long sz = r->ar.end - r->ar.start; 6051cd24303SSeongJae Park struct timespec64 begin, end; 6060e92c2eeSSeongJae Park unsigned long sz_applied = 0; 6072b8a248dSSeongJae Park 608ee801b7dSSeongJae Park if (!s->wmarks.activated) 609ee801b7dSSeongJae Park continue; 610ee801b7dSSeongJae Park 6112b8a248dSSeongJae Park /* Check the quota */ 6121cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 6132b8a248dSSeongJae Park continue; 6142b8a248dSSeongJae Park 61550585192SSeongJae Park /* Skip previously charged regions */ 61650585192SSeongJae Park if (quota->charge_target_from) { 61750585192SSeongJae Park if (t != quota->charge_target_from) 61850585192SSeongJae Park continue; 61950585192SSeongJae Park if (r == damon_last_region(t)) { 62050585192SSeongJae Park quota->charge_target_from = NULL; 62150585192SSeongJae Park quota->charge_addr_from = 0; 62250585192SSeongJae Park continue; 62350585192SSeongJae Park } 62450585192SSeongJae Park if (quota->charge_addr_from && 62550585192SSeongJae Park r->ar.end <= quota->charge_addr_from) 62650585192SSeongJae Park continue; 62750585192SSeongJae Park 62850585192SSeongJae Park if (quota->charge_addr_from && r->ar.start < 62950585192SSeongJae Park quota->charge_addr_from) { 63050585192SSeongJae Park sz = ALIGN_DOWN(quota->charge_addr_from - 63150585192SSeongJae Park r->ar.start, DAMON_MIN_REGION); 63250585192SSeongJae Park if (!sz) { 63350585192SSeongJae Park if (r->ar.end - r->ar.start <= 63450585192SSeongJae Park DAMON_MIN_REGION) 63550585192SSeongJae Park continue; 63650585192SSeongJae Park sz = DAMON_MIN_REGION; 63750585192SSeongJae Park } 63850585192SSeongJae Park damon_split_region_at(c, t, r, sz); 63950585192SSeongJae Park r = damon_next_region(r); 64050585192SSeongJae Park sz = r->ar.end - r->ar.start; 64150585192SSeongJae Park } 64250585192SSeongJae Park quota->charge_target_from = NULL; 64350585192SSeongJae Park quota->charge_addr_from = 0; 64450585192SSeongJae Park } 64550585192SSeongJae Park 64638683e00SSeongJae Park if (!damos_valid_target(c, t, r, s)) 6471f366e42SSeongJae Park continue; 6482b8a248dSSeongJae Park 6492b8a248dSSeongJae Park /* Apply the scheme */ 650f7d911c3SSeongJae Park if (c->ops.apply_scheme) { 6511cd24303SSeongJae Park if (quota->esz && 6521cd24303SSeongJae Park quota->charged_sz + sz > quota->esz) { 6531cd24303SSeongJae Park sz = ALIGN_DOWN(quota->esz - quota->charged_sz, 6542b8a248dSSeongJae Park DAMON_MIN_REGION); 6552b8a248dSSeongJae Park if (!sz) 6562b8a248dSSeongJae Park goto update_stat; 6572b8a248dSSeongJae Park damon_split_region_at(c, t, r, sz); 6582b8a248dSSeongJae Park } 6591cd24303SSeongJae Park ktime_get_coarse_ts64(&begin); 660f7d911c3SSeongJae Park sz_applied = c->ops.apply_scheme(c, t, r, s); 6611cd24303SSeongJae Park ktime_get_coarse_ts64(&end); 6621cd24303SSeongJae Park quota->total_charged_ns += timespec64_to_ns(&end) - 6631cd24303SSeongJae Park timespec64_to_ns(&begin); 6642b8a248dSSeongJae Park quota->charged_sz += sz; 6651cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) { 66650585192SSeongJae Park quota->charge_target_from = t; 66750585192SSeongJae Park quota->charge_addr_from = r->ar.end + 1; 66850585192SSeongJae Park } 6692b8a248dSSeongJae Park } 6702f0b548cSSeongJae Park if (s->action != DAMOS_STAT) 6711f366e42SSeongJae Park r->age = 0; 6722b8a248dSSeongJae Park 6732b8a248dSSeongJae Park update_stat: 6740e92c2eeSSeongJae Park s->stat.nr_tried++; 6750e92c2eeSSeongJae Park s->stat.sz_tried += sz; 6760e92c2eeSSeongJae Park if (sz_applied) 6770e92c2eeSSeongJae Park s->stat.nr_applied++; 6780e92c2eeSSeongJae Park s->stat.sz_applied += sz_applied; 6791f366e42SSeongJae Park } 6801f366e42SSeongJae Park } 6811f366e42SSeongJae Park 6821cd24303SSeongJae Park /* Shouldn't be called if quota->ms and quota->sz are zero */ 6831cd24303SSeongJae Park static void damos_set_effective_quota(struct damos_quota *quota) 6841cd24303SSeongJae Park { 6851cd24303SSeongJae Park unsigned long throughput; 6861cd24303SSeongJae Park unsigned long esz; 6871cd24303SSeongJae Park 6881cd24303SSeongJae Park if (!quota->ms) { 6891cd24303SSeongJae Park quota->esz = quota->sz; 6901cd24303SSeongJae Park return; 6911cd24303SSeongJae Park } 6921cd24303SSeongJae Park 6931cd24303SSeongJae Park if (quota->total_charged_ns) 6941cd24303SSeongJae Park throughput = quota->total_charged_sz * 1000000 / 6951cd24303SSeongJae Park quota->total_charged_ns; 6961cd24303SSeongJae Park else 6971cd24303SSeongJae Park throughput = PAGE_SIZE * 1024; 6981cd24303SSeongJae Park esz = throughput * quota->ms; 6991cd24303SSeongJae Park 7001cd24303SSeongJae Park if (quota->sz && quota->sz < esz) 7011cd24303SSeongJae Park esz = quota->sz; 7021cd24303SSeongJae Park quota->esz = esz; 7031cd24303SSeongJae Park } 7041cd24303SSeongJae Park 7051f366e42SSeongJae Park static void kdamond_apply_schemes(struct damon_ctx *c) 7061f366e42SSeongJae Park { 7071f366e42SSeongJae Park struct damon_target *t; 7082b8a248dSSeongJae Park struct damon_region *r, *next_r; 7092b8a248dSSeongJae Park struct damos *s; 7102b8a248dSSeongJae Park 7112b8a248dSSeongJae Park damon_for_each_scheme(s, c) { 7122b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 71338683e00SSeongJae Park unsigned long cumulated_sz; 71438683e00SSeongJae Park unsigned int score, max_score = 0; 7152b8a248dSSeongJae Park 716ee801b7dSSeongJae Park if (!s->wmarks.activated) 717ee801b7dSSeongJae Park continue; 718ee801b7dSSeongJae Park 7191cd24303SSeongJae Park if (!quota->ms && !quota->sz) 7202b8a248dSSeongJae Park continue; 7212b8a248dSSeongJae Park 7222b8a248dSSeongJae Park /* New charge window starts */ 7232b8a248dSSeongJae Park if (time_after_eq(jiffies, quota->charged_from + 7242b8a248dSSeongJae Park msecs_to_jiffies( 7252b8a248dSSeongJae Park quota->reset_interval))) { 7266268eac3SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 7276268eac3SSeongJae Park s->stat.qt_exceeds++; 7281cd24303SSeongJae Park quota->total_charged_sz += quota->charged_sz; 7292b8a248dSSeongJae Park quota->charged_from = jiffies; 7302b8a248dSSeongJae Park quota->charged_sz = 0; 7311cd24303SSeongJae Park damos_set_effective_quota(quota); 7322b8a248dSSeongJae Park } 73338683e00SSeongJae Park 734f7d911c3SSeongJae Park if (!c->ops.get_scheme_score) 73538683e00SSeongJae Park continue; 73638683e00SSeongJae Park 73738683e00SSeongJae Park /* Fill up the score histogram */ 73838683e00SSeongJae Park memset(quota->histogram, 0, sizeof(quota->histogram)); 73938683e00SSeongJae Park damon_for_each_target(t, c) { 74038683e00SSeongJae Park damon_for_each_region(r, t) { 74138683e00SSeongJae Park if (!__damos_valid_target(r, s)) 74238683e00SSeongJae Park continue; 743f7d911c3SSeongJae Park score = c->ops.get_scheme_score( 74438683e00SSeongJae Park c, t, r, s); 74538683e00SSeongJae Park quota->histogram[score] += 74638683e00SSeongJae Park r->ar.end - r->ar.start; 74738683e00SSeongJae Park if (score > max_score) 74838683e00SSeongJae Park max_score = score; 74938683e00SSeongJae Park } 75038683e00SSeongJae Park } 75138683e00SSeongJae Park 75238683e00SSeongJae Park /* Set the min score limit */ 75338683e00SSeongJae Park for (cumulated_sz = 0, score = max_score; ; score--) { 75438683e00SSeongJae Park cumulated_sz += quota->histogram[score]; 75538683e00SSeongJae Park if (cumulated_sz >= quota->esz || !score) 75638683e00SSeongJae Park break; 75738683e00SSeongJae Park } 75838683e00SSeongJae Park quota->min_score = score; 7592b8a248dSSeongJae Park } 7601f366e42SSeongJae Park 7611f366e42SSeongJae Park damon_for_each_target(t, c) { 7622b8a248dSSeongJae Park damon_for_each_region_safe(r, next_r, t) 7631f366e42SSeongJae Park damon_do_apply_schemes(c, t, r); 7641f366e42SSeongJae Park } 7651f366e42SSeongJae Park } 7661f366e42SSeongJae Park 76788f86dcfSSeongJae Park static inline unsigned long sz_damon_region(struct damon_region *r) 76888f86dcfSSeongJae Park { 76988f86dcfSSeongJae Park return r->ar.end - r->ar.start; 77088f86dcfSSeongJae Park } 771b9a6ac4eSSeongJae Park 772b9a6ac4eSSeongJae Park /* 773b9a6ac4eSSeongJae Park * Merge two adjacent regions into one region 774b9a6ac4eSSeongJae Park */ 775b9a6ac4eSSeongJae Park static void damon_merge_two_regions(struct damon_target *t, 776b9a6ac4eSSeongJae Park struct damon_region *l, struct damon_region *r) 777b9a6ac4eSSeongJae Park { 778b9a6ac4eSSeongJae Park unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r); 779b9a6ac4eSSeongJae Park 780b9a6ac4eSSeongJae Park l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / 781b9a6ac4eSSeongJae Park (sz_l + sz_r); 782fda504faSSeongJae Park l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); 783b9a6ac4eSSeongJae Park l->ar.end = r->ar.end; 784b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 785b9a6ac4eSSeongJae Park } 786b9a6ac4eSSeongJae Park 787b9a6ac4eSSeongJae Park /* 788b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 789b9a6ac4eSSeongJae Park * 790b9a6ac4eSSeongJae Park * t target affected by this merge operation 791b9a6ac4eSSeongJae Park * thres '->nr_accesses' diff threshold for the merge 792b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 793b9a6ac4eSSeongJae Park */ 794b9a6ac4eSSeongJae Park static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, 795b9a6ac4eSSeongJae Park unsigned long sz_limit) 796b9a6ac4eSSeongJae Park { 797b9a6ac4eSSeongJae Park struct damon_region *r, *prev = NULL, *next; 798b9a6ac4eSSeongJae Park 799b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 800d720bbbdSXin Hao if (abs(r->nr_accesses - r->last_nr_accesses) > thres) 801fda504faSSeongJae Park r->age = 0; 802fda504faSSeongJae Park else 803fda504faSSeongJae Park r->age++; 804fda504faSSeongJae Park 805b9a6ac4eSSeongJae Park if (prev && prev->ar.end == r->ar.start && 806d720bbbdSXin Hao abs(prev->nr_accesses - r->nr_accesses) <= thres && 807b9a6ac4eSSeongJae Park sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) 808b9a6ac4eSSeongJae Park damon_merge_two_regions(t, prev, r); 809b9a6ac4eSSeongJae Park else 810b9a6ac4eSSeongJae Park prev = r; 811b9a6ac4eSSeongJae Park } 812b9a6ac4eSSeongJae Park } 813b9a6ac4eSSeongJae Park 814b9a6ac4eSSeongJae Park /* 815b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 816b9a6ac4eSSeongJae Park * 817b9a6ac4eSSeongJae Park * threshold '->nr_accesses' diff threshold for the merge 818b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 819b9a6ac4eSSeongJae Park * 820b9a6ac4eSSeongJae Park * This function merges monitoring target regions which are adjacent and their 821b9a6ac4eSSeongJae Park * access frequencies are similar. This is for minimizing the monitoring 822b9a6ac4eSSeongJae Park * overhead under the dynamically changeable access pattern. If a merge was 823b9a6ac4eSSeongJae Park * unnecessarily made, later 'kdamond_split_regions()' will revert it. 824b9a6ac4eSSeongJae Park */ 825b9a6ac4eSSeongJae Park static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, 826b9a6ac4eSSeongJae Park unsigned long sz_limit) 827b9a6ac4eSSeongJae Park { 828b9a6ac4eSSeongJae Park struct damon_target *t; 829b9a6ac4eSSeongJae Park 830b9a6ac4eSSeongJae Park damon_for_each_target(t, c) 831b9a6ac4eSSeongJae Park damon_merge_regions_of(t, threshold, sz_limit); 832b9a6ac4eSSeongJae Park } 833b9a6ac4eSSeongJae Park 834b9a6ac4eSSeongJae Park /* 835b9a6ac4eSSeongJae Park * Split a region in two 836b9a6ac4eSSeongJae Park * 837b9a6ac4eSSeongJae Park * r the region to be split 838b9a6ac4eSSeongJae Park * sz_r size of the first sub-region that will be made 839b9a6ac4eSSeongJae Park */ 840b9a6ac4eSSeongJae Park static void damon_split_region_at(struct damon_ctx *ctx, 841b9a6ac4eSSeongJae Park struct damon_target *t, struct damon_region *r, 842b9a6ac4eSSeongJae Park unsigned long sz_r) 843b9a6ac4eSSeongJae Park { 844b9a6ac4eSSeongJae Park struct damon_region *new; 845b9a6ac4eSSeongJae Park 846b9a6ac4eSSeongJae Park new = damon_new_region(r->ar.start + sz_r, r->ar.end); 847b9a6ac4eSSeongJae Park if (!new) 848b9a6ac4eSSeongJae Park return; 849b9a6ac4eSSeongJae Park 850b9a6ac4eSSeongJae Park r->ar.end = new->ar.start; 851b9a6ac4eSSeongJae Park 852fda504faSSeongJae Park new->age = r->age; 853fda504faSSeongJae Park new->last_nr_accesses = r->last_nr_accesses; 854fda504faSSeongJae Park 855b9a6ac4eSSeongJae Park damon_insert_region(new, r, damon_next_region(r), t); 856b9a6ac4eSSeongJae Park } 857b9a6ac4eSSeongJae Park 858b9a6ac4eSSeongJae Park /* Split every region in the given target into 'nr_subs' regions */ 859b9a6ac4eSSeongJae Park static void damon_split_regions_of(struct damon_ctx *ctx, 860b9a6ac4eSSeongJae Park struct damon_target *t, int nr_subs) 861b9a6ac4eSSeongJae Park { 862b9a6ac4eSSeongJae Park struct damon_region *r, *next; 863b9a6ac4eSSeongJae Park unsigned long sz_region, sz_sub = 0; 864b9a6ac4eSSeongJae Park int i; 865b9a6ac4eSSeongJae Park 866b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 867b9a6ac4eSSeongJae Park sz_region = r->ar.end - r->ar.start; 868b9a6ac4eSSeongJae Park 869b9a6ac4eSSeongJae Park for (i = 0; i < nr_subs - 1 && 870b9a6ac4eSSeongJae Park sz_region > 2 * DAMON_MIN_REGION; i++) { 871b9a6ac4eSSeongJae Park /* 872b9a6ac4eSSeongJae Park * Randomly select size of left sub-region to be at 873b9a6ac4eSSeongJae Park * least 10 percent and at most 90% of original region 874b9a6ac4eSSeongJae Park */ 875b9a6ac4eSSeongJae Park sz_sub = ALIGN_DOWN(damon_rand(1, 10) * 876b9a6ac4eSSeongJae Park sz_region / 10, DAMON_MIN_REGION); 877b9a6ac4eSSeongJae Park /* Do not allow blank region */ 878b9a6ac4eSSeongJae Park if (sz_sub == 0 || sz_sub >= sz_region) 879b9a6ac4eSSeongJae Park continue; 880b9a6ac4eSSeongJae Park 881b9a6ac4eSSeongJae Park damon_split_region_at(ctx, t, r, sz_sub); 882b9a6ac4eSSeongJae Park sz_region = sz_sub; 883b9a6ac4eSSeongJae Park } 884b9a6ac4eSSeongJae Park } 885b9a6ac4eSSeongJae Park } 886b9a6ac4eSSeongJae Park 887b9a6ac4eSSeongJae Park /* 888b9a6ac4eSSeongJae Park * Split every target region into randomly-sized small regions 889b9a6ac4eSSeongJae Park * 890b9a6ac4eSSeongJae Park * This function splits every target region into random-sized small regions if 891b9a6ac4eSSeongJae Park * current total number of the regions is equal or smaller than half of the 892b9a6ac4eSSeongJae Park * user-specified maximum number of regions. This is for maximizing the 893b9a6ac4eSSeongJae Park * monitoring accuracy under the dynamically changeable access patterns. If a 894b9a6ac4eSSeongJae Park * split was unnecessarily made, later 'kdamond_merge_regions()' will revert 895b9a6ac4eSSeongJae Park * it. 896b9a6ac4eSSeongJae Park */ 897b9a6ac4eSSeongJae Park static void kdamond_split_regions(struct damon_ctx *ctx) 898b9a6ac4eSSeongJae Park { 899b9a6ac4eSSeongJae Park struct damon_target *t; 900b9a6ac4eSSeongJae Park unsigned int nr_regions = 0; 901b9a6ac4eSSeongJae Park static unsigned int last_nr_regions; 902b9a6ac4eSSeongJae Park int nr_subregions = 2; 903b9a6ac4eSSeongJae Park 904b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 905b9a6ac4eSSeongJae Park nr_regions += damon_nr_regions(t); 906b9a6ac4eSSeongJae Park 907b9a6ac4eSSeongJae Park if (nr_regions > ctx->max_nr_regions / 2) 908b9a6ac4eSSeongJae Park return; 909b9a6ac4eSSeongJae Park 910b9a6ac4eSSeongJae Park /* Maybe the middle of the region has different access frequency */ 911b9a6ac4eSSeongJae Park if (last_nr_regions == nr_regions && 912b9a6ac4eSSeongJae Park nr_regions < ctx->max_nr_regions / 3) 913b9a6ac4eSSeongJae Park nr_subregions = 3; 914b9a6ac4eSSeongJae Park 915b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 916b9a6ac4eSSeongJae Park damon_split_regions_of(ctx, t, nr_subregions); 917b9a6ac4eSSeongJae Park 918b9a6ac4eSSeongJae Park last_nr_regions = nr_regions; 919b9a6ac4eSSeongJae Park } 920b9a6ac4eSSeongJae Park 921f23b8eeeSSeongJae Park /* 922f7d911c3SSeongJae Park * Check whether it is time to check and apply the operations-related data 923f7d911c3SSeongJae Park * structures. 9242224d848SSeongJae Park * 9252224d848SSeongJae Park * Returns true if it is. 9262224d848SSeongJae Park */ 927f7d911c3SSeongJae Park static bool kdamond_need_update_operations(struct damon_ctx *ctx) 9282224d848SSeongJae Park { 929f7d911c3SSeongJae Park return damon_check_reset_time_interval(&ctx->last_ops_update, 930f7d911c3SSeongJae Park ctx->ops_update_interval); 9312224d848SSeongJae Park } 9322224d848SSeongJae Park 9332224d848SSeongJae Park /* 9342224d848SSeongJae Park * Check whether current monitoring should be stopped 9352224d848SSeongJae Park * 9362224d848SSeongJae Park * The monitoring is stopped when either the user requested to stop, or all 9372224d848SSeongJae Park * monitoring targets are invalid. 9382224d848SSeongJae Park * 9392224d848SSeongJae Park * Returns true if need to stop current monitoring. 9402224d848SSeongJae Park */ 9412224d848SSeongJae Park static bool kdamond_need_stop(struct damon_ctx *ctx) 9422224d848SSeongJae Park { 943f23b8eeeSSeongJae Park struct damon_target *t; 9442224d848SSeongJae Park 9450f91d133SChangbin Du if (kthread_should_stop()) 9462224d848SSeongJae Park return true; 9472224d848SSeongJae Park 948f7d911c3SSeongJae Park if (!ctx->ops.target_valid) 9492224d848SSeongJae Park return false; 9502224d848SSeongJae Park 951f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 952f7d911c3SSeongJae Park if (ctx->ops.target_valid(t)) 953f23b8eeeSSeongJae Park return false; 954f23b8eeeSSeongJae Park } 955f23b8eeeSSeongJae Park 956f23b8eeeSSeongJae Park return true; 9572224d848SSeongJae Park } 9582224d848SSeongJae Park 959ee801b7dSSeongJae Park static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric) 960ee801b7dSSeongJae Park { 961ee801b7dSSeongJae Park struct sysinfo i; 962ee801b7dSSeongJae Park 963ee801b7dSSeongJae Park switch (metric) { 964ee801b7dSSeongJae Park case DAMOS_WMARK_FREE_MEM_RATE: 965ee801b7dSSeongJae Park si_meminfo(&i); 966ee801b7dSSeongJae Park return i.freeram * 1000 / i.totalram; 967ee801b7dSSeongJae Park default: 968ee801b7dSSeongJae Park break; 969ee801b7dSSeongJae Park } 970ee801b7dSSeongJae Park return -EINVAL; 971ee801b7dSSeongJae Park } 972ee801b7dSSeongJae Park 973ee801b7dSSeongJae Park /* 974ee801b7dSSeongJae Park * Returns zero if the scheme is active. Else, returns time to wait for next 975ee801b7dSSeongJae Park * watermark check in micro-seconds. 976ee801b7dSSeongJae Park */ 977ee801b7dSSeongJae Park static unsigned long damos_wmark_wait_us(struct damos *scheme) 978ee801b7dSSeongJae Park { 979ee801b7dSSeongJae Park unsigned long metric; 980ee801b7dSSeongJae Park 981ee801b7dSSeongJae Park if (scheme->wmarks.metric == DAMOS_WMARK_NONE) 982ee801b7dSSeongJae Park return 0; 983ee801b7dSSeongJae Park 984ee801b7dSSeongJae Park metric = damos_wmark_metric_value(scheme->wmarks.metric); 985ee801b7dSSeongJae Park /* higher than high watermark or lower than low watermark */ 986ee801b7dSSeongJae Park if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { 987ee801b7dSSeongJae Park if (scheme->wmarks.activated) 98801078655SColin Ian King pr_debug("deactivate a scheme (%d) for %s wmark\n", 989ee801b7dSSeongJae Park scheme->action, 990ee801b7dSSeongJae Park metric > scheme->wmarks.high ? 991ee801b7dSSeongJae Park "high" : "low"); 992ee801b7dSSeongJae Park scheme->wmarks.activated = false; 993ee801b7dSSeongJae Park return scheme->wmarks.interval; 994ee801b7dSSeongJae Park } 995ee801b7dSSeongJae Park 996ee801b7dSSeongJae Park /* inactive and higher than middle watermark */ 997ee801b7dSSeongJae Park if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) && 998ee801b7dSSeongJae Park !scheme->wmarks.activated) 999ee801b7dSSeongJae Park return scheme->wmarks.interval; 1000ee801b7dSSeongJae Park 1001ee801b7dSSeongJae Park if (!scheme->wmarks.activated) 1002ee801b7dSSeongJae Park pr_debug("activate a scheme (%d)\n", scheme->action); 1003ee801b7dSSeongJae Park scheme->wmarks.activated = true; 1004ee801b7dSSeongJae Park return 0; 1005ee801b7dSSeongJae Park } 1006ee801b7dSSeongJae Park 1007ee801b7dSSeongJae Park static void kdamond_usleep(unsigned long usecs) 1008ee801b7dSSeongJae Park { 10094de46a30SSeongJae Park /* See Documentation/timers/timers-howto.rst for the thresholds */ 10104de46a30SSeongJae Park if (usecs > 20 * USEC_PER_MSEC) 101170e92748SSeongJae Park schedule_timeout_idle(usecs_to_jiffies(usecs)); 1012ee801b7dSSeongJae Park else 101370e92748SSeongJae Park usleep_idle_range(usecs, usecs + 1); 1014ee801b7dSSeongJae Park } 1015ee801b7dSSeongJae Park 1016ee801b7dSSeongJae Park /* Returns negative error code if it's not activated but should return */ 1017ee801b7dSSeongJae Park static int kdamond_wait_activation(struct damon_ctx *ctx) 1018ee801b7dSSeongJae Park { 1019ee801b7dSSeongJae Park struct damos *s; 1020ee801b7dSSeongJae Park unsigned long wait_time; 1021ee801b7dSSeongJae Park unsigned long min_wait_time = 0; 1022ee801b7dSSeongJae Park 1023ee801b7dSSeongJae Park while (!kdamond_need_stop(ctx)) { 1024ee801b7dSSeongJae Park damon_for_each_scheme(s, ctx) { 1025ee801b7dSSeongJae Park wait_time = damos_wmark_wait_us(s); 1026ee801b7dSSeongJae Park if (!min_wait_time || wait_time < min_wait_time) 1027ee801b7dSSeongJae Park min_wait_time = wait_time; 1028ee801b7dSSeongJae Park } 1029ee801b7dSSeongJae Park if (!min_wait_time) 1030ee801b7dSSeongJae Park return 0; 1031ee801b7dSSeongJae Park 1032ee801b7dSSeongJae Park kdamond_usleep(min_wait_time); 1033ee801b7dSSeongJae Park } 1034ee801b7dSSeongJae Park return -EBUSY; 1035ee801b7dSSeongJae Park } 1036ee801b7dSSeongJae Park 10372224d848SSeongJae Park /* 10382224d848SSeongJae Park * The monitoring daemon that runs as a kernel thread 10392224d848SSeongJae Park */ 10402224d848SSeongJae Park static int kdamond_fn(void *data) 10412224d848SSeongJae Park { 10422224d848SSeongJae Park struct damon_ctx *ctx = (struct damon_ctx *)data; 1043f23b8eeeSSeongJae Park struct damon_target *t; 1044f23b8eeeSSeongJae Park struct damon_region *r, *next; 1045b9a6ac4eSSeongJae Park unsigned int max_nr_accesses = 0; 1046b9a6ac4eSSeongJae Park unsigned long sz_limit = 0; 10470f91d133SChangbin Du bool done = false; 10482224d848SSeongJae Park 104942e4cef5SChangbin Du pr_debug("kdamond (%d) starts\n", current->pid); 10502224d848SSeongJae Park 1051f7d911c3SSeongJae Park if (ctx->ops.init) 1052f7d911c3SSeongJae Park ctx->ops.init(ctx); 10532224d848SSeongJae Park if (ctx->callback.before_start && ctx->callback.before_start(ctx)) 10540f91d133SChangbin Du done = true; 10552224d848SSeongJae Park 1056b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 1057b9a6ac4eSSeongJae Park 10580f91d133SChangbin Du while (!kdamond_need_stop(ctx) && !done) { 1059ee801b7dSSeongJae Park if (kdamond_wait_activation(ctx)) 1060ee801b7dSSeongJae Park continue; 1061ee801b7dSSeongJae Park 1062f7d911c3SSeongJae Park if (ctx->ops.prepare_access_checks) 1063f7d911c3SSeongJae Park ctx->ops.prepare_access_checks(ctx); 10642224d848SSeongJae Park if (ctx->callback.after_sampling && 10652224d848SSeongJae Park ctx->callback.after_sampling(ctx)) 10660f91d133SChangbin Du done = true; 10672224d848SSeongJae Park 106870e92748SSeongJae Park kdamond_usleep(ctx->sample_interval); 10692224d848SSeongJae Park 1070f7d911c3SSeongJae Park if (ctx->ops.check_accesses) 1071f7d911c3SSeongJae Park max_nr_accesses = ctx->ops.check_accesses(ctx); 10722224d848SSeongJae Park 10732224d848SSeongJae Park if (kdamond_aggregate_interval_passed(ctx)) { 1074b9a6ac4eSSeongJae Park kdamond_merge_regions(ctx, 1075b9a6ac4eSSeongJae Park max_nr_accesses / 10, 1076b9a6ac4eSSeongJae Park sz_limit); 10772224d848SSeongJae Park if (ctx->callback.after_aggregation && 10782224d848SSeongJae Park ctx->callback.after_aggregation(ctx)) 10790f91d133SChangbin Du done = true; 10801f366e42SSeongJae Park kdamond_apply_schemes(ctx); 1081f23b8eeeSSeongJae Park kdamond_reset_aggregated(ctx); 1082b9a6ac4eSSeongJae Park kdamond_split_regions(ctx); 1083f7d911c3SSeongJae Park if (ctx->ops.reset_aggregated) 1084f7d911c3SSeongJae Park ctx->ops.reset_aggregated(ctx); 10852224d848SSeongJae Park } 10862224d848SSeongJae Park 1087f7d911c3SSeongJae Park if (kdamond_need_update_operations(ctx)) { 1088f7d911c3SSeongJae Park if (ctx->ops.update) 1089f7d911c3SSeongJae Park ctx->ops.update(ctx); 1090b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 10912224d848SSeongJae Park } 10922224d848SSeongJae Park } 1093f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 1094f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 1095b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 1096f23b8eeeSSeongJae Park } 10972224d848SSeongJae Park 10980f91d133SChangbin Du if (ctx->callback.before_terminate) 10990f91d133SChangbin Du ctx->callback.before_terminate(ctx); 1100f7d911c3SSeongJae Park if (ctx->ops.cleanup) 1101f7d911c3SSeongJae Park ctx->ops.cleanup(ctx); 11022224d848SSeongJae Park 110342e4cef5SChangbin Du pr_debug("kdamond (%d) finishes\n", current->pid); 11042224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 11052224d848SSeongJae Park ctx->kdamond = NULL; 11062224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 11072224d848SSeongJae Park 11082224d848SSeongJae Park mutex_lock(&damon_lock); 11092224d848SSeongJae Park nr_running_ctxs--; 1110*8b9b0d33SSeongJae Park if (!nr_running_ctxs && running_exclusive_ctxs) 1111*8b9b0d33SSeongJae Park running_exclusive_ctxs = false; 11122224d848SSeongJae Park mutex_unlock(&damon_lock); 11132224d848SSeongJae Park 11145f7fe2b9SChangbin Du return 0; 11152224d848SSeongJae Park } 111617ccae8bSSeongJae Park 111717ccae8bSSeongJae Park #include "core-test.h" 1118