12224d848SSeongJae Park // SPDX-License-Identifier: GPL-2.0 22224d848SSeongJae Park /* 32224d848SSeongJae Park * Data Access Monitor 42224d848SSeongJae Park * 52224d848SSeongJae Park * Author: SeongJae Park <sjpark@amazon.de> 62224d848SSeongJae Park */ 72224d848SSeongJae Park 82224d848SSeongJae Park #define pr_fmt(fmt) "damon: " fmt 92224d848SSeongJae Park 102224d848SSeongJae Park #include <linux/damon.h> 112224d848SSeongJae Park #include <linux/delay.h> 122224d848SSeongJae Park #include <linux/kthread.h> 13ee801b7dSSeongJae Park #include <linux/mm.h> 142224d848SSeongJae Park #include <linux/slab.h> 1538683e00SSeongJae Park #include <linux/string.h> 162224d848SSeongJae Park 172fcb9362SSeongJae Park #define CREATE_TRACE_POINTS 182fcb9362SSeongJae Park #include <trace/events/damon.h> 192fcb9362SSeongJae Park 2017ccae8bSSeongJae Park #ifdef CONFIG_DAMON_KUNIT_TEST 2117ccae8bSSeongJae Park #undef DAMON_MIN_REGION 2217ccae8bSSeongJae Park #define DAMON_MIN_REGION 1 2317ccae8bSSeongJae Park #endif 2417ccae8bSSeongJae Park 252224d848SSeongJae Park static DEFINE_MUTEX(damon_lock); 262224d848SSeongJae Park static int nr_running_ctxs; 272224d848SSeongJae Park 28f23b8eeeSSeongJae Park /* 29f23b8eeeSSeongJae Park * Construct a damon_region struct 30f23b8eeeSSeongJae Park * 31f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 32f23b8eeeSSeongJae Park */ 33f23b8eeeSSeongJae Park struct damon_region *damon_new_region(unsigned long start, unsigned long end) 34f23b8eeeSSeongJae Park { 35f23b8eeeSSeongJae Park struct damon_region *region; 36f23b8eeeSSeongJae Park 37f23b8eeeSSeongJae Park region = kmalloc(sizeof(*region), GFP_KERNEL); 38f23b8eeeSSeongJae Park if (!region) 39f23b8eeeSSeongJae Park return NULL; 40f23b8eeeSSeongJae Park 41f23b8eeeSSeongJae Park region->ar.start = start; 42f23b8eeeSSeongJae Park region->ar.end = end; 43f23b8eeeSSeongJae Park region->nr_accesses = 0; 44f23b8eeeSSeongJae Park INIT_LIST_HEAD(®ion->list); 45f23b8eeeSSeongJae Park 46fda504faSSeongJae Park region->age = 0; 47fda504faSSeongJae Park region->last_nr_accesses = 0; 48fda504faSSeongJae Park 49f23b8eeeSSeongJae Park return region; 50f23b8eeeSSeongJae Park } 51f23b8eeeSSeongJae Park 52f23b8eeeSSeongJae Park void damon_add_region(struct damon_region *r, struct damon_target *t) 53f23b8eeeSSeongJae Park { 54f23b8eeeSSeongJae Park list_add_tail(&r->list, &t->regions_list); 55b9a6ac4eSSeongJae Park t->nr_regions++; 56f23b8eeeSSeongJae Park } 57f23b8eeeSSeongJae Park 58b9a6ac4eSSeongJae Park static void damon_del_region(struct damon_region *r, struct damon_target *t) 59f23b8eeeSSeongJae Park { 60f23b8eeeSSeongJae Park list_del(&r->list); 61b9a6ac4eSSeongJae Park t->nr_regions--; 62f23b8eeeSSeongJae Park } 63f23b8eeeSSeongJae Park 64f23b8eeeSSeongJae Park static void damon_free_region(struct damon_region *r) 65f23b8eeeSSeongJae Park { 66f23b8eeeSSeongJae Park kfree(r); 67f23b8eeeSSeongJae Park } 68f23b8eeeSSeongJae Park 69b9a6ac4eSSeongJae Park void damon_destroy_region(struct damon_region *r, struct damon_target *t) 70f23b8eeeSSeongJae Park { 71b9a6ac4eSSeongJae Park damon_del_region(r, t); 72f23b8eeeSSeongJae Park damon_free_region(r); 73f23b8eeeSSeongJae Park } 74f23b8eeeSSeongJae Park 751f366e42SSeongJae Park struct damos *damon_new_scheme( 761f366e42SSeongJae Park unsigned long min_sz_region, unsigned long max_sz_region, 771f366e42SSeongJae Park unsigned int min_nr_accesses, unsigned int max_nr_accesses, 781f366e42SSeongJae Park unsigned int min_age_region, unsigned int max_age_region, 79ee801b7dSSeongJae Park enum damos_action action, struct damos_quota *quota, 80ee801b7dSSeongJae Park struct damos_watermarks *wmarks) 811f366e42SSeongJae Park { 821f366e42SSeongJae Park struct damos *scheme; 831f366e42SSeongJae Park 841f366e42SSeongJae Park scheme = kmalloc(sizeof(*scheme), GFP_KERNEL); 851f366e42SSeongJae Park if (!scheme) 861f366e42SSeongJae Park return NULL; 871f366e42SSeongJae Park scheme->min_sz_region = min_sz_region; 881f366e42SSeongJae Park scheme->max_sz_region = max_sz_region; 891f366e42SSeongJae Park scheme->min_nr_accesses = min_nr_accesses; 901f366e42SSeongJae Park scheme->max_nr_accesses = max_nr_accesses; 911f366e42SSeongJae Park scheme->min_age_region = min_age_region; 921f366e42SSeongJae Park scheme->max_age_region = max_age_region; 931f366e42SSeongJae Park scheme->action = action; 940e92c2eeSSeongJae Park scheme->stat = (struct damos_stat){}; 951f366e42SSeongJae Park INIT_LIST_HEAD(&scheme->list); 961f366e42SSeongJae Park 971cd24303SSeongJae Park scheme->quota.ms = quota->ms; 982b8a248dSSeongJae Park scheme->quota.sz = quota->sz; 992b8a248dSSeongJae Park scheme->quota.reset_interval = quota->reset_interval; 10038683e00SSeongJae Park scheme->quota.weight_sz = quota->weight_sz; 10138683e00SSeongJae Park scheme->quota.weight_nr_accesses = quota->weight_nr_accesses; 10238683e00SSeongJae Park scheme->quota.weight_age = quota->weight_age; 1031cd24303SSeongJae Park scheme->quota.total_charged_sz = 0; 1041cd24303SSeongJae Park scheme->quota.total_charged_ns = 0; 1051cd24303SSeongJae Park scheme->quota.esz = 0; 1062b8a248dSSeongJae Park scheme->quota.charged_sz = 0; 1072b8a248dSSeongJae Park scheme->quota.charged_from = 0; 10850585192SSeongJae Park scheme->quota.charge_target_from = NULL; 10950585192SSeongJae Park scheme->quota.charge_addr_from = 0; 1102b8a248dSSeongJae Park 111ee801b7dSSeongJae Park scheme->wmarks.metric = wmarks->metric; 112ee801b7dSSeongJae Park scheme->wmarks.interval = wmarks->interval; 113ee801b7dSSeongJae Park scheme->wmarks.high = wmarks->high; 114ee801b7dSSeongJae Park scheme->wmarks.mid = wmarks->mid; 115ee801b7dSSeongJae Park scheme->wmarks.low = wmarks->low; 116ee801b7dSSeongJae Park scheme->wmarks.activated = true; 117ee801b7dSSeongJae Park 1181f366e42SSeongJae Park return scheme; 1191f366e42SSeongJae Park } 1201f366e42SSeongJae Park 1211f366e42SSeongJae Park void damon_add_scheme(struct damon_ctx *ctx, struct damos *s) 1221f366e42SSeongJae Park { 1231f366e42SSeongJae Park list_add_tail(&s->list, &ctx->schemes); 1241f366e42SSeongJae Park } 1251f366e42SSeongJae Park 1261f366e42SSeongJae Park static void damon_del_scheme(struct damos *s) 1271f366e42SSeongJae Park { 1281f366e42SSeongJae Park list_del(&s->list); 1291f366e42SSeongJae Park } 1301f366e42SSeongJae Park 1311f366e42SSeongJae Park static void damon_free_scheme(struct damos *s) 1321f366e42SSeongJae Park { 1331f366e42SSeongJae Park kfree(s); 1341f366e42SSeongJae Park } 1351f366e42SSeongJae Park 1361f366e42SSeongJae Park void damon_destroy_scheme(struct damos *s) 1371f366e42SSeongJae Park { 1381f366e42SSeongJae Park damon_del_scheme(s); 1391f366e42SSeongJae Park damon_free_scheme(s); 1401f366e42SSeongJae Park } 1411f366e42SSeongJae Park 142f23b8eeeSSeongJae Park /* 143f23b8eeeSSeongJae Park * Construct a damon_target struct 144f23b8eeeSSeongJae Park * 145f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 146f23b8eeeSSeongJae Park */ 147*1971bd63SSeongJae Park struct damon_target *damon_new_target(void) 148f23b8eeeSSeongJae Park { 149f23b8eeeSSeongJae Park struct damon_target *t; 150f23b8eeeSSeongJae Park 151f23b8eeeSSeongJae Park t = kmalloc(sizeof(*t), GFP_KERNEL); 152f23b8eeeSSeongJae Park if (!t) 153f23b8eeeSSeongJae Park return NULL; 154f23b8eeeSSeongJae Park 155*1971bd63SSeongJae Park t->pid = NULL; 156b9a6ac4eSSeongJae Park t->nr_regions = 0; 157f23b8eeeSSeongJae Park INIT_LIST_HEAD(&t->regions_list); 158f23b8eeeSSeongJae Park 159f23b8eeeSSeongJae Park return t; 160f23b8eeeSSeongJae Park } 161f23b8eeeSSeongJae Park 162f23b8eeeSSeongJae Park void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) 163f23b8eeeSSeongJae Park { 164b9a6ac4eSSeongJae Park list_add_tail(&t->list, &ctx->adaptive_targets); 165f23b8eeeSSeongJae Park } 166f23b8eeeSSeongJae Park 167b5ca3e83SXin Hao bool damon_targets_empty(struct damon_ctx *ctx) 168b5ca3e83SXin Hao { 169b5ca3e83SXin Hao return list_empty(&ctx->adaptive_targets); 170b5ca3e83SXin Hao } 171b5ca3e83SXin Hao 172f23b8eeeSSeongJae Park static void damon_del_target(struct damon_target *t) 173f23b8eeeSSeongJae Park { 174f23b8eeeSSeongJae Park list_del(&t->list); 175f23b8eeeSSeongJae Park } 176f23b8eeeSSeongJae Park 177f23b8eeeSSeongJae Park void damon_free_target(struct damon_target *t) 178f23b8eeeSSeongJae Park { 179f23b8eeeSSeongJae Park struct damon_region *r, *next; 180f23b8eeeSSeongJae Park 181f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 182f23b8eeeSSeongJae Park damon_free_region(r); 183f23b8eeeSSeongJae Park kfree(t); 184f23b8eeeSSeongJae Park } 185f23b8eeeSSeongJae Park 186f23b8eeeSSeongJae Park void damon_destroy_target(struct damon_target *t) 187f23b8eeeSSeongJae Park { 188f23b8eeeSSeongJae Park damon_del_target(t); 189f23b8eeeSSeongJae Park damon_free_target(t); 190f23b8eeeSSeongJae Park } 191f23b8eeeSSeongJae Park 192b9a6ac4eSSeongJae Park unsigned int damon_nr_regions(struct damon_target *t) 193b9a6ac4eSSeongJae Park { 194b9a6ac4eSSeongJae Park return t->nr_regions; 195b9a6ac4eSSeongJae Park } 196b9a6ac4eSSeongJae Park 1972224d848SSeongJae Park struct damon_ctx *damon_new_ctx(void) 1982224d848SSeongJae Park { 1992224d848SSeongJae Park struct damon_ctx *ctx; 2002224d848SSeongJae Park 2012224d848SSeongJae Park ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 2022224d848SSeongJae Park if (!ctx) 2032224d848SSeongJae Park return NULL; 2042224d848SSeongJae Park 2052224d848SSeongJae Park ctx->sample_interval = 5 * 1000; 2062224d848SSeongJae Park ctx->aggr_interval = 100 * 1000; 2072224d848SSeongJae Park ctx->primitive_update_interval = 60 * 1000 * 1000; 2082224d848SSeongJae Park 2092224d848SSeongJae Park ktime_get_coarse_ts64(&ctx->last_aggregation); 2102224d848SSeongJae Park ctx->last_primitive_update = ctx->last_aggregation; 2112224d848SSeongJae Park 2122224d848SSeongJae Park mutex_init(&ctx->kdamond_lock); 2132224d848SSeongJae Park 214b9a6ac4eSSeongJae Park ctx->min_nr_regions = 10; 215b9a6ac4eSSeongJae Park ctx->max_nr_regions = 1000; 216b9a6ac4eSSeongJae Park 217b9a6ac4eSSeongJae Park INIT_LIST_HEAD(&ctx->adaptive_targets); 2181f366e42SSeongJae Park INIT_LIST_HEAD(&ctx->schemes); 2192224d848SSeongJae Park 2202224d848SSeongJae Park return ctx; 2212224d848SSeongJae Park } 2222224d848SSeongJae Park 223f23b8eeeSSeongJae Park static void damon_destroy_targets(struct damon_ctx *ctx) 224f23b8eeeSSeongJae Park { 225f23b8eeeSSeongJae Park struct damon_target *t, *next_t; 226f23b8eeeSSeongJae Park 227f23b8eeeSSeongJae Park if (ctx->primitive.cleanup) { 228f23b8eeeSSeongJae Park ctx->primitive.cleanup(ctx); 229f23b8eeeSSeongJae Park return; 230f23b8eeeSSeongJae Park } 231f23b8eeeSSeongJae Park 232f23b8eeeSSeongJae Park damon_for_each_target_safe(t, next_t, ctx) 233f23b8eeeSSeongJae Park damon_destroy_target(t); 234f23b8eeeSSeongJae Park } 235f23b8eeeSSeongJae Park 2362224d848SSeongJae Park void damon_destroy_ctx(struct damon_ctx *ctx) 2372224d848SSeongJae Park { 2381f366e42SSeongJae Park struct damos *s, *next_s; 2391f366e42SSeongJae Park 240f23b8eeeSSeongJae Park damon_destroy_targets(ctx); 2411f366e42SSeongJae Park 2421f366e42SSeongJae Park damon_for_each_scheme_safe(s, next_s, ctx) 2431f366e42SSeongJae Park damon_destroy_scheme(s); 2441f366e42SSeongJae Park 2452224d848SSeongJae Park kfree(ctx); 2462224d848SSeongJae Park } 2472224d848SSeongJae Park 2482224d848SSeongJae Park /** 2492224d848SSeongJae Park * damon_set_attrs() - Set attributes for the monitoring. 2502224d848SSeongJae Park * @ctx: monitoring context 2512224d848SSeongJae Park * @sample_int: time interval between samplings 2522224d848SSeongJae Park * @aggr_int: time interval between aggregations 2532224d848SSeongJae Park * @primitive_upd_int: time interval between monitoring primitive updates 254b9a6ac4eSSeongJae Park * @min_nr_reg: minimal number of regions 255b9a6ac4eSSeongJae Park * @max_nr_reg: maximum number of regions 2562224d848SSeongJae Park * 2572224d848SSeongJae Park * This function should not be called while the kdamond is running. 2582224d848SSeongJae Park * Every time interval is in micro-seconds. 2592224d848SSeongJae Park * 2602224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 2612224d848SSeongJae Park */ 2622224d848SSeongJae Park int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, 263b9a6ac4eSSeongJae Park unsigned long aggr_int, unsigned long primitive_upd_int, 264b9a6ac4eSSeongJae Park unsigned long min_nr_reg, unsigned long max_nr_reg) 2652224d848SSeongJae Park { 2661afaf5cbSSeongJae Park if (min_nr_reg < 3) 267b9a6ac4eSSeongJae Park return -EINVAL; 2681afaf5cbSSeongJae Park if (min_nr_reg > max_nr_reg) 269b9a6ac4eSSeongJae Park return -EINVAL; 270b9a6ac4eSSeongJae Park 2712224d848SSeongJae Park ctx->sample_interval = sample_int; 2722224d848SSeongJae Park ctx->aggr_interval = aggr_int; 2732224d848SSeongJae Park ctx->primitive_update_interval = primitive_upd_int; 274b9a6ac4eSSeongJae Park ctx->min_nr_regions = min_nr_reg; 275b9a6ac4eSSeongJae Park ctx->max_nr_regions = max_nr_reg; 2762224d848SSeongJae Park 2772224d848SSeongJae Park return 0; 2782224d848SSeongJae Park } 2792224d848SSeongJae Park 2804bc05954SSeongJae Park /** 2811f366e42SSeongJae Park * damon_set_schemes() - Set data access monitoring based operation schemes. 2821f366e42SSeongJae Park * @ctx: monitoring context 2831f366e42SSeongJae Park * @schemes: array of the schemes 2841f366e42SSeongJae Park * @nr_schemes: number of entries in @schemes 2851f366e42SSeongJae Park * 2861f366e42SSeongJae Park * This function should not be called while the kdamond of the context is 2871f366e42SSeongJae Park * running. 2881f366e42SSeongJae Park * 2891f366e42SSeongJae Park * Return: 0 if success, or negative error code otherwise. 2901f366e42SSeongJae Park */ 2911f366e42SSeongJae Park int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, 2921f366e42SSeongJae Park ssize_t nr_schemes) 2931f366e42SSeongJae Park { 2941f366e42SSeongJae Park struct damos *s, *next; 2951f366e42SSeongJae Park ssize_t i; 2961f366e42SSeongJae Park 2971f366e42SSeongJae Park damon_for_each_scheme_safe(s, next, ctx) 2981f366e42SSeongJae Park damon_destroy_scheme(s); 2991f366e42SSeongJae Park for (i = 0; i < nr_schemes; i++) 3001f366e42SSeongJae Park damon_add_scheme(ctx, schemes[i]); 3011f366e42SSeongJae Park return 0; 3021f366e42SSeongJae Park } 3031f366e42SSeongJae Park 3041f366e42SSeongJae Park /** 3054bc05954SSeongJae Park * damon_nr_running_ctxs() - Return number of currently running contexts. 3064bc05954SSeongJae Park */ 3074bc05954SSeongJae Park int damon_nr_running_ctxs(void) 3084bc05954SSeongJae Park { 3094bc05954SSeongJae Park int nr_ctxs; 3104bc05954SSeongJae Park 3114bc05954SSeongJae Park mutex_lock(&damon_lock); 3124bc05954SSeongJae Park nr_ctxs = nr_running_ctxs; 3134bc05954SSeongJae Park mutex_unlock(&damon_lock); 3144bc05954SSeongJae Park 3154bc05954SSeongJae Park return nr_ctxs; 3164bc05954SSeongJae Park } 3174bc05954SSeongJae Park 318b9a6ac4eSSeongJae Park /* Returns the size upper limit for each monitoring region */ 319b9a6ac4eSSeongJae Park static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) 320b9a6ac4eSSeongJae Park { 321b9a6ac4eSSeongJae Park struct damon_target *t; 322b9a6ac4eSSeongJae Park struct damon_region *r; 323b9a6ac4eSSeongJae Park unsigned long sz = 0; 324b9a6ac4eSSeongJae Park 325b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) { 326b9a6ac4eSSeongJae Park damon_for_each_region(r, t) 327b9a6ac4eSSeongJae Park sz += r->ar.end - r->ar.start; 328b9a6ac4eSSeongJae Park } 329b9a6ac4eSSeongJae Park 330b9a6ac4eSSeongJae Park if (ctx->min_nr_regions) 331b9a6ac4eSSeongJae Park sz /= ctx->min_nr_regions; 332b9a6ac4eSSeongJae Park if (sz < DAMON_MIN_REGION) 333b9a6ac4eSSeongJae Park sz = DAMON_MIN_REGION; 334b9a6ac4eSSeongJae Park 335b9a6ac4eSSeongJae Park return sz; 336b9a6ac4eSSeongJae Park } 337b9a6ac4eSSeongJae Park 3382224d848SSeongJae Park static int kdamond_fn(void *data); 3392224d848SSeongJae Park 3402224d848SSeongJae Park /* 3412224d848SSeongJae Park * __damon_start() - Starts monitoring with given context. 3422224d848SSeongJae Park * @ctx: monitoring context 3432224d848SSeongJae Park * 3442224d848SSeongJae Park * This function should be called while damon_lock is hold. 3452224d848SSeongJae Park * 3462224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 3472224d848SSeongJae Park */ 3482224d848SSeongJae Park static int __damon_start(struct damon_ctx *ctx) 3492224d848SSeongJae Park { 3502224d848SSeongJae Park int err = -EBUSY; 3512224d848SSeongJae Park 3522224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 3532224d848SSeongJae Park if (!ctx->kdamond) { 3542224d848SSeongJae Park err = 0; 3552224d848SSeongJae Park ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d", 3562224d848SSeongJae Park nr_running_ctxs); 3572224d848SSeongJae Park if (IS_ERR(ctx->kdamond)) { 3582224d848SSeongJae Park err = PTR_ERR(ctx->kdamond); 3597ec1992bSColin Ian King ctx->kdamond = NULL; 3602224d848SSeongJae Park } 3612224d848SSeongJae Park } 3622224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 3632224d848SSeongJae Park 3642224d848SSeongJae Park return err; 3652224d848SSeongJae Park } 3662224d848SSeongJae Park 3672224d848SSeongJae Park /** 3682224d848SSeongJae Park * damon_start() - Starts the monitorings for a given group of contexts. 3692224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to start monitoring 3702224d848SSeongJae Park * @nr_ctxs: size of @ctxs 3712224d848SSeongJae Park * 3722224d848SSeongJae Park * This function starts a group of monitoring threads for a group of monitoring 3732224d848SSeongJae Park * contexts. One thread per each context is created and run in parallel. The 3742224d848SSeongJae Park * caller should handle synchronization between the threads by itself. If a 3752224d848SSeongJae Park * group of threads that created by other 'damon_start()' call is currently 3762224d848SSeongJae Park * running, this function does nothing but returns -EBUSY. 3772224d848SSeongJae Park * 3782224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 3792224d848SSeongJae Park */ 3802224d848SSeongJae Park int damon_start(struct damon_ctx **ctxs, int nr_ctxs) 3812224d848SSeongJae Park { 3822224d848SSeongJae Park int i; 3832224d848SSeongJae Park int err = 0; 3842224d848SSeongJae Park 3852224d848SSeongJae Park mutex_lock(&damon_lock); 3862224d848SSeongJae Park if (nr_running_ctxs) { 3872224d848SSeongJae Park mutex_unlock(&damon_lock); 3882224d848SSeongJae Park return -EBUSY; 3892224d848SSeongJae Park } 3902224d848SSeongJae Park 3912224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 3922224d848SSeongJae Park err = __damon_start(ctxs[i]); 3932224d848SSeongJae Park if (err) 3942224d848SSeongJae Park break; 3952224d848SSeongJae Park nr_running_ctxs++; 3962224d848SSeongJae Park } 3972224d848SSeongJae Park mutex_unlock(&damon_lock); 3982224d848SSeongJae Park 3992224d848SSeongJae Park return err; 4002224d848SSeongJae Park } 4012224d848SSeongJae Park 4022224d848SSeongJae Park /* 4032224d848SSeongJae Park * __damon_stop() - Stops monitoring of given context. 4042224d848SSeongJae Park * @ctx: monitoring context 4052224d848SSeongJae Park * 4062224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4072224d848SSeongJae Park */ 4082224d848SSeongJae Park static int __damon_stop(struct damon_ctx *ctx) 4092224d848SSeongJae Park { 4100f91d133SChangbin Du struct task_struct *tsk; 4110f91d133SChangbin Du 4122224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 4130f91d133SChangbin Du tsk = ctx->kdamond; 4140f91d133SChangbin Du if (tsk) { 4150f91d133SChangbin Du get_task_struct(tsk); 4162224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 4170f91d133SChangbin Du kthread_stop(tsk); 4180f91d133SChangbin Du put_task_struct(tsk); 4192224d848SSeongJae Park return 0; 4202224d848SSeongJae Park } 4212224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 4222224d848SSeongJae Park 4232224d848SSeongJae Park return -EPERM; 4242224d848SSeongJae Park } 4252224d848SSeongJae Park 4262224d848SSeongJae Park /** 4272224d848SSeongJae Park * damon_stop() - Stops the monitorings for a given group of contexts. 4282224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to stop monitoring 4292224d848SSeongJae Park * @nr_ctxs: size of @ctxs 4302224d848SSeongJae Park * 4312224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4322224d848SSeongJae Park */ 4332224d848SSeongJae Park int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) 4342224d848SSeongJae Park { 4352224d848SSeongJae Park int i, err = 0; 4362224d848SSeongJae Park 4372224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 4382224d848SSeongJae Park /* nr_running_ctxs is decremented in kdamond_fn */ 4392224d848SSeongJae Park err = __damon_stop(ctxs[i]); 4402224d848SSeongJae Park if (err) 4412224d848SSeongJae Park return err; 4422224d848SSeongJae Park } 4432224d848SSeongJae Park 4442224d848SSeongJae Park return err; 4452224d848SSeongJae Park } 4462224d848SSeongJae Park 4472224d848SSeongJae Park /* 4482224d848SSeongJae Park * damon_check_reset_time_interval() - Check if a time interval is elapsed. 4492224d848SSeongJae Park * @baseline: the time to check whether the interval has elapsed since 4502224d848SSeongJae Park * @interval: the time interval (microseconds) 4512224d848SSeongJae Park * 4522224d848SSeongJae Park * See whether the given time interval has passed since the given baseline 4532224d848SSeongJae Park * time. If so, it also updates the baseline to current time for next check. 4542224d848SSeongJae Park * 4552224d848SSeongJae Park * Return: true if the time interval has passed, or false otherwise. 4562224d848SSeongJae Park */ 4572224d848SSeongJae Park static bool damon_check_reset_time_interval(struct timespec64 *baseline, 4582224d848SSeongJae Park unsigned long interval) 4592224d848SSeongJae Park { 4602224d848SSeongJae Park struct timespec64 now; 4612224d848SSeongJae Park 4622224d848SSeongJae Park ktime_get_coarse_ts64(&now); 4632224d848SSeongJae Park if ((timespec64_to_ns(&now) - timespec64_to_ns(baseline)) < 4642224d848SSeongJae Park interval * 1000) 4652224d848SSeongJae Park return false; 4662224d848SSeongJae Park *baseline = now; 4672224d848SSeongJae Park return true; 4682224d848SSeongJae Park } 4692224d848SSeongJae Park 4702224d848SSeongJae Park /* 4712224d848SSeongJae Park * Check whether it is time to flush the aggregated information 4722224d848SSeongJae Park */ 4732224d848SSeongJae Park static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) 4742224d848SSeongJae Park { 4752224d848SSeongJae Park return damon_check_reset_time_interval(&ctx->last_aggregation, 4762224d848SSeongJae Park ctx->aggr_interval); 4772224d848SSeongJae Park } 4782224d848SSeongJae Park 4792224d848SSeongJae Park /* 480f23b8eeeSSeongJae Park * Reset the aggregated monitoring results ('nr_accesses' of each region). 481f23b8eeeSSeongJae Park */ 482f23b8eeeSSeongJae Park static void kdamond_reset_aggregated(struct damon_ctx *c) 483f23b8eeeSSeongJae Park { 484f23b8eeeSSeongJae Park struct damon_target *t; 48576fd0285SSeongJae Park unsigned int ti = 0; /* target's index */ 486f23b8eeeSSeongJae Park 487f23b8eeeSSeongJae Park damon_for_each_target(t, c) { 488f23b8eeeSSeongJae Park struct damon_region *r; 489f23b8eeeSSeongJae Park 4902fcb9362SSeongJae Park damon_for_each_region(r, t) { 49176fd0285SSeongJae Park trace_damon_aggregated(t, ti, r, damon_nr_regions(t)); 492fda504faSSeongJae Park r->last_nr_accesses = r->nr_accesses; 493f23b8eeeSSeongJae Park r->nr_accesses = 0; 494f23b8eeeSSeongJae Park } 49576fd0285SSeongJae Park ti++; 496f23b8eeeSSeongJae Park } 4972fcb9362SSeongJae Park } 498f23b8eeeSSeongJae Park 4992b8a248dSSeongJae Park static void damon_split_region_at(struct damon_ctx *ctx, 5002b8a248dSSeongJae Park struct damon_target *t, struct damon_region *r, 5012b8a248dSSeongJae Park unsigned long sz_r); 5022b8a248dSSeongJae Park 50338683e00SSeongJae Park static bool __damos_valid_target(struct damon_region *r, struct damos *s) 50438683e00SSeongJae Park { 50538683e00SSeongJae Park unsigned long sz; 50638683e00SSeongJae Park 50738683e00SSeongJae Park sz = r->ar.end - r->ar.start; 50838683e00SSeongJae Park return s->min_sz_region <= sz && sz <= s->max_sz_region && 50938683e00SSeongJae Park s->min_nr_accesses <= r->nr_accesses && 51038683e00SSeongJae Park r->nr_accesses <= s->max_nr_accesses && 51138683e00SSeongJae Park s->min_age_region <= r->age && r->age <= s->max_age_region; 51238683e00SSeongJae Park } 51338683e00SSeongJae Park 51438683e00SSeongJae Park static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, 51538683e00SSeongJae Park struct damon_region *r, struct damos *s) 51638683e00SSeongJae Park { 51738683e00SSeongJae Park bool ret = __damos_valid_target(r, s); 51838683e00SSeongJae Park 51938683e00SSeongJae Park if (!ret || !s->quota.esz || !c->primitive.get_scheme_score) 52038683e00SSeongJae Park return ret; 52138683e00SSeongJae Park 52238683e00SSeongJae Park return c->primitive.get_scheme_score(c, t, r, s) >= s->quota.min_score; 52338683e00SSeongJae Park } 52438683e00SSeongJae Park 5251f366e42SSeongJae Park static void damon_do_apply_schemes(struct damon_ctx *c, 5261f366e42SSeongJae Park struct damon_target *t, 5271f366e42SSeongJae Park struct damon_region *r) 5281f366e42SSeongJae Park { 5291f366e42SSeongJae Park struct damos *s; 5301f366e42SSeongJae Park 5311f366e42SSeongJae Park damon_for_each_scheme(s, c) { 5322b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 5332b8a248dSSeongJae Park unsigned long sz = r->ar.end - r->ar.start; 5341cd24303SSeongJae Park struct timespec64 begin, end; 5350e92c2eeSSeongJae Park unsigned long sz_applied = 0; 5362b8a248dSSeongJae Park 537ee801b7dSSeongJae Park if (!s->wmarks.activated) 538ee801b7dSSeongJae Park continue; 539ee801b7dSSeongJae Park 5402b8a248dSSeongJae Park /* Check the quota */ 5411cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 5422b8a248dSSeongJae Park continue; 5432b8a248dSSeongJae Park 54450585192SSeongJae Park /* Skip previously charged regions */ 54550585192SSeongJae Park if (quota->charge_target_from) { 54650585192SSeongJae Park if (t != quota->charge_target_from) 54750585192SSeongJae Park continue; 54850585192SSeongJae Park if (r == damon_last_region(t)) { 54950585192SSeongJae Park quota->charge_target_from = NULL; 55050585192SSeongJae Park quota->charge_addr_from = 0; 55150585192SSeongJae Park continue; 55250585192SSeongJae Park } 55350585192SSeongJae Park if (quota->charge_addr_from && 55450585192SSeongJae Park r->ar.end <= quota->charge_addr_from) 55550585192SSeongJae Park continue; 55650585192SSeongJae Park 55750585192SSeongJae Park if (quota->charge_addr_from && r->ar.start < 55850585192SSeongJae Park quota->charge_addr_from) { 55950585192SSeongJae Park sz = ALIGN_DOWN(quota->charge_addr_from - 56050585192SSeongJae Park r->ar.start, DAMON_MIN_REGION); 56150585192SSeongJae Park if (!sz) { 56250585192SSeongJae Park if (r->ar.end - r->ar.start <= 56350585192SSeongJae Park DAMON_MIN_REGION) 56450585192SSeongJae Park continue; 56550585192SSeongJae Park sz = DAMON_MIN_REGION; 56650585192SSeongJae Park } 56750585192SSeongJae Park damon_split_region_at(c, t, r, sz); 56850585192SSeongJae Park r = damon_next_region(r); 56950585192SSeongJae Park sz = r->ar.end - r->ar.start; 57050585192SSeongJae Park } 57150585192SSeongJae Park quota->charge_target_from = NULL; 57250585192SSeongJae Park quota->charge_addr_from = 0; 57350585192SSeongJae Park } 57450585192SSeongJae Park 57538683e00SSeongJae Park if (!damos_valid_target(c, t, r, s)) 5761f366e42SSeongJae Park continue; 5772b8a248dSSeongJae Park 5782b8a248dSSeongJae Park /* Apply the scheme */ 5792b8a248dSSeongJae Park if (c->primitive.apply_scheme) { 5801cd24303SSeongJae Park if (quota->esz && 5811cd24303SSeongJae Park quota->charged_sz + sz > quota->esz) { 5821cd24303SSeongJae Park sz = ALIGN_DOWN(quota->esz - quota->charged_sz, 5832b8a248dSSeongJae Park DAMON_MIN_REGION); 5842b8a248dSSeongJae Park if (!sz) 5852b8a248dSSeongJae Park goto update_stat; 5862b8a248dSSeongJae Park damon_split_region_at(c, t, r, sz); 5872b8a248dSSeongJae Park } 5881cd24303SSeongJae Park ktime_get_coarse_ts64(&begin); 5890e92c2eeSSeongJae Park sz_applied = c->primitive.apply_scheme(c, t, r, s); 5901cd24303SSeongJae Park ktime_get_coarse_ts64(&end); 5911cd24303SSeongJae Park quota->total_charged_ns += timespec64_to_ns(&end) - 5921cd24303SSeongJae Park timespec64_to_ns(&begin); 5932b8a248dSSeongJae Park quota->charged_sz += sz; 5941cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) { 59550585192SSeongJae Park quota->charge_target_from = t; 59650585192SSeongJae Park quota->charge_addr_from = r->ar.end + 1; 59750585192SSeongJae Park } 5982b8a248dSSeongJae Park } 5992f0b548cSSeongJae Park if (s->action != DAMOS_STAT) 6001f366e42SSeongJae Park r->age = 0; 6012b8a248dSSeongJae Park 6022b8a248dSSeongJae Park update_stat: 6030e92c2eeSSeongJae Park s->stat.nr_tried++; 6040e92c2eeSSeongJae Park s->stat.sz_tried += sz; 6050e92c2eeSSeongJae Park if (sz_applied) 6060e92c2eeSSeongJae Park s->stat.nr_applied++; 6070e92c2eeSSeongJae Park s->stat.sz_applied += sz_applied; 6081f366e42SSeongJae Park } 6091f366e42SSeongJae Park } 6101f366e42SSeongJae Park 6111cd24303SSeongJae Park /* Shouldn't be called if quota->ms and quota->sz are zero */ 6121cd24303SSeongJae Park static void damos_set_effective_quota(struct damos_quota *quota) 6131cd24303SSeongJae Park { 6141cd24303SSeongJae Park unsigned long throughput; 6151cd24303SSeongJae Park unsigned long esz; 6161cd24303SSeongJae Park 6171cd24303SSeongJae Park if (!quota->ms) { 6181cd24303SSeongJae Park quota->esz = quota->sz; 6191cd24303SSeongJae Park return; 6201cd24303SSeongJae Park } 6211cd24303SSeongJae Park 6221cd24303SSeongJae Park if (quota->total_charged_ns) 6231cd24303SSeongJae Park throughput = quota->total_charged_sz * 1000000 / 6241cd24303SSeongJae Park quota->total_charged_ns; 6251cd24303SSeongJae Park else 6261cd24303SSeongJae Park throughput = PAGE_SIZE * 1024; 6271cd24303SSeongJae Park esz = throughput * quota->ms; 6281cd24303SSeongJae Park 6291cd24303SSeongJae Park if (quota->sz && quota->sz < esz) 6301cd24303SSeongJae Park esz = quota->sz; 6311cd24303SSeongJae Park quota->esz = esz; 6321cd24303SSeongJae Park } 6331cd24303SSeongJae Park 6341f366e42SSeongJae Park static void kdamond_apply_schemes(struct damon_ctx *c) 6351f366e42SSeongJae Park { 6361f366e42SSeongJae Park struct damon_target *t; 6372b8a248dSSeongJae Park struct damon_region *r, *next_r; 6382b8a248dSSeongJae Park struct damos *s; 6392b8a248dSSeongJae Park 6402b8a248dSSeongJae Park damon_for_each_scheme(s, c) { 6412b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 64238683e00SSeongJae Park unsigned long cumulated_sz; 64338683e00SSeongJae Park unsigned int score, max_score = 0; 6442b8a248dSSeongJae Park 645ee801b7dSSeongJae Park if (!s->wmarks.activated) 646ee801b7dSSeongJae Park continue; 647ee801b7dSSeongJae Park 6481cd24303SSeongJae Park if (!quota->ms && !quota->sz) 6492b8a248dSSeongJae Park continue; 6502b8a248dSSeongJae Park 6512b8a248dSSeongJae Park /* New charge window starts */ 6522b8a248dSSeongJae Park if (time_after_eq(jiffies, quota->charged_from + 6532b8a248dSSeongJae Park msecs_to_jiffies( 6542b8a248dSSeongJae Park quota->reset_interval))) { 6556268eac3SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 6566268eac3SSeongJae Park s->stat.qt_exceeds++; 6571cd24303SSeongJae Park quota->total_charged_sz += quota->charged_sz; 6582b8a248dSSeongJae Park quota->charged_from = jiffies; 6592b8a248dSSeongJae Park quota->charged_sz = 0; 6601cd24303SSeongJae Park damos_set_effective_quota(quota); 6612b8a248dSSeongJae Park } 66238683e00SSeongJae Park 66338683e00SSeongJae Park if (!c->primitive.get_scheme_score) 66438683e00SSeongJae Park continue; 66538683e00SSeongJae Park 66638683e00SSeongJae Park /* Fill up the score histogram */ 66738683e00SSeongJae Park memset(quota->histogram, 0, sizeof(quota->histogram)); 66838683e00SSeongJae Park damon_for_each_target(t, c) { 66938683e00SSeongJae Park damon_for_each_region(r, t) { 67038683e00SSeongJae Park if (!__damos_valid_target(r, s)) 67138683e00SSeongJae Park continue; 67238683e00SSeongJae Park score = c->primitive.get_scheme_score( 67338683e00SSeongJae Park c, t, r, s); 67438683e00SSeongJae Park quota->histogram[score] += 67538683e00SSeongJae Park r->ar.end - r->ar.start; 67638683e00SSeongJae Park if (score > max_score) 67738683e00SSeongJae Park max_score = score; 67838683e00SSeongJae Park } 67938683e00SSeongJae Park } 68038683e00SSeongJae Park 68138683e00SSeongJae Park /* Set the min score limit */ 68238683e00SSeongJae Park for (cumulated_sz = 0, score = max_score; ; score--) { 68338683e00SSeongJae Park cumulated_sz += quota->histogram[score]; 68438683e00SSeongJae Park if (cumulated_sz >= quota->esz || !score) 68538683e00SSeongJae Park break; 68638683e00SSeongJae Park } 68738683e00SSeongJae Park quota->min_score = score; 6882b8a248dSSeongJae Park } 6891f366e42SSeongJae Park 6901f366e42SSeongJae Park damon_for_each_target(t, c) { 6912b8a248dSSeongJae Park damon_for_each_region_safe(r, next_r, t) 6921f366e42SSeongJae Park damon_do_apply_schemes(c, t, r); 6931f366e42SSeongJae Park } 6941f366e42SSeongJae Park } 6951f366e42SSeongJae Park 69688f86dcfSSeongJae Park static inline unsigned long sz_damon_region(struct damon_region *r) 69788f86dcfSSeongJae Park { 69888f86dcfSSeongJae Park return r->ar.end - r->ar.start; 69988f86dcfSSeongJae Park } 700b9a6ac4eSSeongJae Park 701b9a6ac4eSSeongJae Park /* 702b9a6ac4eSSeongJae Park * Merge two adjacent regions into one region 703b9a6ac4eSSeongJae Park */ 704b9a6ac4eSSeongJae Park static void damon_merge_two_regions(struct damon_target *t, 705b9a6ac4eSSeongJae Park struct damon_region *l, struct damon_region *r) 706b9a6ac4eSSeongJae Park { 707b9a6ac4eSSeongJae Park unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r); 708b9a6ac4eSSeongJae Park 709b9a6ac4eSSeongJae Park l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / 710b9a6ac4eSSeongJae Park (sz_l + sz_r); 711fda504faSSeongJae Park l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); 712b9a6ac4eSSeongJae Park l->ar.end = r->ar.end; 713b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 714b9a6ac4eSSeongJae Park } 715b9a6ac4eSSeongJae Park 716b9a6ac4eSSeongJae Park /* 717b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 718b9a6ac4eSSeongJae Park * 719b9a6ac4eSSeongJae Park * t target affected by this merge operation 720b9a6ac4eSSeongJae Park * thres '->nr_accesses' diff threshold for the merge 721b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 722b9a6ac4eSSeongJae Park */ 723b9a6ac4eSSeongJae Park static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, 724b9a6ac4eSSeongJae Park unsigned long sz_limit) 725b9a6ac4eSSeongJae Park { 726b9a6ac4eSSeongJae Park struct damon_region *r, *prev = NULL, *next; 727b9a6ac4eSSeongJae Park 728b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 729d720bbbdSXin Hao if (abs(r->nr_accesses - r->last_nr_accesses) > thres) 730fda504faSSeongJae Park r->age = 0; 731fda504faSSeongJae Park else 732fda504faSSeongJae Park r->age++; 733fda504faSSeongJae Park 734b9a6ac4eSSeongJae Park if (prev && prev->ar.end == r->ar.start && 735d720bbbdSXin Hao abs(prev->nr_accesses - r->nr_accesses) <= thres && 736b9a6ac4eSSeongJae Park sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) 737b9a6ac4eSSeongJae Park damon_merge_two_regions(t, prev, r); 738b9a6ac4eSSeongJae Park else 739b9a6ac4eSSeongJae Park prev = r; 740b9a6ac4eSSeongJae Park } 741b9a6ac4eSSeongJae Park } 742b9a6ac4eSSeongJae Park 743b9a6ac4eSSeongJae Park /* 744b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 745b9a6ac4eSSeongJae Park * 746b9a6ac4eSSeongJae Park * threshold '->nr_accesses' diff threshold for the merge 747b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 748b9a6ac4eSSeongJae Park * 749b9a6ac4eSSeongJae Park * This function merges monitoring target regions which are adjacent and their 750b9a6ac4eSSeongJae Park * access frequencies are similar. This is for minimizing the monitoring 751b9a6ac4eSSeongJae Park * overhead under the dynamically changeable access pattern. If a merge was 752b9a6ac4eSSeongJae Park * unnecessarily made, later 'kdamond_split_regions()' will revert it. 753b9a6ac4eSSeongJae Park */ 754b9a6ac4eSSeongJae Park static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, 755b9a6ac4eSSeongJae Park unsigned long sz_limit) 756b9a6ac4eSSeongJae Park { 757b9a6ac4eSSeongJae Park struct damon_target *t; 758b9a6ac4eSSeongJae Park 759b9a6ac4eSSeongJae Park damon_for_each_target(t, c) 760b9a6ac4eSSeongJae Park damon_merge_regions_of(t, threshold, sz_limit); 761b9a6ac4eSSeongJae Park } 762b9a6ac4eSSeongJae Park 763b9a6ac4eSSeongJae Park /* 764b9a6ac4eSSeongJae Park * Split a region in two 765b9a6ac4eSSeongJae Park * 766b9a6ac4eSSeongJae Park * r the region to be split 767b9a6ac4eSSeongJae Park * sz_r size of the first sub-region that will be made 768b9a6ac4eSSeongJae Park */ 769b9a6ac4eSSeongJae Park static void damon_split_region_at(struct damon_ctx *ctx, 770b9a6ac4eSSeongJae Park struct damon_target *t, struct damon_region *r, 771b9a6ac4eSSeongJae Park unsigned long sz_r) 772b9a6ac4eSSeongJae Park { 773b9a6ac4eSSeongJae Park struct damon_region *new; 774b9a6ac4eSSeongJae Park 775b9a6ac4eSSeongJae Park new = damon_new_region(r->ar.start + sz_r, r->ar.end); 776b9a6ac4eSSeongJae Park if (!new) 777b9a6ac4eSSeongJae Park return; 778b9a6ac4eSSeongJae Park 779b9a6ac4eSSeongJae Park r->ar.end = new->ar.start; 780b9a6ac4eSSeongJae Park 781fda504faSSeongJae Park new->age = r->age; 782fda504faSSeongJae Park new->last_nr_accesses = r->last_nr_accesses; 783fda504faSSeongJae Park 784b9a6ac4eSSeongJae Park damon_insert_region(new, r, damon_next_region(r), t); 785b9a6ac4eSSeongJae Park } 786b9a6ac4eSSeongJae Park 787b9a6ac4eSSeongJae Park /* Split every region in the given target into 'nr_subs' regions */ 788b9a6ac4eSSeongJae Park static void damon_split_regions_of(struct damon_ctx *ctx, 789b9a6ac4eSSeongJae Park struct damon_target *t, int nr_subs) 790b9a6ac4eSSeongJae Park { 791b9a6ac4eSSeongJae Park struct damon_region *r, *next; 792b9a6ac4eSSeongJae Park unsigned long sz_region, sz_sub = 0; 793b9a6ac4eSSeongJae Park int i; 794b9a6ac4eSSeongJae Park 795b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 796b9a6ac4eSSeongJae Park sz_region = r->ar.end - r->ar.start; 797b9a6ac4eSSeongJae Park 798b9a6ac4eSSeongJae Park for (i = 0; i < nr_subs - 1 && 799b9a6ac4eSSeongJae Park sz_region > 2 * DAMON_MIN_REGION; i++) { 800b9a6ac4eSSeongJae Park /* 801b9a6ac4eSSeongJae Park * Randomly select size of left sub-region to be at 802b9a6ac4eSSeongJae Park * least 10 percent and at most 90% of original region 803b9a6ac4eSSeongJae Park */ 804b9a6ac4eSSeongJae Park sz_sub = ALIGN_DOWN(damon_rand(1, 10) * 805b9a6ac4eSSeongJae Park sz_region / 10, DAMON_MIN_REGION); 806b9a6ac4eSSeongJae Park /* Do not allow blank region */ 807b9a6ac4eSSeongJae Park if (sz_sub == 0 || sz_sub >= sz_region) 808b9a6ac4eSSeongJae Park continue; 809b9a6ac4eSSeongJae Park 810b9a6ac4eSSeongJae Park damon_split_region_at(ctx, t, r, sz_sub); 811b9a6ac4eSSeongJae Park sz_region = sz_sub; 812b9a6ac4eSSeongJae Park } 813b9a6ac4eSSeongJae Park } 814b9a6ac4eSSeongJae Park } 815b9a6ac4eSSeongJae Park 816b9a6ac4eSSeongJae Park /* 817b9a6ac4eSSeongJae Park * Split every target region into randomly-sized small regions 818b9a6ac4eSSeongJae Park * 819b9a6ac4eSSeongJae Park * This function splits every target region into random-sized small regions if 820b9a6ac4eSSeongJae Park * current total number of the regions is equal or smaller than half of the 821b9a6ac4eSSeongJae Park * user-specified maximum number of regions. This is for maximizing the 822b9a6ac4eSSeongJae Park * monitoring accuracy under the dynamically changeable access patterns. If a 823b9a6ac4eSSeongJae Park * split was unnecessarily made, later 'kdamond_merge_regions()' will revert 824b9a6ac4eSSeongJae Park * it. 825b9a6ac4eSSeongJae Park */ 826b9a6ac4eSSeongJae Park static void kdamond_split_regions(struct damon_ctx *ctx) 827b9a6ac4eSSeongJae Park { 828b9a6ac4eSSeongJae Park struct damon_target *t; 829b9a6ac4eSSeongJae Park unsigned int nr_regions = 0; 830b9a6ac4eSSeongJae Park static unsigned int last_nr_regions; 831b9a6ac4eSSeongJae Park int nr_subregions = 2; 832b9a6ac4eSSeongJae Park 833b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 834b9a6ac4eSSeongJae Park nr_regions += damon_nr_regions(t); 835b9a6ac4eSSeongJae Park 836b9a6ac4eSSeongJae Park if (nr_regions > ctx->max_nr_regions / 2) 837b9a6ac4eSSeongJae Park return; 838b9a6ac4eSSeongJae Park 839b9a6ac4eSSeongJae Park /* Maybe the middle of the region has different access frequency */ 840b9a6ac4eSSeongJae Park if (last_nr_regions == nr_regions && 841b9a6ac4eSSeongJae Park nr_regions < ctx->max_nr_regions / 3) 842b9a6ac4eSSeongJae Park nr_subregions = 3; 843b9a6ac4eSSeongJae Park 844b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 845b9a6ac4eSSeongJae Park damon_split_regions_of(ctx, t, nr_subregions); 846b9a6ac4eSSeongJae Park 847b9a6ac4eSSeongJae Park last_nr_regions = nr_regions; 848b9a6ac4eSSeongJae Park } 849b9a6ac4eSSeongJae Park 850f23b8eeeSSeongJae Park /* 8512224d848SSeongJae Park * Check whether it is time to check and apply the target monitoring regions 8522224d848SSeongJae Park * 8532224d848SSeongJae Park * Returns true if it is. 8542224d848SSeongJae Park */ 8552224d848SSeongJae Park static bool kdamond_need_update_primitive(struct damon_ctx *ctx) 8562224d848SSeongJae Park { 8572224d848SSeongJae Park return damon_check_reset_time_interval(&ctx->last_primitive_update, 8582224d848SSeongJae Park ctx->primitive_update_interval); 8592224d848SSeongJae Park } 8602224d848SSeongJae Park 8612224d848SSeongJae Park /* 8622224d848SSeongJae Park * Check whether current monitoring should be stopped 8632224d848SSeongJae Park * 8642224d848SSeongJae Park * The monitoring is stopped when either the user requested to stop, or all 8652224d848SSeongJae Park * monitoring targets are invalid. 8662224d848SSeongJae Park * 8672224d848SSeongJae Park * Returns true if need to stop current monitoring. 8682224d848SSeongJae Park */ 8692224d848SSeongJae Park static bool kdamond_need_stop(struct damon_ctx *ctx) 8702224d848SSeongJae Park { 871f23b8eeeSSeongJae Park struct damon_target *t; 8722224d848SSeongJae Park 8730f91d133SChangbin Du if (kthread_should_stop()) 8742224d848SSeongJae Park return true; 8752224d848SSeongJae Park 8762224d848SSeongJae Park if (!ctx->primitive.target_valid) 8772224d848SSeongJae Park return false; 8782224d848SSeongJae Park 879f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 880f23b8eeeSSeongJae Park if (ctx->primitive.target_valid(t)) 881f23b8eeeSSeongJae Park return false; 882f23b8eeeSSeongJae Park } 883f23b8eeeSSeongJae Park 884f23b8eeeSSeongJae Park return true; 8852224d848SSeongJae Park } 8862224d848SSeongJae Park 887ee801b7dSSeongJae Park static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric) 888ee801b7dSSeongJae Park { 889ee801b7dSSeongJae Park struct sysinfo i; 890ee801b7dSSeongJae Park 891ee801b7dSSeongJae Park switch (metric) { 892ee801b7dSSeongJae Park case DAMOS_WMARK_FREE_MEM_RATE: 893ee801b7dSSeongJae Park si_meminfo(&i); 894ee801b7dSSeongJae Park return i.freeram * 1000 / i.totalram; 895ee801b7dSSeongJae Park default: 896ee801b7dSSeongJae Park break; 897ee801b7dSSeongJae Park } 898ee801b7dSSeongJae Park return -EINVAL; 899ee801b7dSSeongJae Park } 900ee801b7dSSeongJae Park 901ee801b7dSSeongJae Park /* 902ee801b7dSSeongJae Park * Returns zero if the scheme is active. Else, returns time to wait for next 903ee801b7dSSeongJae Park * watermark check in micro-seconds. 904ee801b7dSSeongJae Park */ 905ee801b7dSSeongJae Park static unsigned long damos_wmark_wait_us(struct damos *scheme) 906ee801b7dSSeongJae Park { 907ee801b7dSSeongJae Park unsigned long metric; 908ee801b7dSSeongJae Park 909ee801b7dSSeongJae Park if (scheme->wmarks.metric == DAMOS_WMARK_NONE) 910ee801b7dSSeongJae Park return 0; 911ee801b7dSSeongJae Park 912ee801b7dSSeongJae Park metric = damos_wmark_metric_value(scheme->wmarks.metric); 913ee801b7dSSeongJae Park /* higher than high watermark or lower than low watermark */ 914ee801b7dSSeongJae Park if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { 915ee801b7dSSeongJae Park if (scheme->wmarks.activated) 91601078655SColin Ian King pr_debug("deactivate a scheme (%d) for %s wmark\n", 917ee801b7dSSeongJae Park scheme->action, 918ee801b7dSSeongJae Park metric > scheme->wmarks.high ? 919ee801b7dSSeongJae Park "high" : "low"); 920ee801b7dSSeongJae Park scheme->wmarks.activated = false; 921ee801b7dSSeongJae Park return scheme->wmarks.interval; 922ee801b7dSSeongJae Park } 923ee801b7dSSeongJae Park 924ee801b7dSSeongJae Park /* inactive and higher than middle watermark */ 925ee801b7dSSeongJae Park if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) && 926ee801b7dSSeongJae Park !scheme->wmarks.activated) 927ee801b7dSSeongJae Park return scheme->wmarks.interval; 928ee801b7dSSeongJae Park 929ee801b7dSSeongJae Park if (!scheme->wmarks.activated) 930ee801b7dSSeongJae Park pr_debug("activate a scheme (%d)\n", scheme->action); 931ee801b7dSSeongJae Park scheme->wmarks.activated = true; 932ee801b7dSSeongJae Park return 0; 933ee801b7dSSeongJae Park } 934ee801b7dSSeongJae Park 935ee801b7dSSeongJae Park static void kdamond_usleep(unsigned long usecs) 936ee801b7dSSeongJae Park { 9374de46a30SSeongJae Park /* See Documentation/timers/timers-howto.rst for the thresholds */ 9384de46a30SSeongJae Park if (usecs > 20 * USEC_PER_MSEC) 93970e92748SSeongJae Park schedule_timeout_idle(usecs_to_jiffies(usecs)); 940ee801b7dSSeongJae Park else 94170e92748SSeongJae Park usleep_idle_range(usecs, usecs + 1); 942ee801b7dSSeongJae Park } 943ee801b7dSSeongJae Park 944ee801b7dSSeongJae Park /* Returns negative error code if it's not activated but should return */ 945ee801b7dSSeongJae Park static int kdamond_wait_activation(struct damon_ctx *ctx) 946ee801b7dSSeongJae Park { 947ee801b7dSSeongJae Park struct damos *s; 948ee801b7dSSeongJae Park unsigned long wait_time; 949ee801b7dSSeongJae Park unsigned long min_wait_time = 0; 950ee801b7dSSeongJae Park 951ee801b7dSSeongJae Park while (!kdamond_need_stop(ctx)) { 952ee801b7dSSeongJae Park damon_for_each_scheme(s, ctx) { 953ee801b7dSSeongJae Park wait_time = damos_wmark_wait_us(s); 954ee801b7dSSeongJae Park if (!min_wait_time || wait_time < min_wait_time) 955ee801b7dSSeongJae Park min_wait_time = wait_time; 956ee801b7dSSeongJae Park } 957ee801b7dSSeongJae Park if (!min_wait_time) 958ee801b7dSSeongJae Park return 0; 959ee801b7dSSeongJae Park 960ee801b7dSSeongJae Park kdamond_usleep(min_wait_time); 961ee801b7dSSeongJae Park } 962ee801b7dSSeongJae Park return -EBUSY; 963ee801b7dSSeongJae Park } 964ee801b7dSSeongJae Park 9652224d848SSeongJae Park /* 9662224d848SSeongJae Park * The monitoring daemon that runs as a kernel thread 9672224d848SSeongJae Park */ 9682224d848SSeongJae Park static int kdamond_fn(void *data) 9692224d848SSeongJae Park { 9702224d848SSeongJae Park struct damon_ctx *ctx = (struct damon_ctx *)data; 971f23b8eeeSSeongJae Park struct damon_target *t; 972f23b8eeeSSeongJae Park struct damon_region *r, *next; 973b9a6ac4eSSeongJae Park unsigned int max_nr_accesses = 0; 974b9a6ac4eSSeongJae Park unsigned long sz_limit = 0; 9750f91d133SChangbin Du bool done = false; 9762224d848SSeongJae Park 97742e4cef5SChangbin Du pr_debug("kdamond (%d) starts\n", current->pid); 9782224d848SSeongJae Park 9792224d848SSeongJae Park if (ctx->primitive.init) 9802224d848SSeongJae Park ctx->primitive.init(ctx); 9812224d848SSeongJae Park if (ctx->callback.before_start && ctx->callback.before_start(ctx)) 9820f91d133SChangbin Du done = true; 9832224d848SSeongJae Park 984b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 985b9a6ac4eSSeongJae Park 9860f91d133SChangbin Du while (!kdamond_need_stop(ctx) && !done) { 987ee801b7dSSeongJae Park if (kdamond_wait_activation(ctx)) 988ee801b7dSSeongJae Park continue; 989ee801b7dSSeongJae Park 9902224d848SSeongJae Park if (ctx->primitive.prepare_access_checks) 9912224d848SSeongJae Park ctx->primitive.prepare_access_checks(ctx); 9922224d848SSeongJae Park if (ctx->callback.after_sampling && 9932224d848SSeongJae Park ctx->callback.after_sampling(ctx)) 9940f91d133SChangbin Du done = true; 9952224d848SSeongJae Park 99670e92748SSeongJae Park kdamond_usleep(ctx->sample_interval); 9972224d848SSeongJae Park 9982224d848SSeongJae Park if (ctx->primitive.check_accesses) 999b9a6ac4eSSeongJae Park max_nr_accesses = ctx->primitive.check_accesses(ctx); 10002224d848SSeongJae Park 10012224d848SSeongJae Park if (kdamond_aggregate_interval_passed(ctx)) { 1002b9a6ac4eSSeongJae Park kdamond_merge_regions(ctx, 1003b9a6ac4eSSeongJae Park max_nr_accesses / 10, 1004b9a6ac4eSSeongJae Park sz_limit); 10052224d848SSeongJae Park if (ctx->callback.after_aggregation && 10062224d848SSeongJae Park ctx->callback.after_aggregation(ctx)) 10070f91d133SChangbin Du done = true; 10081f366e42SSeongJae Park kdamond_apply_schemes(ctx); 1009f23b8eeeSSeongJae Park kdamond_reset_aggregated(ctx); 1010b9a6ac4eSSeongJae Park kdamond_split_regions(ctx); 10112224d848SSeongJae Park if (ctx->primitive.reset_aggregated) 10122224d848SSeongJae Park ctx->primitive.reset_aggregated(ctx); 10132224d848SSeongJae Park } 10142224d848SSeongJae Park 10152224d848SSeongJae Park if (kdamond_need_update_primitive(ctx)) { 10162224d848SSeongJae Park if (ctx->primitive.update) 10172224d848SSeongJae Park ctx->primitive.update(ctx); 1018b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 10192224d848SSeongJae Park } 10202224d848SSeongJae Park } 1021f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 1022f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 1023b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 1024f23b8eeeSSeongJae Park } 10252224d848SSeongJae Park 10260f91d133SChangbin Du if (ctx->callback.before_terminate) 10270f91d133SChangbin Du ctx->callback.before_terminate(ctx); 10282224d848SSeongJae Park if (ctx->primitive.cleanup) 10292224d848SSeongJae Park ctx->primitive.cleanup(ctx); 10302224d848SSeongJae Park 103142e4cef5SChangbin Du pr_debug("kdamond (%d) finishes\n", current->pid); 10322224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 10332224d848SSeongJae Park ctx->kdamond = NULL; 10342224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 10352224d848SSeongJae Park 10362224d848SSeongJae Park mutex_lock(&damon_lock); 10372224d848SSeongJae Park nr_running_ctxs--; 10382224d848SSeongJae Park mutex_unlock(&damon_lock); 10392224d848SSeongJae Park 10405f7fe2b9SChangbin Du return 0; 10412224d848SSeongJae Park } 104217ccae8bSSeongJae Park 104317ccae8bSSeongJae Park #include "core-test.h" 1044