12224d848SSeongJae Park // SPDX-License-Identifier: GPL-2.0 22224d848SSeongJae Park /* 32224d848SSeongJae Park * Data Access Monitor 42224d848SSeongJae Park * 52224d848SSeongJae Park * Author: SeongJae Park <sjpark@amazon.de> 62224d848SSeongJae Park */ 72224d848SSeongJae Park 82224d848SSeongJae Park #define pr_fmt(fmt) "damon: " fmt 92224d848SSeongJae Park 102224d848SSeongJae Park #include <linux/damon.h> 112224d848SSeongJae Park #include <linux/delay.h> 122224d848SSeongJae Park #include <linux/kthread.h> 13*ee801b7dSSeongJae Park #include <linux/mm.h> 14b9a6ac4eSSeongJae Park #include <linux/random.h> 152224d848SSeongJae Park #include <linux/slab.h> 1638683e00SSeongJae Park #include <linux/string.h> 172224d848SSeongJae Park 182fcb9362SSeongJae Park #define CREATE_TRACE_POINTS 192fcb9362SSeongJae Park #include <trace/events/damon.h> 202fcb9362SSeongJae Park 2117ccae8bSSeongJae Park #ifdef CONFIG_DAMON_KUNIT_TEST 2217ccae8bSSeongJae Park #undef DAMON_MIN_REGION 2317ccae8bSSeongJae Park #define DAMON_MIN_REGION 1 2417ccae8bSSeongJae Park #endif 2517ccae8bSSeongJae Park 26b9a6ac4eSSeongJae Park /* Get a random number in [l, r) */ 27b9a6ac4eSSeongJae Park #define damon_rand(l, r) (l + prandom_u32_max(r - l)) 28b9a6ac4eSSeongJae Park 292224d848SSeongJae Park static DEFINE_MUTEX(damon_lock); 302224d848SSeongJae Park static int nr_running_ctxs; 312224d848SSeongJae Park 32f23b8eeeSSeongJae Park /* 33f23b8eeeSSeongJae Park * Construct a damon_region struct 34f23b8eeeSSeongJae Park * 35f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 36f23b8eeeSSeongJae Park */ 37f23b8eeeSSeongJae Park struct damon_region *damon_new_region(unsigned long start, unsigned long end) 38f23b8eeeSSeongJae Park { 39f23b8eeeSSeongJae Park struct damon_region *region; 40f23b8eeeSSeongJae Park 41f23b8eeeSSeongJae Park region = kmalloc(sizeof(*region), GFP_KERNEL); 42f23b8eeeSSeongJae Park if (!region) 43f23b8eeeSSeongJae Park return NULL; 44f23b8eeeSSeongJae Park 45f23b8eeeSSeongJae Park region->ar.start = start; 46f23b8eeeSSeongJae Park region->ar.end = end; 47f23b8eeeSSeongJae Park region->nr_accesses = 0; 48f23b8eeeSSeongJae Park INIT_LIST_HEAD(®ion->list); 49f23b8eeeSSeongJae Park 50fda504faSSeongJae Park region->age = 0; 51fda504faSSeongJae Park region->last_nr_accesses = 0; 52fda504faSSeongJae Park 53f23b8eeeSSeongJae Park return region; 54f23b8eeeSSeongJae Park } 55f23b8eeeSSeongJae Park 56f23b8eeeSSeongJae Park /* 57f23b8eeeSSeongJae Park * Add a region between two other regions 58f23b8eeeSSeongJae Park */ 59f23b8eeeSSeongJae Park inline void damon_insert_region(struct damon_region *r, 60b9a6ac4eSSeongJae Park struct damon_region *prev, struct damon_region *next, 61b9a6ac4eSSeongJae Park struct damon_target *t) 62f23b8eeeSSeongJae Park { 63f23b8eeeSSeongJae Park __list_add(&r->list, &prev->list, &next->list); 64b9a6ac4eSSeongJae Park t->nr_regions++; 65f23b8eeeSSeongJae Park } 66f23b8eeeSSeongJae Park 67f23b8eeeSSeongJae Park void damon_add_region(struct damon_region *r, struct damon_target *t) 68f23b8eeeSSeongJae Park { 69f23b8eeeSSeongJae Park list_add_tail(&r->list, &t->regions_list); 70b9a6ac4eSSeongJae Park t->nr_regions++; 71f23b8eeeSSeongJae Park } 72f23b8eeeSSeongJae Park 73b9a6ac4eSSeongJae Park static void damon_del_region(struct damon_region *r, struct damon_target *t) 74f23b8eeeSSeongJae Park { 75f23b8eeeSSeongJae Park list_del(&r->list); 76b9a6ac4eSSeongJae Park t->nr_regions--; 77f23b8eeeSSeongJae Park } 78f23b8eeeSSeongJae Park 79f23b8eeeSSeongJae Park static void damon_free_region(struct damon_region *r) 80f23b8eeeSSeongJae Park { 81f23b8eeeSSeongJae Park kfree(r); 82f23b8eeeSSeongJae Park } 83f23b8eeeSSeongJae Park 84b9a6ac4eSSeongJae Park void damon_destroy_region(struct damon_region *r, struct damon_target *t) 85f23b8eeeSSeongJae Park { 86b9a6ac4eSSeongJae Park damon_del_region(r, t); 87f23b8eeeSSeongJae Park damon_free_region(r); 88f23b8eeeSSeongJae Park } 89f23b8eeeSSeongJae Park 901f366e42SSeongJae Park struct damos *damon_new_scheme( 911f366e42SSeongJae Park unsigned long min_sz_region, unsigned long max_sz_region, 921f366e42SSeongJae Park unsigned int min_nr_accesses, unsigned int max_nr_accesses, 931f366e42SSeongJae Park unsigned int min_age_region, unsigned int max_age_region, 94*ee801b7dSSeongJae Park enum damos_action action, struct damos_quota *quota, 95*ee801b7dSSeongJae Park struct damos_watermarks *wmarks) 961f366e42SSeongJae Park { 971f366e42SSeongJae Park struct damos *scheme; 981f366e42SSeongJae Park 991f366e42SSeongJae Park scheme = kmalloc(sizeof(*scheme), GFP_KERNEL); 1001f366e42SSeongJae Park if (!scheme) 1011f366e42SSeongJae Park return NULL; 1021f366e42SSeongJae Park scheme->min_sz_region = min_sz_region; 1031f366e42SSeongJae Park scheme->max_sz_region = max_sz_region; 1041f366e42SSeongJae Park scheme->min_nr_accesses = min_nr_accesses; 1051f366e42SSeongJae Park scheme->max_nr_accesses = max_nr_accesses; 1061f366e42SSeongJae Park scheme->min_age_region = min_age_region; 1071f366e42SSeongJae Park scheme->max_age_region = max_age_region; 1081f366e42SSeongJae Park scheme->action = action; 1092f0b548cSSeongJae Park scheme->stat_count = 0; 1102f0b548cSSeongJae Park scheme->stat_sz = 0; 1111f366e42SSeongJae Park INIT_LIST_HEAD(&scheme->list); 1121f366e42SSeongJae Park 1131cd24303SSeongJae Park scheme->quota.ms = quota->ms; 1142b8a248dSSeongJae Park scheme->quota.sz = quota->sz; 1152b8a248dSSeongJae Park scheme->quota.reset_interval = quota->reset_interval; 11638683e00SSeongJae Park scheme->quota.weight_sz = quota->weight_sz; 11738683e00SSeongJae Park scheme->quota.weight_nr_accesses = quota->weight_nr_accesses; 11838683e00SSeongJae Park scheme->quota.weight_age = quota->weight_age; 1191cd24303SSeongJae Park scheme->quota.total_charged_sz = 0; 1201cd24303SSeongJae Park scheme->quota.total_charged_ns = 0; 1211cd24303SSeongJae Park scheme->quota.esz = 0; 1222b8a248dSSeongJae Park scheme->quota.charged_sz = 0; 1232b8a248dSSeongJae Park scheme->quota.charged_from = 0; 12450585192SSeongJae Park scheme->quota.charge_target_from = NULL; 12550585192SSeongJae Park scheme->quota.charge_addr_from = 0; 1262b8a248dSSeongJae Park 127*ee801b7dSSeongJae Park scheme->wmarks.metric = wmarks->metric; 128*ee801b7dSSeongJae Park scheme->wmarks.interval = wmarks->interval; 129*ee801b7dSSeongJae Park scheme->wmarks.high = wmarks->high; 130*ee801b7dSSeongJae Park scheme->wmarks.mid = wmarks->mid; 131*ee801b7dSSeongJae Park scheme->wmarks.low = wmarks->low; 132*ee801b7dSSeongJae Park scheme->wmarks.activated = true; 133*ee801b7dSSeongJae Park 1341f366e42SSeongJae Park return scheme; 1351f366e42SSeongJae Park } 1361f366e42SSeongJae Park 1371f366e42SSeongJae Park void damon_add_scheme(struct damon_ctx *ctx, struct damos *s) 1381f366e42SSeongJae Park { 1391f366e42SSeongJae Park list_add_tail(&s->list, &ctx->schemes); 1401f366e42SSeongJae Park } 1411f366e42SSeongJae Park 1421f366e42SSeongJae Park static void damon_del_scheme(struct damos *s) 1431f366e42SSeongJae Park { 1441f366e42SSeongJae Park list_del(&s->list); 1451f366e42SSeongJae Park } 1461f366e42SSeongJae Park 1471f366e42SSeongJae Park static void damon_free_scheme(struct damos *s) 1481f366e42SSeongJae Park { 1491f366e42SSeongJae Park kfree(s); 1501f366e42SSeongJae Park } 1511f366e42SSeongJae Park 1521f366e42SSeongJae Park void damon_destroy_scheme(struct damos *s) 1531f366e42SSeongJae Park { 1541f366e42SSeongJae Park damon_del_scheme(s); 1551f366e42SSeongJae Park damon_free_scheme(s); 1561f366e42SSeongJae Park } 1571f366e42SSeongJae Park 158f23b8eeeSSeongJae Park /* 159f23b8eeeSSeongJae Park * Construct a damon_target struct 160f23b8eeeSSeongJae Park * 161f23b8eeeSSeongJae Park * Returns the pointer to the new struct if success, or NULL otherwise 162f23b8eeeSSeongJae Park */ 163f23b8eeeSSeongJae Park struct damon_target *damon_new_target(unsigned long id) 164f23b8eeeSSeongJae Park { 165f23b8eeeSSeongJae Park struct damon_target *t; 166f23b8eeeSSeongJae Park 167f23b8eeeSSeongJae Park t = kmalloc(sizeof(*t), GFP_KERNEL); 168f23b8eeeSSeongJae Park if (!t) 169f23b8eeeSSeongJae Park return NULL; 170f23b8eeeSSeongJae Park 171f23b8eeeSSeongJae Park t->id = id; 172b9a6ac4eSSeongJae Park t->nr_regions = 0; 173f23b8eeeSSeongJae Park INIT_LIST_HEAD(&t->regions_list); 174f23b8eeeSSeongJae Park 175f23b8eeeSSeongJae Park return t; 176f23b8eeeSSeongJae Park } 177f23b8eeeSSeongJae Park 178f23b8eeeSSeongJae Park void damon_add_target(struct damon_ctx *ctx, struct damon_target *t) 179f23b8eeeSSeongJae Park { 180b9a6ac4eSSeongJae Park list_add_tail(&t->list, &ctx->adaptive_targets); 181f23b8eeeSSeongJae Park } 182f23b8eeeSSeongJae Park 183f23b8eeeSSeongJae Park static void damon_del_target(struct damon_target *t) 184f23b8eeeSSeongJae Park { 185f23b8eeeSSeongJae Park list_del(&t->list); 186f23b8eeeSSeongJae Park } 187f23b8eeeSSeongJae Park 188f23b8eeeSSeongJae Park void damon_free_target(struct damon_target *t) 189f23b8eeeSSeongJae Park { 190f23b8eeeSSeongJae Park struct damon_region *r, *next; 191f23b8eeeSSeongJae Park 192f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 193f23b8eeeSSeongJae Park damon_free_region(r); 194f23b8eeeSSeongJae Park kfree(t); 195f23b8eeeSSeongJae Park } 196f23b8eeeSSeongJae Park 197f23b8eeeSSeongJae Park void damon_destroy_target(struct damon_target *t) 198f23b8eeeSSeongJae Park { 199f23b8eeeSSeongJae Park damon_del_target(t); 200f23b8eeeSSeongJae Park damon_free_target(t); 201f23b8eeeSSeongJae Park } 202f23b8eeeSSeongJae Park 203b9a6ac4eSSeongJae Park unsigned int damon_nr_regions(struct damon_target *t) 204b9a6ac4eSSeongJae Park { 205b9a6ac4eSSeongJae Park return t->nr_regions; 206b9a6ac4eSSeongJae Park } 207b9a6ac4eSSeongJae Park 2082224d848SSeongJae Park struct damon_ctx *damon_new_ctx(void) 2092224d848SSeongJae Park { 2102224d848SSeongJae Park struct damon_ctx *ctx; 2112224d848SSeongJae Park 2122224d848SSeongJae Park ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 2132224d848SSeongJae Park if (!ctx) 2142224d848SSeongJae Park return NULL; 2152224d848SSeongJae Park 2162224d848SSeongJae Park ctx->sample_interval = 5 * 1000; 2172224d848SSeongJae Park ctx->aggr_interval = 100 * 1000; 2182224d848SSeongJae Park ctx->primitive_update_interval = 60 * 1000 * 1000; 2192224d848SSeongJae Park 2202224d848SSeongJae Park ktime_get_coarse_ts64(&ctx->last_aggregation); 2212224d848SSeongJae Park ctx->last_primitive_update = ctx->last_aggregation; 2222224d848SSeongJae Park 2232224d848SSeongJae Park mutex_init(&ctx->kdamond_lock); 2242224d848SSeongJae Park 225b9a6ac4eSSeongJae Park ctx->min_nr_regions = 10; 226b9a6ac4eSSeongJae Park ctx->max_nr_regions = 1000; 227b9a6ac4eSSeongJae Park 228b9a6ac4eSSeongJae Park INIT_LIST_HEAD(&ctx->adaptive_targets); 2291f366e42SSeongJae Park INIT_LIST_HEAD(&ctx->schemes); 2302224d848SSeongJae Park 2312224d848SSeongJae Park return ctx; 2322224d848SSeongJae Park } 2332224d848SSeongJae Park 234f23b8eeeSSeongJae Park static void damon_destroy_targets(struct damon_ctx *ctx) 235f23b8eeeSSeongJae Park { 236f23b8eeeSSeongJae Park struct damon_target *t, *next_t; 237f23b8eeeSSeongJae Park 238f23b8eeeSSeongJae Park if (ctx->primitive.cleanup) { 239f23b8eeeSSeongJae Park ctx->primitive.cleanup(ctx); 240f23b8eeeSSeongJae Park return; 241f23b8eeeSSeongJae Park } 242f23b8eeeSSeongJae Park 243f23b8eeeSSeongJae Park damon_for_each_target_safe(t, next_t, ctx) 244f23b8eeeSSeongJae Park damon_destroy_target(t); 245f23b8eeeSSeongJae Park } 246f23b8eeeSSeongJae Park 2472224d848SSeongJae Park void damon_destroy_ctx(struct damon_ctx *ctx) 2482224d848SSeongJae Park { 2491f366e42SSeongJae Park struct damos *s, *next_s; 2501f366e42SSeongJae Park 251f23b8eeeSSeongJae Park damon_destroy_targets(ctx); 2521f366e42SSeongJae Park 2531f366e42SSeongJae Park damon_for_each_scheme_safe(s, next_s, ctx) 2541f366e42SSeongJae Park damon_destroy_scheme(s); 2551f366e42SSeongJae Park 2562224d848SSeongJae Park kfree(ctx); 2572224d848SSeongJae Park } 2582224d848SSeongJae Park 2592224d848SSeongJae Park /** 2604bc05954SSeongJae Park * damon_set_targets() - Set monitoring targets. 2614bc05954SSeongJae Park * @ctx: monitoring context 2624bc05954SSeongJae Park * @ids: array of target ids 2634bc05954SSeongJae Park * @nr_ids: number of entries in @ids 2644bc05954SSeongJae Park * 2654bc05954SSeongJae Park * This function should not be called while the kdamond is running. 2664bc05954SSeongJae Park * 2674bc05954SSeongJae Park * Return: 0 on success, negative error code otherwise. 2684bc05954SSeongJae Park */ 2694bc05954SSeongJae Park int damon_set_targets(struct damon_ctx *ctx, 2704bc05954SSeongJae Park unsigned long *ids, ssize_t nr_ids) 2714bc05954SSeongJae Park { 2724bc05954SSeongJae Park ssize_t i; 2734bc05954SSeongJae Park struct damon_target *t, *next; 2744bc05954SSeongJae Park 2754bc05954SSeongJae Park damon_destroy_targets(ctx); 2764bc05954SSeongJae Park 2774bc05954SSeongJae Park for (i = 0; i < nr_ids; i++) { 2784bc05954SSeongJae Park t = damon_new_target(ids[i]); 2794bc05954SSeongJae Park if (!t) { 2804bc05954SSeongJae Park pr_err("Failed to alloc damon_target\n"); 2814bc05954SSeongJae Park /* The caller should do cleanup of the ids itself */ 2824bc05954SSeongJae Park damon_for_each_target_safe(t, next, ctx) 2834bc05954SSeongJae Park damon_destroy_target(t); 2844bc05954SSeongJae Park return -ENOMEM; 2854bc05954SSeongJae Park } 2864bc05954SSeongJae Park damon_add_target(ctx, t); 2874bc05954SSeongJae Park } 2884bc05954SSeongJae Park 2894bc05954SSeongJae Park return 0; 2904bc05954SSeongJae Park } 2914bc05954SSeongJae Park 2924bc05954SSeongJae Park /** 2932224d848SSeongJae Park * damon_set_attrs() - Set attributes for the monitoring. 2942224d848SSeongJae Park * @ctx: monitoring context 2952224d848SSeongJae Park * @sample_int: time interval between samplings 2962224d848SSeongJae Park * @aggr_int: time interval between aggregations 2972224d848SSeongJae Park * @primitive_upd_int: time interval between monitoring primitive updates 298b9a6ac4eSSeongJae Park * @min_nr_reg: minimal number of regions 299b9a6ac4eSSeongJae Park * @max_nr_reg: maximum number of regions 3002224d848SSeongJae Park * 3012224d848SSeongJae Park * This function should not be called while the kdamond is running. 3022224d848SSeongJae Park * Every time interval is in micro-seconds. 3032224d848SSeongJae Park * 3042224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 3052224d848SSeongJae Park */ 3062224d848SSeongJae Park int damon_set_attrs(struct damon_ctx *ctx, unsigned long sample_int, 307b9a6ac4eSSeongJae Park unsigned long aggr_int, unsigned long primitive_upd_int, 308b9a6ac4eSSeongJae Park unsigned long min_nr_reg, unsigned long max_nr_reg) 3092224d848SSeongJae Park { 310b9a6ac4eSSeongJae Park if (min_nr_reg < 3) { 311b9a6ac4eSSeongJae Park pr_err("min_nr_regions (%lu) must be at least 3\n", 312b9a6ac4eSSeongJae Park min_nr_reg); 313b9a6ac4eSSeongJae Park return -EINVAL; 314b9a6ac4eSSeongJae Park } 315b9a6ac4eSSeongJae Park if (min_nr_reg > max_nr_reg) { 316b9a6ac4eSSeongJae Park pr_err("invalid nr_regions. min (%lu) > max (%lu)\n", 317b9a6ac4eSSeongJae Park min_nr_reg, max_nr_reg); 318b9a6ac4eSSeongJae Park return -EINVAL; 319b9a6ac4eSSeongJae Park } 320b9a6ac4eSSeongJae Park 3212224d848SSeongJae Park ctx->sample_interval = sample_int; 3222224d848SSeongJae Park ctx->aggr_interval = aggr_int; 3232224d848SSeongJae Park ctx->primitive_update_interval = primitive_upd_int; 324b9a6ac4eSSeongJae Park ctx->min_nr_regions = min_nr_reg; 325b9a6ac4eSSeongJae Park ctx->max_nr_regions = max_nr_reg; 3262224d848SSeongJae Park 3272224d848SSeongJae Park return 0; 3282224d848SSeongJae Park } 3292224d848SSeongJae Park 3304bc05954SSeongJae Park /** 3311f366e42SSeongJae Park * damon_set_schemes() - Set data access monitoring based operation schemes. 3321f366e42SSeongJae Park * @ctx: monitoring context 3331f366e42SSeongJae Park * @schemes: array of the schemes 3341f366e42SSeongJae Park * @nr_schemes: number of entries in @schemes 3351f366e42SSeongJae Park * 3361f366e42SSeongJae Park * This function should not be called while the kdamond of the context is 3371f366e42SSeongJae Park * running. 3381f366e42SSeongJae Park * 3391f366e42SSeongJae Park * Return: 0 if success, or negative error code otherwise. 3401f366e42SSeongJae Park */ 3411f366e42SSeongJae Park int damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, 3421f366e42SSeongJae Park ssize_t nr_schemes) 3431f366e42SSeongJae Park { 3441f366e42SSeongJae Park struct damos *s, *next; 3451f366e42SSeongJae Park ssize_t i; 3461f366e42SSeongJae Park 3471f366e42SSeongJae Park damon_for_each_scheme_safe(s, next, ctx) 3481f366e42SSeongJae Park damon_destroy_scheme(s); 3491f366e42SSeongJae Park for (i = 0; i < nr_schemes; i++) 3501f366e42SSeongJae Park damon_add_scheme(ctx, schemes[i]); 3511f366e42SSeongJae Park return 0; 3521f366e42SSeongJae Park } 3531f366e42SSeongJae Park 3541f366e42SSeongJae Park /** 3554bc05954SSeongJae Park * damon_nr_running_ctxs() - Return number of currently running contexts. 3564bc05954SSeongJae Park */ 3574bc05954SSeongJae Park int damon_nr_running_ctxs(void) 3584bc05954SSeongJae Park { 3594bc05954SSeongJae Park int nr_ctxs; 3604bc05954SSeongJae Park 3614bc05954SSeongJae Park mutex_lock(&damon_lock); 3624bc05954SSeongJae Park nr_ctxs = nr_running_ctxs; 3634bc05954SSeongJae Park mutex_unlock(&damon_lock); 3644bc05954SSeongJae Park 3654bc05954SSeongJae Park return nr_ctxs; 3664bc05954SSeongJae Park } 3674bc05954SSeongJae Park 368b9a6ac4eSSeongJae Park /* Returns the size upper limit for each monitoring region */ 369b9a6ac4eSSeongJae Park static unsigned long damon_region_sz_limit(struct damon_ctx *ctx) 370b9a6ac4eSSeongJae Park { 371b9a6ac4eSSeongJae Park struct damon_target *t; 372b9a6ac4eSSeongJae Park struct damon_region *r; 373b9a6ac4eSSeongJae Park unsigned long sz = 0; 374b9a6ac4eSSeongJae Park 375b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) { 376b9a6ac4eSSeongJae Park damon_for_each_region(r, t) 377b9a6ac4eSSeongJae Park sz += r->ar.end - r->ar.start; 378b9a6ac4eSSeongJae Park } 379b9a6ac4eSSeongJae Park 380b9a6ac4eSSeongJae Park if (ctx->min_nr_regions) 381b9a6ac4eSSeongJae Park sz /= ctx->min_nr_regions; 382b9a6ac4eSSeongJae Park if (sz < DAMON_MIN_REGION) 383b9a6ac4eSSeongJae Park sz = DAMON_MIN_REGION; 384b9a6ac4eSSeongJae Park 385b9a6ac4eSSeongJae Park return sz; 386b9a6ac4eSSeongJae Park } 387b9a6ac4eSSeongJae Park 3882224d848SSeongJae Park static bool damon_kdamond_running(struct damon_ctx *ctx) 3892224d848SSeongJae Park { 3902224d848SSeongJae Park bool running; 3912224d848SSeongJae Park 3922224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 3932224d848SSeongJae Park running = ctx->kdamond != NULL; 3942224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 3952224d848SSeongJae Park 3962224d848SSeongJae Park return running; 3972224d848SSeongJae Park } 3982224d848SSeongJae Park 3992224d848SSeongJae Park static int kdamond_fn(void *data); 4002224d848SSeongJae Park 4012224d848SSeongJae Park /* 4022224d848SSeongJae Park * __damon_start() - Starts monitoring with given context. 4032224d848SSeongJae Park * @ctx: monitoring context 4042224d848SSeongJae Park * 4052224d848SSeongJae Park * This function should be called while damon_lock is hold. 4062224d848SSeongJae Park * 4072224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4082224d848SSeongJae Park */ 4092224d848SSeongJae Park static int __damon_start(struct damon_ctx *ctx) 4102224d848SSeongJae Park { 4112224d848SSeongJae Park int err = -EBUSY; 4122224d848SSeongJae Park 4132224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 4142224d848SSeongJae Park if (!ctx->kdamond) { 4152224d848SSeongJae Park err = 0; 4162224d848SSeongJae Park ctx->kdamond_stop = false; 4172224d848SSeongJae Park ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d", 4182224d848SSeongJae Park nr_running_ctxs); 4192224d848SSeongJae Park if (IS_ERR(ctx->kdamond)) { 4202224d848SSeongJae Park err = PTR_ERR(ctx->kdamond); 4217ec1992bSColin Ian King ctx->kdamond = NULL; 4222224d848SSeongJae Park } 4232224d848SSeongJae Park } 4242224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 4252224d848SSeongJae Park 4262224d848SSeongJae Park return err; 4272224d848SSeongJae Park } 4282224d848SSeongJae Park 4292224d848SSeongJae Park /** 4302224d848SSeongJae Park * damon_start() - Starts the monitorings for a given group of contexts. 4312224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to start monitoring 4322224d848SSeongJae Park * @nr_ctxs: size of @ctxs 4332224d848SSeongJae Park * 4342224d848SSeongJae Park * This function starts a group of monitoring threads for a group of monitoring 4352224d848SSeongJae Park * contexts. One thread per each context is created and run in parallel. The 4362224d848SSeongJae Park * caller should handle synchronization between the threads by itself. If a 4372224d848SSeongJae Park * group of threads that created by other 'damon_start()' call is currently 4382224d848SSeongJae Park * running, this function does nothing but returns -EBUSY. 4392224d848SSeongJae Park * 4402224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4412224d848SSeongJae Park */ 4422224d848SSeongJae Park int damon_start(struct damon_ctx **ctxs, int nr_ctxs) 4432224d848SSeongJae Park { 4442224d848SSeongJae Park int i; 4452224d848SSeongJae Park int err = 0; 4462224d848SSeongJae Park 4472224d848SSeongJae Park mutex_lock(&damon_lock); 4482224d848SSeongJae Park if (nr_running_ctxs) { 4492224d848SSeongJae Park mutex_unlock(&damon_lock); 4502224d848SSeongJae Park return -EBUSY; 4512224d848SSeongJae Park } 4522224d848SSeongJae Park 4532224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 4542224d848SSeongJae Park err = __damon_start(ctxs[i]); 4552224d848SSeongJae Park if (err) 4562224d848SSeongJae Park break; 4572224d848SSeongJae Park nr_running_ctxs++; 4582224d848SSeongJae Park } 4592224d848SSeongJae Park mutex_unlock(&damon_lock); 4602224d848SSeongJae Park 4612224d848SSeongJae Park return err; 4622224d848SSeongJae Park } 4632224d848SSeongJae Park 4642224d848SSeongJae Park /* 4652224d848SSeongJae Park * __damon_stop() - Stops monitoring of given context. 4662224d848SSeongJae Park * @ctx: monitoring context 4672224d848SSeongJae Park * 4682224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4692224d848SSeongJae Park */ 4702224d848SSeongJae Park static int __damon_stop(struct damon_ctx *ctx) 4712224d848SSeongJae Park { 4722224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 4732224d848SSeongJae Park if (ctx->kdamond) { 4742224d848SSeongJae Park ctx->kdamond_stop = true; 4752224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 4762224d848SSeongJae Park while (damon_kdamond_running(ctx)) 4772224d848SSeongJae Park usleep_range(ctx->sample_interval, 4782224d848SSeongJae Park ctx->sample_interval * 2); 4792224d848SSeongJae Park return 0; 4802224d848SSeongJae Park } 4812224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 4822224d848SSeongJae Park 4832224d848SSeongJae Park return -EPERM; 4842224d848SSeongJae Park } 4852224d848SSeongJae Park 4862224d848SSeongJae Park /** 4872224d848SSeongJae Park * damon_stop() - Stops the monitorings for a given group of contexts. 4882224d848SSeongJae Park * @ctxs: an array of the pointers for contexts to stop monitoring 4892224d848SSeongJae Park * @nr_ctxs: size of @ctxs 4902224d848SSeongJae Park * 4912224d848SSeongJae Park * Return: 0 on success, negative error code otherwise. 4922224d848SSeongJae Park */ 4932224d848SSeongJae Park int damon_stop(struct damon_ctx **ctxs, int nr_ctxs) 4942224d848SSeongJae Park { 4952224d848SSeongJae Park int i, err = 0; 4962224d848SSeongJae Park 4972224d848SSeongJae Park for (i = 0; i < nr_ctxs; i++) { 4982224d848SSeongJae Park /* nr_running_ctxs is decremented in kdamond_fn */ 4992224d848SSeongJae Park err = __damon_stop(ctxs[i]); 5002224d848SSeongJae Park if (err) 5012224d848SSeongJae Park return err; 5022224d848SSeongJae Park } 5032224d848SSeongJae Park 5042224d848SSeongJae Park return err; 5052224d848SSeongJae Park } 5062224d848SSeongJae Park 5072224d848SSeongJae Park /* 5082224d848SSeongJae Park * damon_check_reset_time_interval() - Check if a time interval is elapsed. 5092224d848SSeongJae Park * @baseline: the time to check whether the interval has elapsed since 5102224d848SSeongJae Park * @interval: the time interval (microseconds) 5112224d848SSeongJae Park * 5122224d848SSeongJae Park * See whether the given time interval has passed since the given baseline 5132224d848SSeongJae Park * time. If so, it also updates the baseline to current time for next check. 5142224d848SSeongJae Park * 5152224d848SSeongJae Park * Return: true if the time interval has passed, or false otherwise. 5162224d848SSeongJae Park */ 5172224d848SSeongJae Park static bool damon_check_reset_time_interval(struct timespec64 *baseline, 5182224d848SSeongJae Park unsigned long interval) 5192224d848SSeongJae Park { 5202224d848SSeongJae Park struct timespec64 now; 5212224d848SSeongJae Park 5222224d848SSeongJae Park ktime_get_coarse_ts64(&now); 5232224d848SSeongJae Park if ((timespec64_to_ns(&now) - timespec64_to_ns(baseline)) < 5242224d848SSeongJae Park interval * 1000) 5252224d848SSeongJae Park return false; 5262224d848SSeongJae Park *baseline = now; 5272224d848SSeongJae Park return true; 5282224d848SSeongJae Park } 5292224d848SSeongJae Park 5302224d848SSeongJae Park /* 5312224d848SSeongJae Park * Check whether it is time to flush the aggregated information 5322224d848SSeongJae Park */ 5332224d848SSeongJae Park static bool kdamond_aggregate_interval_passed(struct damon_ctx *ctx) 5342224d848SSeongJae Park { 5352224d848SSeongJae Park return damon_check_reset_time_interval(&ctx->last_aggregation, 5362224d848SSeongJae Park ctx->aggr_interval); 5372224d848SSeongJae Park } 5382224d848SSeongJae Park 5392224d848SSeongJae Park /* 540f23b8eeeSSeongJae Park * Reset the aggregated monitoring results ('nr_accesses' of each region). 541f23b8eeeSSeongJae Park */ 542f23b8eeeSSeongJae Park static void kdamond_reset_aggregated(struct damon_ctx *c) 543f23b8eeeSSeongJae Park { 544f23b8eeeSSeongJae Park struct damon_target *t; 545f23b8eeeSSeongJae Park 546f23b8eeeSSeongJae Park damon_for_each_target(t, c) { 547f23b8eeeSSeongJae Park struct damon_region *r; 548f23b8eeeSSeongJae Park 5492fcb9362SSeongJae Park damon_for_each_region(r, t) { 5502fcb9362SSeongJae Park trace_damon_aggregated(t, r, damon_nr_regions(t)); 551fda504faSSeongJae Park r->last_nr_accesses = r->nr_accesses; 552f23b8eeeSSeongJae Park r->nr_accesses = 0; 553f23b8eeeSSeongJae Park } 554f23b8eeeSSeongJae Park } 5552fcb9362SSeongJae Park } 556f23b8eeeSSeongJae Park 5572b8a248dSSeongJae Park static void damon_split_region_at(struct damon_ctx *ctx, 5582b8a248dSSeongJae Park struct damon_target *t, struct damon_region *r, 5592b8a248dSSeongJae Park unsigned long sz_r); 5602b8a248dSSeongJae Park 56138683e00SSeongJae Park static bool __damos_valid_target(struct damon_region *r, struct damos *s) 56238683e00SSeongJae Park { 56338683e00SSeongJae Park unsigned long sz; 56438683e00SSeongJae Park 56538683e00SSeongJae Park sz = r->ar.end - r->ar.start; 56638683e00SSeongJae Park return s->min_sz_region <= sz && sz <= s->max_sz_region && 56738683e00SSeongJae Park s->min_nr_accesses <= r->nr_accesses && 56838683e00SSeongJae Park r->nr_accesses <= s->max_nr_accesses && 56938683e00SSeongJae Park s->min_age_region <= r->age && r->age <= s->max_age_region; 57038683e00SSeongJae Park } 57138683e00SSeongJae Park 57238683e00SSeongJae Park static bool damos_valid_target(struct damon_ctx *c, struct damon_target *t, 57338683e00SSeongJae Park struct damon_region *r, struct damos *s) 57438683e00SSeongJae Park { 57538683e00SSeongJae Park bool ret = __damos_valid_target(r, s); 57638683e00SSeongJae Park 57738683e00SSeongJae Park if (!ret || !s->quota.esz || !c->primitive.get_scheme_score) 57838683e00SSeongJae Park return ret; 57938683e00SSeongJae Park 58038683e00SSeongJae Park return c->primitive.get_scheme_score(c, t, r, s) >= s->quota.min_score; 58138683e00SSeongJae Park } 58238683e00SSeongJae Park 5831f366e42SSeongJae Park static void damon_do_apply_schemes(struct damon_ctx *c, 5841f366e42SSeongJae Park struct damon_target *t, 5851f366e42SSeongJae Park struct damon_region *r) 5861f366e42SSeongJae Park { 5871f366e42SSeongJae Park struct damos *s; 5881f366e42SSeongJae Park 5891f366e42SSeongJae Park damon_for_each_scheme(s, c) { 5902b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 5912b8a248dSSeongJae Park unsigned long sz = r->ar.end - r->ar.start; 5921cd24303SSeongJae Park struct timespec64 begin, end; 5932b8a248dSSeongJae Park 594*ee801b7dSSeongJae Park if (!s->wmarks.activated) 595*ee801b7dSSeongJae Park continue; 596*ee801b7dSSeongJae Park 5972b8a248dSSeongJae Park /* Check the quota */ 5981cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) 5992b8a248dSSeongJae Park continue; 6002b8a248dSSeongJae Park 60150585192SSeongJae Park /* Skip previously charged regions */ 60250585192SSeongJae Park if (quota->charge_target_from) { 60350585192SSeongJae Park if (t != quota->charge_target_from) 60450585192SSeongJae Park continue; 60550585192SSeongJae Park if (r == damon_last_region(t)) { 60650585192SSeongJae Park quota->charge_target_from = NULL; 60750585192SSeongJae Park quota->charge_addr_from = 0; 60850585192SSeongJae Park continue; 60950585192SSeongJae Park } 61050585192SSeongJae Park if (quota->charge_addr_from && 61150585192SSeongJae Park r->ar.end <= quota->charge_addr_from) 61250585192SSeongJae Park continue; 61350585192SSeongJae Park 61450585192SSeongJae Park if (quota->charge_addr_from && r->ar.start < 61550585192SSeongJae Park quota->charge_addr_from) { 61650585192SSeongJae Park sz = ALIGN_DOWN(quota->charge_addr_from - 61750585192SSeongJae Park r->ar.start, DAMON_MIN_REGION); 61850585192SSeongJae Park if (!sz) { 61950585192SSeongJae Park if (r->ar.end - r->ar.start <= 62050585192SSeongJae Park DAMON_MIN_REGION) 62150585192SSeongJae Park continue; 62250585192SSeongJae Park sz = DAMON_MIN_REGION; 62350585192SSeongJae Park } 62450585192SSeongJae Park damon_split_region_at(c, t, r, sz); 62550585192SSeongJae Park r = damon_next_region(r); 62650585192SSeongJae Park sz = r->ar.end - r->ar.start; 62750585192SSeongJae Park } 62850585192SSeongJae Park quota->charge_target_from = NULL; 62950585192SSeongJae Park quota->charge_addr_from = 0; 63050585192SSeongJae Park } 63150585192SSeongJae Park 63238683e00SSeongJae Park if (!damos_valid_target(c, t, r, s)) 6331f366e42SSeongJae Park continue; 6342b8a248dSSeongJae Park 6352b8a248dSSeongJae Park /* Apply the scheme */ 6362b8a248dSSeongJae Park if (c->primitive.apply_scheme) { 6371cd24303SSeongJae Park if (quota->esz && 6381cd24303SSeongJae Park quota->charged_sz + sz > quota->esz) { 6391cd24303SSeongJae Park sz = ALIGN_DOWN(quota->esz - quota->charged_sz, 6402b8a248dSSeongJae Park DAMON_MIN_REGION); 6412b8a248dSSeongJae Park if (!sz) 6422b8a248dSSeongJae Park goto update_stat; 6432b8a248dSSeongJae Park damon_split_region_at(c, t, r, sz); 6442b8a248dSSeongJae Park } 6451cd24303SSeongJae Park ktime_get_coarse_ts64(&begin); 6461f366e42SSeongJae Park c->primitive.apply_scheme(c, t, r, s); 6471cd24303SSeongJae Park ktime_get_coarse_ts64(&end); 6481cd24303SSeongJae Park quota->total_charged_ns += timespec64_to_ns(&end) - 6491cd24303SSeongJae Park timespec64_to_ns(&begin); 6502b8a248dSSeongJae Park quota->charged_sz += sz; 6511cd24303SSeongJae Park if (quota->esz && quota->charged_sz >= quota->esz) { 65250585192SSeongJae Park quota->charge_target_from = t; 65350585192SSeongJae Park quota->charge_addr_from = r->ar.end + 1; 65450585192SSeongJae Park } 6552b8a248dSSeongJae Park } 6562f0b548cSSeongJae Park if (s->action != DAMOS_STAT) 6571f366e42SSeongJae Park r->age = 0; 6582b8a248dSSeongJae Park 6592b8a248dSSeongJae Park update_stat: 6602b8a248dSSeongJae Park s->stat_count++; 6612b8a248dSSeongJae Park s->stat_sz += sz; 6621f366e42SSeongJae Park } 6631f366e42SSeongJae Park } 6641f366e42SSeongJae Park 6651cd24303SSeongJae Park /* Shouldn't be called if quota->ms and quota->sz are zero */ 6661cd24303SSeongJae Park static void damos_set_effective_quota(struct damos_quota *quota) 6671cd24303SSeongJae Park { 6681cd24303SSeongJae Park unsigned long throughput; 6691cd24303SSeongJae Park unsigned long esz; 6701cd24303SSeongJae Park 6711cd24303SSeongJae Park if (!quota->ms) { 6721cd24303SSeongJae Park quota->esz = quota->sz; 6731cd24303SSeongJae Park return; 6741cd24303SSeongJae Park } 6751cd24303SSeongJae Park 6761cd24303SSeongJae Park if (quota->total_charged_ns) 6771cd24303SSeongJae Park throughput = quota->total_charged_sz * 1000000 / 6781cd24303SSeongJae Park quota->total_charged_ns; 6791cd24303SSeongJae Park else 6801cd24303SSeongJae Park throughput = PAGE_SIZE * 1024; 6811cd24303SSeongJae Park esz = throughput * quota->ms; 6821cd24303SSeongJae Park 6831cd24303SSeongJae Park if (quota->sz && quota->sz < esz) 6841cd24303SSeongJae Park esz = quota->sz; 6851cd24303SSeongJae Park quota->esz = esz; 6861cd24303SSeongJae Park } 6871cd24303SSeongJae Park 6881f366e42SSeongJae Park static void kdamond_apply_schemes(struct damon_ctx *c) 6891f366e42SSeongJae Park { 6901f366e42SSeongJae Park struct damon_target *t; 6912b8a248dSSeongJae Park struct damon_region *r, *next_r; 6922b8a248dSSeongJae Park struct damos *s; 6932b8a248dSSeongJae Park 6942b8a248dSSeongJae Park damon_for_each_scheme(s, c) { 6952b8a248dSSeongJae Park struct damos_quota *quota = &s->quota; 69638683e00SSeongJae Park unsigned long cumulated_sz; 69738683e00SSeongJae Park unsigned int score, max_score = 0; 6982b8a248dSSeongJae Park 699*ee801b7dSSeongJae Park if (!s->wmarks.activated) 700*ee801b7dSSeongJae Park continue; 701*ee801b7dSSeongJae Park 7021cd24303SSeongJae Park if (!quota->ms && !quota->sz) 7032b8a248dSSeongJae Park continue; 7042b8a248dSSeongJae Park 7052b8a248dSSeongJae Park /* New charge window starts */ 7062b8a248dSSeongJae Park if (time_after_eq(jiffies, quota->charged_from + 7072b8a248dSSeongJae Park msecs_to_jiffies( 7082b8a248dSSeongJae Park quota->reset_interval))) { 7091cd24303SSeongJae Park quota->total_charged_sz += quota->charged_sz; 7102b8a248dSSeongJae Park quota->charged_from = jiffies; 7112b8a248dSSeongJae Park quota->charged_sz = 0; 7121cd24303SSeongJae Park damos_set_effective_quota(quota); 7132b8a248dSSeongJae Park } 71438683e00SSeongJae Park 71538683e00SSeongJae Park if (!c->primitive.get_scheme_score) 71638683e00SSeongJae Park continue; 71738683e00SSeongJae Park 71838683e00SSeongJae Park /* Fill up the score histogram */ 71938683e00SSeongJae Park memset(quota->histogram, 0, sizeof(quota->histogram)); 72038683e00SSeongJae Park damon_for_each_target(t, c) { 72138683e00SSeongJae Park damon_for_each_region(r, t) { 72238683e00SSeongJae Park if (!__damos_valid_target(r, s)) 72338683e00SSeongJae Park continue; 72438683e00SSeongJae Park score = c->primitive.get_scheme_score( 72538683e00SSeongJae Park c, t, r, s); 72638683e00SSeongJae Park quota->histogram[score] += 72738683e00SSeongJae Park r->ar.end - r->ar.start; 72838683e00SSeongJae Park if (score > max_score) 72938683e00SSeongJae Park max_score = score; 73038683e00SSeongJae Park } 73138683e00SSeongJae Park } 73238683e00SSeongJae Park 73338683e00SSeongJae Park /* Set the min score limit */ 73438683e00SSeongJae Park for (cumulated_sz = 0, score = max_score; ; score--) { 73538683e00SSeongJae Park cumulated_sz += quota->histogram[score]; 73638683e00SSeongJae Park if (cumulated_sz >= quota->esz || !score) 73738683e00SSeongJae Park break; 73838683e00SSeongJae Park } 73938683e00SSeongJae Park quota->min_score = score; 7402b8a248dSSeongJae Park } 7411f366e42SSeongJae Park 7421f366e42SSeongJae Park damon_for_each_target(t, c) { 7432b8a248dSSeongJae Park damon_for_each_region_safe(r, next_r, t) 7441f366e42SSeongJae Park damon_do_apply_schemes(c, t, r); 7451f366e42SSeongJae Park } 7461f366e42SSeongJae Park } 7471f366e42SSeongJae Park 748b9a6ac4eSSeongJae Park #define sz_damon_region(r) (r->ar.end - r->ar.start) 749b9a6ac4eSSeongJae Park 750b9a6ac4eSSeongJae Park /* 751b9a6ac4eSSeongJae Park * Merge two adjacent regions into one region 752b9a6ac4eSSeongJae Park */ 753b9a6ac4eSSeongJae Park static void damon_merge_two_regions(struct damon_target *t, 754b9a6ac4eSSeongJae Park struct damon_region *l, struct damon_region *r) 755b9a6ac4eSSeongJae Park { 756b9a6ac4eSSeongJae Park unsigned long sz_l = sz_damon_region(l), sz_r = sz_damon_region(r); 757b9a6ac4eSSeongJae Park 758b9a6ac4eSSeongJae Park l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) / 759b9a6ac4eSSeongJae Park (sz_l + sz_r); 760fda504faSSeongJae Park l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r); 761b9a6ac4eSSeongJae Park l->ar.end = r->ar.end; 762b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 763b9a6ac4eSSeongJae Park } 764b9a6ac4eSSeongJae Park 765b9a6ac4eSSeongJae Park #define diff_of(a, b) (a > b ? a - b : b - a) 766b9a6ac4eSSeongJae Park 767b9a6ac4eSSeongJae Park /* 768b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 769b9a6ac4eSSeongJae Park * 770b9a6ac4eSSeongJae Park * t target affected by this merge operation 771b9a6ac4eSSeongJae Park * thres '->nr_accesses' diff threshold for the merge 772b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 773b9a6ac4eSSeongJae Park */ 774b9a6ac4eSSeongJae Park static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, 775b9a6ac4eSSeongJae Park unsigned long sz_limit) 776b9a6ac4eSSeongJae Park { 777b9a6ac4eSSeongJae Park struct damon_region *r, *prev = NULL, *next; 778b9a6ac4eSSeongJae Park 779b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 780fda504faSSeongJae Park if (diff_of(r->nr_accesses, r->last_nr_accesses) > thres) 781fda504faSSeongJae Park r->age = 0; 782fda504faSSeongJae Park else 783fda504faSSeongJae Park r->age++; 784fda504faSSeongJae Park 785b9a6ac4eSSeongJae Park if (prev && prev->ar.end == r->ar.start && 786b9a6ac4eSSeongJae Park diff_of(prev->nr_accesses, r->nr_accesses) <= thres && 787b9a6ac4eSSeongJae Park sz_damon_region(prev) + sz_damon_region(r) <= sz_limit) 788b9a6ac4eSSeongJae Park damon_merge_two_regions(t, prev, r); 789b9a6ac4eSSeongJae Park else 790b9a6ac4eSSeongJae Park prev = r; 791b9a6ac4eSSeongJae Park } 792b9a6ac4eSSeongJae Park } 793b9a6ac4eSSeongJae Park 794b9a6ac4eSSeongJae Park /* 795b9a6ac4eSSeongJae Park * Merge adjacent regions having similar access frequencies 796b9a6ac4eSSeongJae Park * 797b9a6ac4eSSeongJae Park * threshold '->nr_accesses' diff threshold for the merge 798b9a6ac4eSSeongJae Park * sz_limit size upper limit of each region 799b9a6ac4eSSeongJae Park * 800b9a6ac4eSSeongJae Park * This function merges monitoring target regions which are adjacent and their 801b9a6ac4eSSeongJae Park * access frequencies are similar. This is for minimizing the monitoring 802b9a6ac4eSSeongJae Park * overhead under the dynamically changeable access pattern. If a merge was 803b9a6ac4eSSeongJae Park * unnecessarily made, later 'kdamond_split_regions()' will revert it. 804b9a6ac4eSSeongJae Park */ 805b9a6ac4eSSeongJae Park static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, 806b9a6ac4eSSeongJae Park unsigned long sz_limit) 807b9a6ac4eSSeongJae Park { 808b9a6ac4eSSeongJae Park struct damon_target *t; 809b9a6ac4eSSeongJae Park 810b9a6ac4eSSeongJae Park damon_for_each_target(t, c) 811b9a6ac4eSSeongJae Park damon_merge_regions_of(t, threshold, sz_limit); 812b9a6ac4eSSeongJae Park } 813b9a6ac4eSSeongJae Park 814b9a6ac4eSSeongJae Park /* 815b9a6ac4eSSeongJae Park * Split a region in two 816b9a6ac4eSSeongJae Park * 817b9a6ac4eSSeongJae Park * r the region to be split 818b9a6ac4eSSeongJae Park * sz_r size of the first sub-region that will be made 819b9a6ac4eSSeongJae Park */ 820b9a6ac4eSSeongJae Park static void damon_split_region_at(struct damon_ctx *ctx, 821b9a6ac4eSSeongJae Park struct damon_target *t, struct damon_region *r, 822b9a6ac4eSSeongJae Park unsigned long sz_r) 823b9a6ac4eSSeongJae Park { 824b9a6ac4eSSeongJae Park struct damon_region *new; 825b9a6ac4eSSeongJae Park 826b9a6ac4eSSeongJae Park new = damon_new_region(r->ar.start + sz_r, r->ar.end); 827b9a6ac4eSSeongJae Park if (!new) 828b9a6ac4eSSeongJae Park return; 829b9a6ac4eSSeongJae Park 830b9a6ac4eSSeongJae Park r->ar.end = new->ar.start; 831b9a6ac4eSSeongJae Park 832fda504faSSeongJae Park new->age = r->age; 833fda504faSSeongJae Park new->last_nr_accesses = r->last_nr_accesses; 834fda504faSSeongJae Park 835b9a6ac4eSSeongJae Park damon_insert_region(new, r, damon_next_region(r), t); 836b9a6ac4eSSeongJae Park } 837b9a6ac4eSSeongJae Park 838b9a6ac4eSSeongJae Park /* Split every region in the given target into 'nr_subs' regions */ 839b9a6ac4eSSeongJae Park static void damon_split_regions_of(struct damon_ctx *ctx, 840b9a6ac4eSSeongJae Park struct damon_target *t, int nr_subs) 841b9a6ac4eSSeongJae Park { 842b9a6ac4eSSeongJae Park struct damon_region *r, *next; 843b9a6ac4eSSeongJae Park unsigned long sz_region, sz_sub = 0; 844b9a6ac4eSSeongJae Park int i; 845b9a6ac4eSSeongJae Park 846b9a6ac4eSSeongJae Park damon_for_each_region_safe(r, next, t) { 847b9a6ac4eSSeongJae Park sz_region = r->ar.end - r->ar.start; 848b9a6ac4eSSeongJae Park 849b9a6ac4eSSeongJae Park for (i = 0; i < nr_subs - 1 && 850b9a6ac4eSSeongJae Park sz_region > 2 * DAMON_MIN_REGION; i++) { 851b9a6ac4eSSeongJae Park /* 852b9a6ac4eSSeongJae Park * Randomly select size of left sub-region to be at 853b9a6ac4eSSeongJae Park * least 10 percent and at most 90% of original region 854b9a6ac4eSSeongJae Park */ 855b9a6ac4eSSeongJae Park sz_sub = ALIGN_DOWN(damon_rand(1, 10) * 856b9a6ac4eSSeongJae Park sz_region / 10, DAMON_MIN_REGION); 857b9a6ac4eSSeongJae Park /* Do not allow blank region */ 858b9a6ac4eSSeongJae Park if (sz_sub == 0 || sz_sub >= sz_region) 859b9a6ac4eSSeongJae Park continue; 860b9a6ac4eSSeongJae Park 861b9a6ac4eSSeongJae Park damon_split_region_at(ctx, t, r, sz_sub); 862b9a6ac4eSSeongJae Park sz_region = sz_sub; 863b9a6ac4eSSeongJae Park } 864b9a6ac4eSSeongJae Park } 865b9a6ac4eSSeongJae Park } 866b9a6ac4eSSeongJae Park 867b9a6ac4eSSeongJae Park /* 868b9a6ac4eSSeongJae Park * Split every target region into randomly-sized small regions 869b9a6ac4eSSeongJae Park * 870b9a6ac4eSSeongJae Park * This function splits every target region into random-sized small regions if 871b9a6ac4eSSeongJae Park * current total number of the regions is equal or smaller than half of the 872b9a6ac4eSSeongJae Park * user-specified maximum number of regions. This is for maximizing the 873b9a6ac4eSSeongJae Park * monitoring accuracy under the dynamically changeable access patterns. If a 874b9a6ac4eSSeongJae Park * split was unnecessarily made, later 'kdamond_merge_regions()' will revert 875b9a6ac4eSSeongJae Park * it. 876b9a6ac4eSSeongJae Park */ 877b9a6ac4eSSeongJae Park static void kdamond_split_regions(struct damon_ctx *ctx) 878b9a6ac4eSSeongJae Park { 879b9a6ac4eSSeongJae Park struct damon_target *t; 880b9a6ac4eSSeongJae Park unsigned int nr_regions = 0; 881b9a6ac4eSSeongJae Park static unsigned int last_nr_regions; 882b9a6ac4eSSeongJae Park int nr_subregions = 2; 883b9a6ac4eSSeongJae Park 884b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 885b9a6ac4eSSeongJae Park nr_regions += damon_nr_regions(t); 886b9a6ac4eSSeongJae Park 887b9a6ac4eSSeongJae Park if (nr_regions > ctx->max_nr_regions / 2) 888b9a6ac4eSSeongJae Park return; 889b9a6ac4eSSeongJae Park 890b9a6ac4eSSeongJae Park /* Maybe the middle of the region has different access frequency */ 891b9a6ac4eSSeongJae Park if (last_nr_regions == nr_regions && 892b9a6ac4eSSeongJae Park nr_regions < ctx->max_nr_regions / 3) 893b9a6ac4eSSeongJae Park nr_subregions = 3; 894b9a6ac4eSSeongJae Park 895b9a6ac4eSSeongJae Park damon_for_each_target(t, ctx) 896b9a6ac4eSSeongJae Park damon_split_regions_of(ctx, t, nr_subregions); 897b9a6ac4eSSeongJae Park 898b9a6ac4eSSeongJae Park last_nr_regions = nr_regions; 899b9a6ac4eSSeongJae Park } 900b9a6ac4eSSeongJae Park 901f23b8eeeSSeongJae Park /* 9022224d848SSeongJae Park * Check whether it is time to check and apply the target monitoring regions 9032224d848SSeongJae Park * 9042224d848SSeongJae Park * Returns true if it is. 9052224d848SSeongJae Park */ 9062224d848SSeongJae Park static bool kdamond_need_update_primitive(struct damon_ctx *ctx) 9072224d848SSeongJae Park { 9082224d848SSeongJae Park return damon_check_reset_time_interval(&ctx->last_primitive_update, 9092224d848SSeongJae Park ctx->primitive_update_interval); 9102224d848SSeongJae Park } 9112224d848SSeongJae Park 9122224d848SSeongJae Park /* 9132224d848SSeongJae Park * Check whether current monitoring should be stopped 9142224d848SSeongJae Park * 9152224d848SSeongJae Park * The monitoring is stopped when either the user requested to stop, or all 9162224d848SSeongJae Park * monitoring targets are invalid. 9172224d848SSeongJae Park * 9182224d848SSeongJae Park * Returns true if need to stop current monitoring. 9192224d848SSeongJae Park */ 9202224d848SSeongJae Park static bool kdamond_need_stop(struct damon_ctx *ctx) 9212224d848SSeongJae Park { 922f23b8eeeSSeongJae Park struct damon_target *t; 9232224d848SSeongJae Park bool stop; 9242224d848SSeongJae Park 9252224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 9262224d848SSeongJae Park stop = ctx->kdamond_stop; 9272224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 9282224d848SSeongJae Park if (stop) 9292224d848SSeongJae Park return true; 9302224d848SSeongJae Park 9312224d848SSeongJae Park if (!ctx->primitive.target_valid) 9322224d848SSeongJae Park return false; 9332224d848SSeongJae Park 934f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 935f23b8eeeSSeongJae Park if (ctx->primitive.target_valid(t)) 936f23b8eeeSSeongJae Park return false; 937f23b8eeeSSeongJae Park } 938f23b8eeeSSeongJae Park 939f23b8eeeSSeongJae Park return true; 9402224d848SSeongJae Park } 9412224d848SSeongJae Park 942*ee801b7dSSeongJae Park static unsigned long damos_wmark_metric_value(enum damos_wmark_metric metric) 943*ee801b7dSSeongJae Park { 944*ee801b7dSSeongJae Park struct sysinfo i; 945*ee801b7dSSeongJae Park 946*ee801b7dSSeongJae Park switch (metric) { 947*ee801b7dSSeongJae Park case DAMOS_WMARK_FREE_MEM_RATE: 948*ee801b7dSSeongJae Park si_meminfo(&i); 949*ee801b7dSSeongJae Park return i.freeram * 1000 / i.totalram; 950*ee801b7dSSeongJae Park default: 951*ee801b7dSSeongJae Park break; 952*ee801b7dSSeongJae Park } 953*ee801b7dSSeongJae Park return -EINVAL; 954*ee801b7dSSeongJae Park } 955*ee801b7dSSeongJae Park 956*ee801b7dSSeongJae Park /* 957*ee801b7dSSeongJae Park * Returns zero if the scheme is active. Else, returns time to wait for next 958*ee801b7dSSeongJae Park * watermark check in micro-seconds. 959*ee801b7dSSeongJae Park */ 960*ee801b7dSSeongJae Park static unsigned long damos_wmark_wait_us(struct damos *scheme) 961*ee801b7dSSeongJae Park { 962*ee801b7dSSeongJae Park unsigned long metric; 963*ee801b7dSSeongJae Park 964*ee801b7dSSeongJae Park if (scheme->wmarks.metric == DAMOS_WMARK_NONE) 965*ee801b7dSSeongJae Park return 0; 966*ee801b7dSSeongJae Park 967*ee801b7dSSeongJae Park metric = damos_wmark_metric_value(scheme->wmarks.metric); 968*ee801b7dSSeongJae Park /* higher than high watermark or lower than low watermark */ 969*ee801b7dSSeongJae Park if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) { 970*ee801b7dSSeongJae Park if (scheme->wmarks.activated) 971*ee801b7dSSeongJae Park pr_debug("inactivate a scheme (%d) for %s wmark\n", 972*ee801b7dSSeongJae Park scheme->action, 973*ee801b7dSSeongJae Park metric > scheme->wmarks.high ? 974*ee801b7dSSeongJae Park "high" : "low"); 975*ee801b7dSSeongJae Park scheme->wmarks.activated = false; 976*ee801b7dSSeongJae Park return scheme->wmarks.interval; 977*ee801b7dSSeongJae Park } 978*ee801b7dSSeongJae Park 979*ee801b7dSSeongJae Park /* inactive and higher than middle watermark */ 980*ee801b7dSSeongJae Park if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) && 981*ee801b7dSSeongJae Park !scheme->wmarks.activated) 982*ee801b7dSSeongJae Park return scheme->wmarks.interval; 983*ee801b7dSSeongJae Park 984*ee801b7dSSeongJae Park if (!scheme->wmarks.activated) 985*ee801b7dSSeongJae Park pr_debug("activate a scheme (%d)\n", scheme->action); 986*ee801b7dSSeongJae Park scheme->wmarks.activated = true; 987*ee801b7dSSeongJae Park return 0; 988*ee801b7dSSeongJae Park } 989*ee801b7dSSeongJae Park 990*ee801b7dSSeongJae Park static void kdamond_usleep(unsigned long usecs) 991*ee801b7dSSeongJae Park { 992*ee801b7dSSeongJae Park if (usecs > 100 * 1000) 993*ee801b7dSSeongJae Park schedule_timeout_interruptible(usecs_to_jiffies(usecs)); 994*ee801b7dSSeongJae Park else 995*ee801b7dSSeongJae Park usleep_range(usecs, usecs + 1); 996*ee801b7dSSeongJae Park } 997*ee801b7dSSeongJae Park 998*ee801b7dSSeongJae Park /* Returns negative error code if it's not activated but should return */ 999*ee801b7dSSeongJae Park static int kdamond_wait_activation(struct damon_ctx *ctx) 1000*ee801b7dSSeongJae Park { 1001*ee801b7dSSeongJae Park struct damos *s; 1002*ee801b7dSSeongJae Park unsigned long wait_time; 1003*ee801b7dSSeongJae Park unsigned long min_wait_time = 0; 1004*ee801b7dSSeongJae Park 1005*ee801b7dSSeongJae Park while (!kdamond_need_stop(ctx)) { 1006*ee801b7dSSeongJae Park damon_for_each_scheme(s, ctx) { 1007*ee801b7dSSeongJae Park wait_time = damos_wmark_wait_us(s); 1008*ee801b7dSSeongJae Park if (!min_wait_time || wait_time < min_wait_time) 1009*ee801b7dSSeongJae Park min_wait_time = wait_time; 1010*ee801b7dSSeongJae Park } 1011*ee801b7dSSeongJae Park if (!min_wait_time) 1012*ee801b7dSSeongJae Park return 0; 1013*ee801b7dSSeongJae Park 1014*ee801b7dSSeongJae Park kdamond_usleep(min_wait_time); 1015*ee801b7dSSeongJae Park } 1016*ee801b7dSSeongJae Park return -EBUSY; 1017*ee801b7dSSeongJae Park } 1018*ee801b7dSSeongJae Park 10192224d848SSeongJae Park static void set_kdamond_stop(struct damon_ctx *ctx) 10202224d848SSeongJae Park { 10212224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 10222224d848SSeongJae Park ctx->kdamond_stop = true; 10232224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 10242224d848SSeongJae Park } 10252224d848SSeongJae Park 10262224d848SSeongJae Park /* 10272224d848SSeongJae Park * The monitoring daemon that runs as a kernel thread 10282224d848SSeongJae Park */ 10292224d848SSeongJae Park static int kdamond_fn(void *data) 10302224d848SSeongJae Park { 10312224d848SSeongJae Park struct damon_ctx *ctx = (struct damon_ctx *)data; 1032f23b8eeeSSeongJae Park struct damon_target *t; 1033f23b8eeeSSeongJae Park struct damon_region *r, *next; 1034b9a6ac4eSSeongJae Park unsigned int max_nr_accesses = 0; 1035b9a6ac4eSSeongJae Park unsigned long sz_limit = 0; 10362224d848SSeongJae Park 103742e4cef5SChangbin Du pr_debug("kdamond (%d) starts\n", current->pid); 10382224d848SSeongJae Park 10392224d848SSeongJae Park if (ctx->primitive.init) 10402224d848SSeongJae Park ctx->primitive.init(ctx); 10412224d848SSeongJae Park if (ctx->callback.before_start && ctx->callback.before_start(ctx)) 10422224d848SSeongJae Park set_kdamond_stop(ctx); 10432224d848SSeongJae Park 1044b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 1045b9a6ac4eSSeongJae Park 10462224d848SSeongJae Park while (!kdamond_need_stop(ctx)) { 1047*ee801b7dSSeongJae Park if (kdamond_wait_activation(ctx)) 1048*ee801b7dSSeongJae Park continue; 1049*ee801b7dSSeongJae Park 10502224d848SSeongJae Park if (ctx->primitive.prepare_access_checks) 10512224d848SSeongJae Park ctx->primitive.prepare_access_checks(ctx); 10522224d848SSeongJae Park if (ctx->callback.after_sampling && 10532224d848SSeongJae Park ctx->callback.after_sampling(ctx)) 10542224d848SSeongJae Park set_kdamond_stop(ctx); 10552224d848SSeongJae Park 10562224d848SSeongJae Park usleep_range(ctx->sample_interval, ctx->sample_interval + 1); 10572224d848SSeongJae Park 10582224d848SSeongJae Park if (ctx->primitive.check_accesses) 1059b9a6ac4eSSeongJae Park max_nr_accesses = ctx->primitive.check_accesses(ctx); 10602224d848SSeongJae Park 10612224d848SSeongJae Park if (kdamond_aggregate_interval_passed(ctx)) { 1062b9a6ac4eSSeongJae Park kdamond_merge_regions(ctx, 1063b9a6ac4eSSeongJae Park max_nr_accesses / 10, 1064b9a6ac4eSSeongJae Park sz_limit); 10652224d848SSeongJae Park if (ctx->callback.after_aggregation && 10662224d848SSeongJae Park ctx->callback.after_aggregation(ctx)) 10672224d848SSeongJae Park set_kdamond_stop(ctx); 10681f366e42SSeongJae Park kdamond_apply_schemes(ctx); 1069f23b8eeeSSeongJae Park kdamond_reset_aggregated(ctx); 1070b9a6ac4eSSeongJae Park kdamond_split_regions(ctx); 10712224d848SSeongJae Park if (ctx->primitive.reset_aggregated) 10722224d848SSeongJae Park ctx->primitive.reset_aggregated(ctx); 10732224d848SSeongJae Park } 10742224d848SSeongJae Park 10752224d848SSeongJae Park if (kdamond_need_update_primitive(ctx)) { 10762224d848SSeongJae Park if (ctx->primitive.update) 10772224d848SSeongJae Park ctx->primitive.update(ctx); 1078b9a6ac4eSSeongJae Park sz_limit = damon_region_sz_limit(ctx); 10792224d848SSeongJae Park } 10802224d848SSeongJae Park } 1081f23b8eeeSSeongJae Park damon_for_each_target(t, ctx) { 1082f23b8eeeSSeongJae Park damon_for_each_region_safe(r, next, t) 1083b9a6ac4eSSeongJae Park damon_destroy_region(r, t); 1084f23b8eeeSSeongJae Park } 10852224d848SSeongJae Park 10862224d848SSeongJae Park if (ctx->callback.before_terminate && 10872224d848SSeongJae Park ctx->callback.before_terminate(ctx)) 10882224d848SSeongJae Park set_kdamond_stop(ctx); 10892224d848SSeongJae Park if (ctx->primitive.cleanup) 10902224d848SSeongJae Park ctx->primitive.cleanup(ctx); 10912224d848SSeongJae Park 109242e4cef5SChangbin Du pr_debug("kdamond (%d) finishes\n", current->pid); 10932224d848SSeongJae Park mutex_lock(&ctx->kdamond_lock); 10942224d848SSeongJae Park ctx->kdamond = NULL; 10952224d848SSeongJae Park mutex_unlock(&ctx->kdamond_lock); 10962224d848SSeongJae Park 10972224d848SSeongJae Park mutex_lock(&damon_lock); 10982224d848SSeongJae Park nr_running_ctxs--; 10992224d848SSeongJae Park mutex_unlock(&damon_lock); 11002224d848SSeongJae Park 11015f7fe2b9SChangbin Du return 0; 11022224d848SSeongJae Park } 110317ccae8bSSeongJae Park 110417ccae8bSSeongJae Park #include "core-test.h" 1105