1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON-based page reclamation 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #define pr_fmt(fmt) "damon-reclaim: " fmt 9 10 #include <linux/damon.h> 11 #include <linux/kstrtox.h> 12 #include <linux/module.h> 13 14 #include "modules-common.h" 15 16 #ifdef MODULE_PARAM_PREFIX 17 #undef MODULE_PARAM_PREFIX 18 #endif 19 #define MODULE_PARAM_PREFIX "damon_reclaim." 20 21 /* 22 * Enable or disable DAMON_RECLAIM. 23 * 24 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. 25 * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could 26 * do no real monitoring and reclamation due to the watermarks-based activation 27 * condition. Refer to below descriptions for the watermarks parameter for 28 * this. 29 */ 30 static bool enabled __read_mostly; 31 32 /* 33 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``. 34 * 35 * Input parameters that updated while DAMON_RECLAIM is running are not applied 36 * by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values 37 * of parametrs except ``enabled`` again. Once the re-reading is done, this 38 * parameter is set as ``N``. If invalid parameters are found while the 39 * re-reading, DAMON_RECLAIM will be disabled. 40 */ 41 static bool commit_inputs __read_mostly; 42 module_param(commit_inputs, bool, 0600); 43 44 /* 45 * Time threshold for cold memory regions identification in microseconds. 46 * 47 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM 48 * identifies the region as cold, and reclaims. 120 seconds by default. 49 */ 50 static unsigned long min_age __read_mostly = 120000000; 51 module_param(min_age, ulong, 0600); 52 53 static struct damos_quota damon_reclaim_quota = { 54 /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ 55 .ms = 10, 56 .sz = 128 * 1024 * 1024, 57 .reset_interval = 1000, 58 /* Within the quota, page out older regions first. */ 59 .weight_sz = 0, 60 .weight_nr_accesses = 0, 61 .weight_age = 1 62 }; 63 DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); 64 65 static struct damos_watermarks damon_reclaim_wmarks = { 66 .metric = DAMOS_WMARK_FREE_MEM_RATE, 67 .interval = 5000000, /* 5 seconds */ 68 .high = 500, /* 50 percent */ 69 .mid = 400, /* 40 percent */ 70 .low = 200, /* 20 percent */ 71 }; 72 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); 73 74 static struct damon_attrs damon_reclaim_mon_attrs = { 75 .sample_interval = 5000, /* 5 ms */ 76 .aggr_interval = 100000, /* 100 ms */ 77 .ops_update_interval = 0, 78 .min_nr_regions = 10, 79 .max_nr_regions = 1000, 80 }; 81 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); 82 83 /* 84 * Start of the target memory region in physical address. 85 * 86 * The start physical address of memory region that DAMON_RECLAIM will do work 87 * against. By default, biggest System RAM is used as the region. 88 */ 89 static unsigned long monitor_region_start __read_mostly; 90 module_param(monitor_region_start, ulong, 0600); 91 92 /* 93 * End of the target memory region in physical address. 94 * 95 * The end physical address of memory region that DAMON_RECLAIM will do work 96 * against. By default, biggest System RAM is used as the region. 97 */ 98 static unsigned long monitor_region_end __read_mostly; 99 module_param(monitor_region_end, ulong, 0600); 100 101 /* 102 * Skip anonymous pages reclamation. 103 * 104 * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous 105 * pages. By default, ``N``. 106 */ 107 static bool skip_anon __read_mostly; 108 module_param(skip_anon, bool, 0600); 109 110 /* 111 * PID of the DAMON thread 112 * 113 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. 114 * Else, -1. 115 */ 116 static int kdamond_pid __read_mostly = -1; 117 module_param(kdamond_pid, int, 0400); 118 119 static struct damos_stat damon_reclaim_stat; 120 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, 121 reclaim_tried_regions, reclaimed_regions, quota_exceeds); 122 123 static struct damon_ctx *ctx; 124 static struct damon_target *target; 125 126 static struct damos *damon_reclaim_new_scheme(void) 127 { 128 struct damos_access_pattern pattern = { 129 /* Find regions having PAGE_SIZE or larger size */ 130 .min_sz_region = PAGE_SIZE, 131 .max_sz_region = ULONG_MAX, 132 /* and not accessed at all */ 133 .min_nr_accesses = 0, 134 .max_nr_accesses = 0, 135 /* for min_age or more micro-seconds */ 136 .min_age_region = min_age / 137 damon_reclaim_mon_attrs.aggr_interval, 138 .max_age_region = UINT_MAX, 139 }; 140 141 return damon_new_scheme( 142 &pattern, 143 /* page out those, as soon as found */ 144 DAMOS_PAGEOUT, 145 /* for each aggregation interval */ 146 0, 147 /* under the quota. */ 148 &damon_reclaim_quota, 149 /* (De)activate this according to the watermarks. */ 150 &damon_reclaim_wmarks); 151 } 152 153 static void damon_reclaim_copy_quota_status(struct damos_quota *dst, 154 struct damos_quota *src) 155 { 156 dst->total_charged_sz = src->total_charged_sz; 157 dst->total_charged_ns = src->total_charged_ns; 158 dst->charged_sz = src->charged_sz; 159 dst->charged_from = src->charged_from; 160 dst->charge_target_from = src->charge_target_from; 161 dst->charge_addr_from = src->charge_addr_from; 162 } 163 164 static int damon_reclaim_apply_parameters(void) 165 { 166 struct damos *scheme, *old_scheme; 167 struct damos_filter *filter; 168 int err = 0; 169 170 err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs); 171 if (err) 172 return err; 173 174 /* Will be freed by next 'damon_set_schemes()' below */ 175 scheme = damon_reclaim_new_scheme(); 176 if (!scheme) 177 return -ENOMEM; 178 if (!list_empty(&ctx->schemes)) { 179 damon_for_each_scheme(old_scheme, ctx) 180 damon_reclaim_copy_quota_status(&scheme->quota, 181 &old_scheme->quota); 182 } 183 if (skip_anon) { 184 filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true); 185 if (!filter) { 186 /* Will be freed by next 'damon_set_schemes()' below */ 187 damon_destroy_scheme(scheme); 188 return -ENOMEM; 189 } 190 damos_add_filter(scheme, filter); 191 } 192 damon_set_schemes(ctx, &scheme, 1); 193 194 return damon_set_region_biggest_system_ram_default(target, 195 &monitor_region_start, 196 &monitor_region_end); 197 } 198 199 static int damon_reclaim_turn(bool on) 200 { 201 int err; 202 203 if (!on) { 204 err = damon_stop(&ctx, 1); 205 if (!err) 206 kdamond_pid = -1; 207 return err; 208 } 209 210 err = damon_reclaim_apply_parameters(); 211 if (err) 212 return err; 213 214 err = damon_start(&ctx, 1, true); 215 if (err) 216 return err; 217 kdamond_pid = ctx->kdamond->pid; 218 return 0; 219 } 220 221 static int damon_reclaim_enabled_store(const char *val, 222 const struct kernel_param *kp) 223 { 224 bool is_enabled = enabled; 225 bool enable; 226 int err; 227 228 err = kstrtobool(val, &enable); 229 if (err) 230 return err; 231 232 if (is_enabled == enable) 233 return 0; 234 235 /* Called before init function. The function will handle this. */ 236 if (!ctx) 237 goto set_param_out; 238 239 err = damon_reclaim_turn(enable); 240 if (err) 241 return err; 242 243 set_param_out: 244 enabled = enable; 245 return err; 246 } 247 248 static const struct kernel_param_ops enabled_param_ops = { 249 .set = damon_reclaim_enabled_store, 250 .get = param_get_bool, 251 }; 252 253 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 254 MODULE_PARM_DESC(enabled, 255 "Enable or disable DAMON_RECLAIM (default: disabled)"); 256 257 static int damon_reclaim_handle_commit_inputs(void) 258 { 259 int err; 260 261 if (!commit_inputs) 262 return 0; 263 264 err = damon_reclaim_apply_parameters(); 265 commit_inputs = false; 266 return err; 267 } 268 269 static int damon_reclaim_after_aggregation(struct damon_ctx *c) 270 { 271 struct damos *s; 272 273 /* update the stats parameter */ 274 damon_for_each_scheme(s, c) 275 damon_reclaim_stat = s->stat; 276 277 return damon_reclaim_handle_commit_inputs(); 278 } 279 280 static int damon_reclaim_after_wmarks_check(struct damon_ctx *c) 281 { 282 return damon_reclaim_handle_commit_inputs(); 283 } 284 285 static int __init damon_reclaim_init(void) 286 { 287 int err = damon_modules_new_paddr_ctx_target(&ctx, &target); 288 289 if (err) 290 return err; 291 292 ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check; 293 ctx->callback.after_aggregation = damon_reclaim_after_aggregation; 294 295 /* 'enabled' has set before this function, probably via command line */ 296 if (enabled) 297 err = damon_reclaim_turn(true); 298 299 return err; 300 } 301 302 module_init(damon_reclaim_init); 303