1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DAMON-based page reclamation 4 * 5 * Author: SeongJae Park <sj@kernel.org> 6 */ 7 8 #define pr_fmt(fmt) "damon-reclaim: " fmt 9 10 #include <linux/damon.h> 11 #include <linux/ioport.h> 12 #include <linux/module.h> 13 #include <linux/sched.h> 14 #include <linux/workqueue.h> 15 16 #include "modules-common.h" 17 18 #ifdef MODULE_PARAM_PREFIX 19 #undef MODULE_PARAM_PREFIX 20 #endif 21 #define MODULE_PARAM_PREFIX "damon_reclaim." 22 23 /* 24 * Enable or disable DAMON_RECLAIM. 25 * 26 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``. 27 * Setting it as ``N`` disables DAMON_RECLAIM. Note that DAMON_RECLAIM could 28 * do no real monitoring and reclamation due to the watermarks-based activation 29 * condition. Refer to below descriptions for the watermarks parameter for 30 * this. 31 */ 32 static bool enabled __read_mostly; 33 34 /* 35 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``. 36 * 37 * Input parameters that updated while DAMON_RECLAIM is running are not applied 38 * by default. Once this parameter is set as ``Y``, DAMON_RECLAIM reads values 39 * of parametrs except ``enabled`` again. Once the re-reading is done, this 40 * parameter is set as ``N``. If invalid parameters are found while the 41 * re-reading, DAMON_RECLAIM will be disabled. 42 */ 43 static bool commit_inputs __read_mostly; 44 module_param(commit_inputs, bool, 0600); 45 46 /* 47 * Time threshold for cold memory regions identification in microseconds. 48 * 49 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM 50 * identifies the region as cold, and reclaims. 120 seconds by default. 51 */ 52 static unsigned long min_age __read_mostly = 120000000; 53 module_param(min_age, ulong, 0600); 54 55 static struct damos_quota damon_reclaim_quota = { 56 /* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */ 57 .ms = 10, 58 .sz = 128 * 1024 * 1024, 59 .reset_interval = 1000, 60 /* Within the quota, page out older regions first. */ 61 .weight_sz = 0, 62 .weight_nr_accesses = 0, 63 .weight_age = 1 64 }; 65 DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota); 66 67 struct damos_watermarks damon_reclaim_wmarks = { 68 .metric = DAMOS_WMARK_FREE_MEM_RATE, 69 .interval = 5000000, /* 5 seconds */ 70 .high = 500, /* 50 percent */ 71 .mid = 400, /* 40 percent */ 72 .low = 200, /* 20 percent */ 73 }; 74 DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks); 75 76 static struct damon_attrs damon_reclaim_mon_attrs = { 77 .sample_interval = 5000, /* 5 ms */ 78 .aggr_interval = 100000, /* 100 ms */ 79 .ops_update_interval = 0, 80 .min_nr_regions = 10, 81 .max_nr_regions = 1000, 82 }; 83 DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs); 84 85 /* 86 * Start of the target memory region in physical address. 87 * 88 * The start physical address of memory region that DAMON_RECLAIM will do work 89 * against. By default, biggest System RAM is used as the region. 90 */ 91 static unsigned long monitor_region_start __read_mostly; 92 module_param(monitor_region_start, ulong, 0600); 93 94 /* 95 * End of the target memory region in physical address. 96 * 97 * The end physical address of memory region that DAMON_RECLAIM will do work 98 * against. By default, biggest System RAM is used as the region. 99 */ 100 static unsigned long monitor_region_end __read_mostly; 101 module_param(monitor_region_end, ulong, 0600); 102 103 /* 104 * PID of the DAMON thread 105 * 106 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread. 107 * Else, -1. 108 */ 109 static int kdamond_pid __read_mostly = -1; 110 module_param(kdamond_pid, int, 0400); 111 112 static struct damos_stat damon_reclaim_stat; 113 DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat, 114 reclaim_tried_regions, reclaimed_regions, quota_exceeds); 115 116 static struct damon_ctx *ctx; 117 static struct damon_target *target; 118 119 static struct damos *damon_reclaim_new_scheme(void) 120 { 121 struct damos_access_pattern pattern = { 122 /* Find regions having PAGE_SIZE or larger size */ 123 .min_sz_region = PAGE_SIZE, 124 .max_sz_region = ULONG_MAX, 125 /* and not accessed at all */ 126 .min_nr_accesses = 0, 127 .max_nr_accesses = 0, 128 /* for min_age or more micro-seconds */ 129 .min_age_region = min_age / 130 damon_reclaim_mon_attrs.aggr_interval, 131 .max_age_region = UINT_MAX, 132 }; 133 134 return damon_new_scheme( 135 &pattern, 136 /* page out those, as soon as found */ 137 DAMOS_PAGEOUT, 138 /* under the quota. */ 139 &damon_reclaim_quota, 140 /* (De)activate this according to the watermarks. */ 141 &damon_reclaim_wmarks); 142 } 143 144 static int damon_reclaim_apply_parameters(void) 145 { 146 struct damos *scheme; 147 struct damon_addr_range addr_range; 148 int err = 0; 149 150 err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs); 151 if (err) 152 return err; 153 154 /* Will be freed by next 'damon_set_schemes()' below */ 155 scheme = damon_reclaim_new_scheme(); 156 if (!scheme) 157 return -ENOMEM; 158 err = damon_set_schemes(ctx, &scheme, 1); 159 if (err) 160 return err; 161 162 if (monitor_region_start > monitor_region_end) 163 return -EINVAL; 164 if (!monitor_region_start && !monitor_region_end && 165 !damon_find_biggest_system_ram(&monitor_region_start, 166 &monitor_region_end)) 167 return -EINVAL; 168 addr_range.start = monitor_region_start; 169 addr_range.end = monitor_region_end; 170 return damon_set_regions(target, &addr_range, 1); 171 } 172 173 static int damon_reclaim_turn(bool on) 174 { 175 int err; 176 177 if (!on) { 178 err = damon_stop(&ctx, 1); 179 if (!err) 180 kdamond_pid = -1; 181 return err; 182 } 183 184 err = damon_reclaim_apply_parameters(); 185 if (err) 186 return err; 187 188 err = damon_start(&ctx, 1, true); 189 if (err) 190 return err; 191 kdamond_pid = ctx->kdamond->pid; 192 return 0; 193 } 194 195 static struct delayed_work damon_reclaim_timer; 196 static void damon_reclaim_timer_fn(struct work_struct *work) 197 { 198 static bool last_enabled; 199 bool now_enabled; 200 201 now_enabled = enabled; 202 if (last_enabled != now_enabled) { 203 if (!damon_reclaim_turn(now_enabled)) 204 last_enabled = now_enabled; 205 else 206 enabled = last_enabled; 207 } 208 } 209 static DECLARE_DELAYED_WORK(damon_reclaim_timer, damon_reclaim_timer_fn); 210 211 static bool damon_reclaim_initialized; 212 213 static int damon_reclaim_enabled_store(const char *val, 214 const struct kernel_param *kp) 215 { 216 int rc = param_set_bool(val, kp); 217 218 if (rc < 0) 219 return rc; 220 221 /* system_wq might not initialized yet */ 222 if (!damon_reclaim_initialized) 223 return rc; 224 225 schedule_delayed_work(&damon_reclaim_timer, 0); 226 return 0; 227 } 228 229 static const struct kernel_param_ops enabled_param_ops = { 230 .set = damon_reclaim_enabled_store, 231 .get = param_get_bool, 232 }; 233 234 module_param_cb(enabled, &enabled_param_ops, &enabled, 0600); 235 MODULE_PARM_DESC(enabled, 236 "Enable or disable DAMON_RECLAIM (default: disabled)"); 237 238 static int damon_reclaim_handle_commit_inputs(void) 239 { 240 int err; 241 242 if (!commit_inputs) 243 return 0; 244 245 err = damon_reclaim_apply_parameters(); 246 commit_inputs = false; 247 return err; 248 } 249 250 static int damon_reclaim_after_aggregation(struct damon_ctx *c) 251 { 252 struct damos *s; 253 254 /* update the stats parameter */ 255 damon_for_each_scheme(s, c) 256 damon_reclaim_stat = s->stat; 257 258 return damon_reclaim_handle_commit_inputs(); 259 } 260 261 static int damon_reclaim_after_wmarks_check(struct damon_ctx *c) 262 { 263 return damon_reclaim_handle_commit_inputs(); 264 } 265 266 static int __init damon_reclaim_init(void) 267 { 268 ctx = damon_new_ctx(); 269 if (!ctx) 270 return -ENOMEM; 271 272 if (damon_select_ops(ctx, DAMON_OPS_PADDR)) { 273 damon_destroy_ctx(ctx); 274 return -EINVAL; 275 } 276 277 ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check; 278 ctx->callback.after_aggregation = damon_reclaim_after_aggregation; 279 280 target = damon_new_target(); 281 if (!target) { 282 damon_destroy_ctx(ctx); 283 return -ENOMEM; 284 } 285 damon_add_target(ctx, target); 286 287 schedule_delayed_work(&damon_reclaim_timer, 0); 288 289 damon_reclaim_initialized = true; 290 return 0; 291 } 292 293 module_init(damon_reclaim_init); 294