1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * DAMON-based page reclamation
  4 *
  5 * Author: SeongJae Park <sj@kernel.org>
  6 */
  7
  8#define pr_fmt(fmt) "damon-reclaim: " fmt
  9
 10#include <linux/damon.h>
 11#include <linux/kstrtox.h>
 12#include <linux/module.h>
 13
 14#include "modules-common.h"
 15
 16#ifdef MODULE_PARAM_PREFIX
 17#undef MODULE_PARAM_PREFIX
 18#endif
 19#define MODULE_PARAM_PREFIX "damon_reclaim."
 20
 21/*
 22 * Enable or disable DAMON_RECLAIM.
 23 *
 24 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
 25 * Setting it as ``N`` disables DAMON_RECLAIM.  Note that DAMON_RECLAIM could
 26 * do no real monitoring and reclamation due to the watermarks-based activation
 27 * condition.  Refer to below descriptions for the watermarks parameter for
 28 * this.
 29 */
 30static bool enabled __read_mostly;
 31
 32/*
 33 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
 34 *
 35 * Input parameters that updated while DAMON_RECLAIM is running are not applied
 36 * by default.  Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
 37 * of parametrs except ``enabled`` again.  Once the re-reading is done, this
 38 * parameter is set as ``N``.  If invalid parameters are found while the
 39 * re-reading, DAMON_RECLAIM will be disabled.
 40 */
 41static bool commit_inputs __read_mostly;
 42module_param(commit_inputs, bool, 0600);
 43
 44/*
 45 * Time threshold for cold memory regions identification in microseconds.
 46 *
 47 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM
 48 * identifies the region as cold, and reclaims.  120 seconds by default.
 49 */
 50static unsigned long min_age __read_mostly = 120000000;
 51module_param(min_age, ulong, 0600);
 52
 53static struct damos_quota damon_reclaim_quota = {
 54	/* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */
 55	.ms = 10,
 56	.sz = 128 * 1024 * 1024,
 57	.reset_interval = 1000,
 58	/* Within the quota, page out older regions first. */
 59	.weight_sz = 0,
 60	.weight_nr_accesses = 0,
 61	.weight_age = 1
 62};
 63DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota);
 64
 65/*
 66 * Desired level of memory pressure-stall time in microseconds.
 67 *
 68 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically
 69 * increases and decreases the effective level of the quota aiming this level of
 70 * memory pressure is incurred.  System-wide ``some`` memory PSI in microseconds
 71 * per quota reset interval (``quota_reset_interval_ms``) is collected and
 72 * compared to this value to see if the aim is satisfied.  Value zero means
 73 * disabling this auto-tuning feature.
 74 *
 75 * Disabled by default.
 76 */
 77static unsigned long quota_mem_pressure_us __read_mostly;
 78module_param(quota_mem_pressure_us, ulong, 0600);
 79
 80/*
 81 * User-specifiable feedback for auto-tuning of the effective quota.
 82 *
 83 * While keeping the caps that set by other quotas, DAMON_RECLAIM automatically
 84 * increases and decreases the effective level of the quota aiming receiving this
 85 * feedback of value ``10,000`` from the user.  DAMON_RECLAIM assumes the feedback
 86 * value and the quota are positively proportional.  Value zero means disabling
 87 * this auto-tuning feature.
 88 *
 89 * Disabled by default.
 90 *
 91 */
 92static unsigned long quota_autotune_feedback __read_mostly;
 93module_param(quota_autotune_feedback, ulong, 0600);
 94
 95static struct damos_watermarks damon_reclaim_wmarks = {
 96	.metric = DAMOS_WMARK_FREE_MEM_RATE,
 97	.interval = 5000000,	/* 5 seconds */
 98	.high = 500,		/* 50 percent */
 99	.mid = 400,		/* 40 percent */
100	.low = 200,		/* 20 percent */
101};
102DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks);
103
104static struct damon_attrs damon_reclaim_mon_attrs = {
105	.sample_interval = 5000,	/* 5 ms */
106	.aggr_interval = 100000,	/* 100 ms */
107	.ops_update_interval = 0,
108	.min_nr_regions = 10,
109	.max_nr_regions = 1000,
110};
111DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);
112
113/*
114 * Start of the target memory region in physical address.
115 *
116 * The start physical address of memory region that DAMON_RECLAIM will do work
117 * against.  By default, biggest System RAM is used as the region.
118 */
119static unsigned long monitor_region_start __read_mostly;
120module_param(monitor_region_start, ulong, 0600);
121
122/*
123 * End of the target memory region in physical address.
124 *
125 * The end physical address of memory region that DAMON_RECLAIM will do work
126 * against.  By default, biggest System RAM is used as the region.
127 */
128static unsigned long monitor_region_end __read_mostly;
129module_param(monitor_region_end, ulong, 0600);
130
131/*
132 * Skip anonymous pages reclamation.
133 *
134 * If this parameter is set as ``Y``, DAMON_RECLAIM does not reclaim anonymous
135 * pages.  By default, ``N``.
136 */
137static bool skip_anon __read_mostly;
138module_param(skip_anon, bool, 0600);
139
140/*
141 * PID of the DAMON thread
142 *
143 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
144 * Else, -1.
145 */
146static int kdamond_pid __read_mostly = -1;
147module_param(kdamond_pid, int, 0400);
148
149static struct damos_stat damon_reclaim_stat;
150DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
151		reclaim_tried_regions, reclaimed_regions, quota_exceeds);
152
153static struct damon_ctx *ctx;
154static struct damon_target *target;
155
156static struct damos *damon_reclaim_new_scheme(void)
157{
158	struct damos_access_pattern pattern = {
159		/* Find regions having PAGE_SIZE or larger size */
160		.min_sz_region = PAGE_SIZE,
161		.max_sz_region = ULONG_MAX,
162		/* and not accessed at all */
163		.min_nr_accesses = 0,
164		.max_nr_accesses = 0,
165		/* for min_age or more micro-seconds */
166		.min_age_region = min_age /
167			damon_reclaim_mon_attrs.aggr_interval,
168		.max_age_region = UINT_MAX,
169	};
170
171	return damon_new_scheme(
172			&pattern,
173			/* page out those, as soon as found */
174			DAMOS_PAGEOUT,
175			/* for each aggregation interval */
176			0,
177			/* under the quota. */
178			&damon_reclaim_quota,
179			/* (De)activate this according to the watermarks. */
180			&damon_reclaim_wmarks,
181			NUMA_NO_NODE);
182}
183
184static int damon_reclaim_apply_parameters(void)
185{
186	struct damon_ctx *param_ctx;
187	struct damon_target *param_target;
188	struct damos *scheme;
189	struct damos_quota_goal *goal;
190	struct damos_filter *filter;
191	int err;
192
193	err = damon_modules_new_paddr_ctx_target(&param_ctx, &param_target);
194	if (err)
195		return err;
196
197	err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs);
198	if (err)
199		goto out;
200
201	err = -ENOMEM;
202	scheme = damon_reclaim_new_scheme();
203	if (!scheme)
204		goto out;
205	damon_set_schemes(ctx, &scheme, 1);
206
207	if (quota_mem_pressure_us) {
208		goal = damos_new_quota_goal(DAMOS_QUOTA_SOME_MEM_PSI_US,
209				quota_mem_pressure_us);
210		if (!goal)
211			goto out;
212		damos_add_quota_goal(&scheme->quota, goal);
213	}
214
215	if (quota_autotune_feedback) {
216		goal = damos_new_quota_goal(DAMOS_QUOTA_USER_INPUT, 10000);
217		if (!goal)
218			goto out;
219		goal->current_value = quota_autotune_feedback;
220		damos_add_quota_goal(&scheme->quota, goal);
221	}
222
223	if (skip_anon) {
224		filter = damos_new_filter(DAMOS_FILTER_TYPE_ANON, true);
225		if (!filter)
226			goto out;
227		damos_add_filter(scheme, filter);
228	}
229
230	err = damon_set_region_biggest_system_ram_default(param_target,
231					&monitor_region_start,
232					&monitor_region_end);
233	if (err)
234		goto out;
235	err = damon_commit_ctx(ctx, param_ctx);
236out:
237	damon_destroy_ctx(param_ctx);
238	return err;
239}
240
241static int damon_reclaim_turn(bool on)
242{
243	int err;
244
245	if (!on) {
246		err = damon_stop(&ctx, 1);
247		if (!err)
248			kdamond_pid = -1;
249		return err;
250	}
251
252	err = damon_reclaim_apply_parameters();
253	if (err)
254		return err;
255
256	err = damon_start(&ctx, 1, true);
257	if (err)
258		return err;
259	kdamond_pid = ctx->kdamond->pid;
260	return 0;
261}
262
263static int damon_reclaim_enabled_store(const char *val,
264		const struct kernel_param *kp)
265{
266	bool is_enabled = enabled;
267	bool enable;
268	int err;
269
270	err = kstrtobool(val, &enable);
271	if (err)
272		return err;
273
274	if (is_enabled == enable)
275		return 0;
276
277	/* Called before init function.  The function will handle this. */
278	if (!ctx)
279		goto set_param_out;
280
281	err = damon_reclaim_turn(enable);
282	if (err)
283		return err;
284
285set_param_out:
286	enabled = enable;
287	return err;
288}
289
290static const struct kernel_param_ops enabled_param_ops = {
291	.set = damon_reclaim_enabled_store,
292	.get = param_get_bool,
293};
294
295module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
296MODULE_PARM_DESC(enabled,
297	"Enable or disable DAMON_RECLAIM (default: disabled)");
298
299static int damon_reclaim_handle_commit_inputs(void)
300{
301	int err;
302
303	if (!commit_inputs)
304		return 0;
305
306	err = damon_reclaim_apply_parameters();
307	commit_inputs = false;
308	return err;
309}
310
311static int damon_reclaim_after_aggregation(struct damon_ctx *c)
312{
313	struct damos *s;
314
315	/* update the stats parameter */
316	damon_for_each_scheme(s, c)
317		damon_reclaim_stat = s->stat;
318
319	return damon_reclaim_handle_commit_inputs();
320}
321
322static int damon_reclaim_after_wmarks_check(struct damon_ctx *c)
323{
324	return damon_reclaim_handle_commit_inputs();
325}
326
327static int __init damon_reclaim_init(void)
328{
329	int err = damon_modules_new_paddr_ctx_target(&ctx, &target);
330
331	if (err)
332		return err;
333
334	ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check;
335	ctx->callback.after_aggregation = damon_reclaim_after_aggregation;
336
337	/* 'enabled' has set before this function, probably via command line */
338	if (enabled)
339		err = damon_reclaim_turn(true);
340
341	return err;
342}
343
344module_init(damon_reclaim_init);