1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * DAMON-based page reclamation
  4 *
  5 * Author: SeongJae Park <sj@kernel.org>
  6 */
  7
  8#define pr_fmt(fmt) "damon-reclaim: " fmt
  9
 10#include <linux/damon.h>
 11#include <linux/kstrtox.h>
 12#include <linux/module.h>
 13
 14#include "modules-common.h"
 15
 16#ifdef MODULE_PARAM_PREFIX
 17#undef MODULE_PARAM_PREFIX
 18#endif
 19#define MODULE_PARAM_PREFIX "damon_reclaim."
 20
 21/*
 22 * Enable or disable DAMON_RECLAIM.
 23 *
 24 * You can enable DAMON_RCLAIM by setting the value of this parameter as ``Y``.
 25 * Setting it as ``N`` disables DAMON_RECLAIM.  Note that DAMON_RECLAIM could
 26 * do no real monitoring and reclamation due to the watermarks-based activation
 27 * condition.  Refer to below descriptions for the watermarks parameter for
 28 * this.
 29 */
 30static bool enabled __read_mostly;
 31
 32/*
 33 * Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
 34 *
 35 * Input parameters that updated while DAMON_RECLAIM is running are not applied
 36 * by default.  Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
 37 * of parametrs except ``enabled`` again.  Once the re-reading is done, this
 38 * parameter is set as ``N``.  If invalid parameters are found while the
 39 * re-reading, DAMON_RECLAIM will be disabled.
 40 */
 41static bool commit_inputs __read_mostly;
 42module_param(commit_inputs, bool, 0600);
 43
 44/*
 45 * Time threshold for cold memory regions identification in microseconds.
 46 *
 47 * If a memory region is not accessed for this or longer time, DAMON_RECLAIM
 48 * identifies the region as cold, and reclaims.  120 seconds by default.
 49 */
 50static unsigned long min_age __read_mostly = 120000000;
 51module_param(min_age, ulong, 0600);
 52
 53static struct damos_quota damon_reclaim_quota = {
 54	/* use up to 10 ms time, reclaim up to 128 MiB per 1 sec by default */
 55	.ms = 10,
 56	.sz = 128 * 1024 * 1024,
 57	.reset_interval = 1000,
 58	/* Within the quota, page out older regions first. */
 59	.weight_sz = 0,
 60	.weight_nr_accesses = 0,
 61	.weight_age = 1
 62};
 63DEFINE_DAMON_MODULES_DAMOS_QUOTAS(damon_reclaim_quota);
 64
 65static struct damos_watermarks damon_reclaim_wmarks = {
 66	.metric = DAMOS_WMARK_FREE_MEM_RATE,
 67	.interval = 5000000,	/* 5 seconds */
 68	.high = 500,		/* 50 percent */
 69	.mid = 400,		/* 40 percent */
 70	.low = 200,		/* 20 percent */
 71};
 72DEFINE_DAMON_MODULES_WMARKS_PARAMS(damon_reclaim_wmarks);
 73
 74static struct damon_attrs damon_reclaim_mon_attrs = {
 75	.sample_interval = 5000,	/* 5 ms */
 76	.aggr_interval = 100000,	/* 100 ms */
 77	.ops_update_interval = 0,
 78	.min_nr_regions = 10,
 79	.max_nr_regions = 1000,
 80};
 81DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);
 82
 83/*
 84 * Start of the target memory region in physical address.
 85 *
 86 * The start physical address of memory region that DAMON_RECLAIM will do work
 87 * against.  By default, biggest System RAM is used as the region.
 88 */
 89static unsigned long monitor_region_start __read_mostly;
 90module_param(monitor_region_start, ulong, 0600);
 91
 92/*
 93 * End of the target memory region in physical address.
 94 *
 95 * The end physical address of memory region that DAMON_RECLAIM will do work
 96 * against.  By default, biggest System RAM is used as the region.
 97 */
 98static unsigned long monitor_region_end __read_mostly;
 99module_param(monitor_region_end, ulong, 0600);
100
101/*
102 * PID of the DAMON thread
103 *
104 * If DAMON_RECLAIM is enabled, this becomes the PID of the worker thread.
105 * Else, -1.
106 */
107static int kdamond_pid __read_mostly = -1;
108module_param(kdamond_pid, int, 0400);
109
110static struct damos_stat damon_reclaim_stat;
111DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
112		reclaim_tried_regions, reclaimed_regions, quota_exceeds);
113
114static struct damon_ctx *ctx;
115static struct damon_target *target;
116
117static struct damos *damon_reclaim_new_scheme(void)
118{
119	struct damos_access_pattern pattern = {
120		/* Find regions having PAGE_SIZE or larger size */
121		.min_sz_region = PAGE_SIZE,
122		.max_sz_region = ULONG_MAX,
123		/* and not accessed at all */
124		.min_nr_accesses = 0,
125		.max_nr_accesses = 0,
126		/* for min_age or more micro-seconds */
127		.min_age_region = min_age /
128			damon_reclaim_mon_attrs.aggr_interval,
129		.max_age_region = UINT_MAX,
130	};
131
132	return damon_new_scheme(
133			&pattern,
134			/* page out those, as soon as found */
135			DAMOS_PAGEOUT,
136			/* under the quota. */
137			&damon_reclaim_quota,
138			/* (De)activate this according to the watermarks. */
139			&damon_reclaim_wmarks);
140}
141
142static int damon_reclaim_apply_parameters(void)
143{
144	struct damos *scheme;
145	int err = 0;
146
147	err = damon_set_attrs(ctx, &damon_reclaim_mon_attrs);
148	if (err)
149		return err;
150
151	/* Will be freed by next 'damon_set_schemes()' below */
152	scheme = damon_reclaim_new_scheme();
153	if (!scheme)
154		return -ENOMEM;
155	damon_set_schemes(ctx, &scheme, 1);
156
157	return damon_set_region_biggest_system_ram_default(target,
158					&monitor_region_start,
159					&monitor_region_end);
160}
161
162static int damon_reclaim_turn(bool on)
163{
164	int err;
165
166	if (!on) {
167		err = damon_stop(&ctx, 1);
168		if (!err)
169			kdamond_pid = -1;
170		return err;
171	}
172
173	err = damon_reclaim_apply_parameters();
174	if (err)
175		return err;
176
177	err = damon_start(&ctx, 1, true);
178	if (err)
179		return err;
180	kdamond_pid = ctx->kdamond->pid;
181	return 0;
182}
183
184static int damon_reclaim_enabled_store(const char *val,
185		const struct kernel_param *kp)
186{
187	bool is_enabled = enabled;
188	bool enable;
189	int err;
190
191	err = kstrtobool(val, &enable);
192	if (err)
193		return err;
194
195	if (is_enabled == enable)
196		return 0;
197
198	/* Called before init function.  The function will handle this. */
199	if (!ctx)
200		goto set_param_out;
201
202	err = damon_reclaim_turn(enable);
203	if (err)
204		return err;
205
206set_param_out:
207	enabled = enable;
208	return err;
209}
210
211static const struct kernel_param_ops enabled_param_ops = {
212	.set = damon_reclaim_enabled_store,
213	.get = param_get_bool,
214};
215
216module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
217MODULE_PARM_DESC(enabled,
218	"Enable or disable DAMON_RECLAIM (default: disabled)");
219
220static int damon_reclaim_handle_commit_inputs(void)
221{
222	int err;
223
224	if (!commit_inputs)
225		return 0;
226
227	err = damon_reclaim_apply_parameters();
228	commit_inputs = false;
229	return err;
230}
231
232static int damon_reclaim_after_aggregation(struct damon_ctx *c)
233{
234	struct damos *s;
235
236	/* update the stats parameter */
237	damon_for_each_scheme(s, c)
238		damon_reclaim_stat = s->stat;
239
240	return damon_reclaim_handle_commit_inputs();
241}
242
243static int damon_reclaim_after_wmarks_check(struct damon_ctx *c)
244{
245	return damon_reclaim_handle_commit_inputs();
246}
247
248static int __init damon_reclaim_init(void)
249{
250	int err = damon_modules_new_paddr_ctx_target(&ctx, &target);
251
252	if (err)
253		return err;
254
255	ctx->callback.after_wmarks_check = damon_reclaim_after_wmarks_check;
256	ctx->callback.after_aggregation = damon_reclaim_after_aggregation;
257
258	/* 'enabled' has set before this function, probably via command line */
259	if (enabled)
260		err = damon_reclaim_turn(true);
261
262	return err;
263}
264
265module_init(damon_reclaim_init);