Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * DAMON sysfs Interface
4 *
5 * Copyright (c) 2022 SeongJae Park <sj@kernel.org>
6 */
7
8#include <linux/slab.h>
9
10#include "sysfs-common.h"
11
12/*
13 * scheme region directory
14 */
15
16struct damon_sysfs_scheme_region {
17 struct kobject kobj;
18 struct damon_addr_range ar;
19 unsigned int nr_accesses;
20 unsigned int age;
21 struct list_head list;
22};
23
24static struct damon_sysfs_scheme_region *damon_sysfs_scheme_region_alloc(
25 struct damon_region *region)
26{
27 struct damon_sysfs_scheme_region *sysfs_region = kmalloc(
28 sizeof(*sysfs_region), GFP_KERNEL);
29
30 if (!sysfs_region)
31 return NULL;
32 sysfs_region->kobj = (struct kobject){};
33 sysfs_region->ar = region->ar;
34 sysfs_region->nr_accesses = region->nr_accesses_bp / 10000;
35 sysfs_region->age = region->age;
36 INIT_LIST_HEAD(&sysfs_region->list);
37 return sysfs_region;
38}
39
40static ssize_t start_show(struct kobject *kobj, struct kobj_attribute *attr,
41 char *buf)
42{
43 struct damon_sysfs_scheme_region *region = container_of(kobj,
44 struct damon_sysfs_scheme_region, kobj);
45
46 return sysfs_emit(buf, "%lu\n", region->ar.start);
47}
48
49static ssize_t end_show(struct kobject *kobj, struct kobj_attribute *attr,
50 char *buf)
51{
52 struct damon_sysfs_scheme_region *region = container_of(kobj,
53 struct damon_sysfs_scheme_region, kobj);
54
55 return sysfs_emit(buf, "%lu\n", region->ar.end);
56}
57
58static ssize_t nr_accesses_show(struct kobject *kobj,
59 struct kobj_attribute *attr, char *buf)
60{
61 struct damon_sysfs_scheme_region *region = container_of(kobj,
62 struct damon_sysfs_scheme_region, kobj);
63
64 return sysfs_emit(buf, "%u\n", region->nr_accesses);
65}
66
67static ssize_t age_show(struct kobject *kobj, struct kobj_attribute *attr,
68 char *buf)
69{
70 struct damon_sysfs_scheme_region *region = container_of(kobj,
71 struct damon_sysfs_scheme_region, kobj);
72
73 return sysfs_emit(buf, "%u\n", region->age);
74}
75
76static void damon_sysfs_scheme_region_release(struct kobject *kobj)
77{
78 struct damon_sysfs_scheme_region *region = container_of(kobj,
79 struct damon_sysfs_scheme_region, kobj);
80
81 list_del(®ion->list);
82 kfree(region);
83}
84
85static struct kobj_attribute damon_sysfs_scheme_region_start_attr =
86 __ATTR_RO_MODE(start, 0400);
87
88static struct kobj_attribute damon_sysfs_scheme_region_end_attr =
89 __ATTR_RO_MODE(end, 0400);
90
91static struct kobj_attribute damon_sysfs_scheme_region_nr_accesses_attr =
92 __ATTR_RO_MODE(nr_accesses, 0400);
93
94static struct kobj_attribute damon_sysfs_scheme_region_age_attr =
95 __ATTR_RO_MODE(age, 0400);
96
97static struct attribute *damon_sysfs_scheme_region_attrs[] = {
98 &damon_sysfs_scheme_region_start_attr.attr,
99 &damon_sysfs_scheme_region_end_attr.attr,
100 &damon_sysfs_scheme_region_nr_accesses_attr.attr,
101 &damon_sysfs_scheme_region_age_attr.attr,
102 NULL,
103};
104ATTRIBUTE_GROUPS(damon_sysfs_scheme_region);
105
106static const struct kobj_type damon_sysfs_scheme_region_ktype = {
107 .release = damon_sysfs_scheme_region_release,
108 .sysfs_ops = &kobj_sysfs_ops,
109 .default_groups = damon_sysfs_scheme_region_groups,
110};
111
112/*
113 * scheme regions directory
114 */
115
116/*
117 * enum damos_sysfs_regions_upd_status - Represent DAMOS tried regions update
118 * status
119 * @DAMOS_TRIED_REGIONS_UPD_IDLE: Waiting for next request.
120 * @DAMOS_TRIED_REGIONS_UPD_STARTED: Update started.
121 * @DAMOS_TRIED_REGIONS_UPD_FINISHED: Update finished.
122 *
123 * Each DAMON-based operation scheme (&struct damos) has its own apply
124 * interval, and we need to expose the scheme tried regions based on only
125 * single snapshot. For this, we keep the tried regions update status for each
126 * scheme. The status becomes 'idle' at the beginning.
127 *
128 * Once the tried regions update request is received, the request handling
129 * start function (damon_sysfs_scheme_update_regions_start()) sets the status
130 * of all schemes as 'idle' again, and register ->before_damos_apply() and
131 * ->after_sampling() callbacks.
132 *
133 * Then, the first followup ->before_damos_apply() callback
134 * (damon_sysfs_before_damos_apply()) sets the status 'started'. The first
135 * ->after_sampling() callback (damon_sysfs_after_sampling()) after the call
136 * is called only after the scheme is completely applied
137 * to the given snapshot. Hence the callback knows the situation by showing
138 * 'started' status, and sets the status as 'finished'. Then,
139 * damon_sysfs_before_damos_apply() understands the situation by showing the
140 * 'finished' status and do nothing.
141 *
142 * If DAMOS is not applied to any region due to any reasons including the
143 * access pattern, the watermarks, the quotas, and the filters,
144 * ->before_damos_apply() will not be called back. Until the situation is
145 * changed, the update will not be finished. To avoid this,
146 * damon_sysfs_after_sampling() set the status as 'finished' if more than two
147 * apply intervals of the scheme is passed while the state is 'idle'.
148 *
149 * Finally, the tried regions request handling finisher function
150 * (damon_sysfs_schemes_update_regions_stop()) unregisters the callbacks.
151 */
152enum damos_sysfs_regions_upd_status {
153 DAMOS_TRIED_REGIONS_UPD_IDLE,
154 DAMOS_TRIED_REGIONS_UPD_STARTED,
155 DAMOS_TRIED_REGIONS_UPD_FINISHED,
156};
157
158struct damon_sysfs_scheme_regions {
159 struct kobject kobj;
160 struct list_head regions_list;
161 int nr_regions;
162 unsigned long total_bytes;
163 enum damos_sysfs_regions_upd_status upd_status;
164 unsigned long upd_timeout_jiffies;
165};
166
167static struct damon_sysfs_scheme_regions *
168damon_sysfs_scheme_regions_alloc(void)
169{
170 struct damon_sysfs_scheme_regions *regions = kmalloc(sizeof(*regions),
171 GFP_KERNEL);
172
173 if (!regions)
174 return NULL;
175
176 regions->kobj = (struct kobject){};
177 INIT_LIST_HEAD(®ions->regions_list);
178 regions->nr_regions = 0;
179 regions->total_bytes = 0;
180 regions->upd_status = DAMOS_TRIED_REGIONS_UPD_IDLE;
181 return regions;
182}
183
184static ssize_t total_bytes_show(struct kobject *kobj,
185 struct kobj_attribute *attr, char *buf)
186{
187 struct damon_sysfs_scheme_regions *regions = container_of(kobj,
188 struct damon_sysfs_scheme_regions, kobj);
189
190 return sysfs_emit(buf, "%lu\n", regions->total_bytes);
191}
192
193static void damon_sysfs_scheme_regions_rm_dirs(
194 struct damon_sysfs_scheme_regions *regions)
195{
196 struct damon_sysfs_scheme_region *r, *next;
197
198 list_for_each_entry_safe(r, next, ®ions->regions_list, list) {
199 /* release function deletes it from the list */
200 kobject_put(&r->kobj);
201 regions->nr_regions--;
202 }
203}
204
205static void damon_sysfs_scheme_regions_release(struct kobject *kobj)
206{
207 kfree(container_of(kobj, struct damon_sysfs_scheme_regions, kobj));
208}
209
210static struct kobj_attribute damon_sysfs_scheme_regions_total_bytes_attr =
211 __ATTR_RO_MODE(total_bytes, 0400);
212
213static struct attribute *damon_sysfs_scheme_regions_attrs[] = {
214 &damon_sysfs_scheme_regions_total_bytes_attr.attr,
215 NULL,
216};
217ATTRIBUTE_GROUPS(damon_sysfs_scheme_regions);
218
219static const struct kobj_type damon_sysfs_scheme_regions_ktype = {
220 .release = damon_sysfs_scheme_regions_release,
221 .sysfs_ops = &kobj_sysfs_ops,
222 .default_groups = damon_sysfs_scheme_regions_groups,
223};
224
225/*
226 * schemes/stats directory
227 */
228
229struct damon_sysfs_stats {
230 struct kobject kobj;
231 unsigned long nr_tried;
232 unsigned long sz_tried;
233 unsigned long nr_applied;
234 unsigned long sz_applied;
235 unsigned long qt_exceeds;
236};
237
238static struct damon_sysfs_stats *damon_sysfs_stats_alloc(void)
239{
240 return kzalloc(sizeof(struct damon_sysfs_stats), GFP_KERNEL);
241}
242
243static ssize_t nr_tried_show(struct kobject *kobj, struct kobj_attribute *attr,
244 char *buf)
245{
246 struct damon_sysfs_stats *stats = container_of(kobj,
247 struct damon_sysfs_stats, kobj);
248
249 return sysfs_emit(buf, "%lu\n", stats->nr_tried);
250}
251
252static ssize_t sz_tried_show(struct kobject *kobj, struct kobj_attribute *attr,
253 char *buf)
254{
255 struct damon_sysfs_stats *stats = container_of(kobj,
256 struct damon_sysfs_stats, kobj);
257
258 return sysfs_emit(buf, "%lu\n", stats->sz_tried);
259}
260
261static ssize_t nr_applied_show(struct kobject *kobj,
262 struct kobj_attribute *attr, char *buf)
263{
264 struct damon_sysfs_stats *stats = container_of(kobj,
265 struct damon_sysfs_stats, kobj);
266
267 return sysfs_emit(buf, "%lu\n", stats->nr_applied);
268}
269
270static ssize_t sz_applied_show(struct kobject *kobj,
271 struct kobj_attribute *attr, char *buf)
272{
273 struct damon_sysfs_stats *stats = container_of(kobj,
274 struct damon_sysfs_stats, kobj);
275
276 return sysfs_emit(buf, "%lu\n", stats->sz_applied);
277}
278
279static ssize_t qt_exceeds_show(struct kobject *kobj,
280 struct kobj_attribute *attr, char *buf)
281{
282 struct damon_sysfs_stats *stats = container_of(kobj,
283 struct damon_sysfs_stats, kobj);
284
285 return sysfs_emit(buf, "%lu\n", stats->qt_exceeds);
286}
287
288static void damon_sysfs_stats_release(struct kobject *kobj)
289{
290 kfree(container_of(kobj, struct damon_sysfs_stats, kobj));
291}
292
293static struct kobj_attribute damon_sysfs_stats_nr_tried_attr =
294 __ATTR_RO_MODE(nr_tried, 0400);
295
296static struct kobj_attribute damon_sysfs_stats_sz_tried_attr =
297 __ATTR_RO_MODE(sz_tried, 0400);
298
299static struct kobj_attribute damon_sysfs_stats_nr_applied_attr =
300 __ATTR_RO_MODE(nr_applied, 0400);
301
302static struct kobj_attribute damon_sysfs_stats_sz_applied_attr =
303 __ATTR_RO_MODE(sz_applied, 0400);
304
305static struct kobj_attribute damon_sysfs_stats_qt_exceeds_attr =
306 __ATTR_RO_MODE(qt_exceeds, 0400);
307
308static struct attribute *damon_sysfs_stats_attrs[] = {
309 &damon_sysfs_stats_nr_tried_attr.attr,
310 &damon_sysfs_stats_sz_tried_attr.attr,
311 &damon_sysfs_stats_nr_applied_attr.attr,
312 &damon_sysfs_stats_sz_applied_attr.attr,
313 &damon_sysfs_stats_qt_exceeds_attr.attr,
314 NULL,
315};
316ATTRIBUTE_GROUPS(damon_sysfs_stats);
317
318static const struct kobj_type damon_sysfs_stats_ktype = {
319 .release = damon_sysfs_stats_release,
320 .sysfs_ops = &kobj_sysfs_ops,
321 .default_groups = damon_sysfs_stats_groups,
322};
323
324/*
325 * filter directory
326 */
327
328struct damon_sysfs_scheme_filter {
329 struct kobject kobj;
330 enum damos_filter_type type;
331 bool matching;
332 char *memcg_path;
333 struct damon_addr_range addr_range;
334 int target_idx;
335};
336
337static struct damon_sysfs_scheme_filter *damon_sysfs_scheme_filter_alloc(void)
338{
339 return kzalloc(sizeof(struct damon_sysfs_scheme_filter), GFP_KERNEL);
340}
341
342/* Should match with enum damos_filter_type */
343static const char * const damon_sysfs_scheme_filter_type_strs[] = {
344 "anon",
345 "memcg",
346 "addr",
347 "target",
348};
349
350static ssize_t type_show(struct kobject *kobj,
351 struct kobj_attribute *attr, char *buf)
352{
353 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
354 struct damon_sysfs_scheme_filter, kobj);
355
356 return sysfs_emit(buf, "%s\n",
357 damon_sysfs_scheme_filter_type_strs[filter->type]);
358}
359
360static ssize_t type_store(struct kobject *kobj,
361 struct kobj_attribute *attr, const char *buf, size_t count)
362{
363 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
364 struct damon_sysfs_scheme_filter, kobj);
365 enum damos_filter_type type;
366 ssize_t ret = -EINVAL;
367
368 for (type = 0; type < NR_DAMOS_FILTER_TYPES; type++) {
369 if (sysfs_streq(buf, damon_sysfs_scheme_filter_type_strs[
370 type])) {
371 filter->type = type;
372 ret = count;
373 break;
374 }
375 }
376 return ret;
377}
378
379static ssize_t matching_show(struct kobject *kobj,
380 struct kobj_attribute *attr, char *buf)
381{
382 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
383 struct damon_sysfs_scheme_filter, kobj);
384
385 return sysfs_emit(buf, "%c\n", filter->matching ? 'Y' : 'N');
386}
387
388static ssize_t matching_store(struct kobject *kobj,
389 struct kobj_attribute *attr, const char *buf, size_t count)
390{
391 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
392 struct damon_sysfs_scheme_filter, kobj);
393 bool matching;
394 int err = kstrtobool(buf, &matching);
395
396 if (err)
397 return err;
398
399 filter->matching = matching;
400 return count;
401}
402
403static ssize_t memcg_path_show(struct kobject *kobj,
404 struct kobj_attribute *attr, char *buf)
405{
406 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
407 struct damon_sysfs_scheme_filter, kobj);
408
409 return sysfs_emit(buf, "%s\n",
410 filter->memcg_path ? filter->memcg_path : "");
411}
412
413static ssize_t memcg_path_store(struct kobject *kobj,
414 struct kobj_attribute *attr, const char *buf, size_t count)
415{
416 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
417 struct damon_sysfs_scheme_filter, kobj);
418 char *path = kmalloc(sizeof(*path) * (count + 1), GFP_KERNEL);
419
420 if (!path)
421 return -ENOMEM;
422
423 strscpy(path, buf, count + 1);
424 filter->memcg_path = path;
425 return count;
426}
427
428static ssize_t addr_start_show(struct kobject *kobj,
429 struct kobj_attribute *attr, char *buf)
430{
431 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
432 struct damon_sysfs_scheme_filter, kobj);
433
434 return sysfs_emit(buf, "%lu\n", filter->addr_range.start);
435}
436
437static ssize_t addr_start_store(struct kobject *kobj,
438 struct kobj_attribute *attr, const char *buf, size_t count)
439{
440 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
441 struct damon_sysfs_scheme_filter, kobj);
442 int err = kstrtoul(buf, 0, &filter->addr_range.start);
443
444 return err ? err : count;
445}
446
447static ssize_t addr_end_show(struct kobject *kobj,
448 struct kobj_attribute *attr, char *buf)
449{
450 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
451 struct damon_sysfs_scheme_filter, kobj);
452
453 return sysfs_emit(buf, "%lu\n", filter->addr_range.end);
454}
455
456static ssize_t addr_end_store(struct kobject *kobj,
457 struct kobj_attribute *attr, const char *buf, size_t count)
458{
459 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
460 struct damon_sysfs_scheme_filter, kobj);
461 int err = kstrtoul(buf, 0, &filter->addr_range.end);
462
463 return err ? err : count;
464}
465
466static ssize_t damon_target_idx_show(struct kobject *kobj,
467 struct kobj_attribute *attr, char *buf)
468{
469 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
470 struct damon_sysfs_scheme_filter, kobj);
471
472 return sysfs_emit(buf, "%d\n", filter->target_idx);
473}
474
475static ssize_t damon_target_idx_store(struct kobject *kobj,
476 struct kobj_attribute *attr, const char *buf, size_t count)
477{
478 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
479 struct damon_sysfs_scheme_filter, kobj);
480 int err = kstrtoint(buf, 0, &filter->target_idx);
481
482 return err ? err : count;
483}
484
485static void damon_sysfs_scheme_filter_release(struct kobject *kobj)
486{
487 struct damon_sysfs_scheme_filter *filter = container_of(kobj,
488 struct damon_sysfs_scheme_filter, kobj);
489
490 kfree(filter->memcg_path);
491 kfree(filter);
492}
493
494static struct kobj_attribute damon_sysfs_scheme_filter_type_attr =
495 __ATTR_RW_MODE(type, 0600);
496
497static struct kobj_attribute damon_sysfs_scheme_filter_matching_attr =
498 __ATTR_RW_MODE(matching, 0600);
499
500static struct kobj_attribute damon_sysfs_scheme_filter_memcg_path_attr =
501 __ATTR_RW_MODE(memcg_path, 0600);
502
503static struct kobj_attribute damon_sysfs_scheme_filter_addr_start_attr =
504 __ATTR_RW_MODE(addr_start, 0600);
505
506static struct kobj_attribute damon_sysfs_scheme_filter_addr_end_attr =
507 __ATTR_RW_MODE(addr_end, 0600);
508
509static struct kobj_attribute damon_sysfs_scheme_filter_damon_target_idx_attr =
510 __ATTR_RW_MODE(damon_target_idx, 0600);
511
512static struct attribute *damon_sysfs_scheme_filter_attrs[] = {
513 &damon_sysfs_scheme_filter_type_attr.attr,
514 &damon_sysfs_scheme_filter_matching_attr.attr,
515 &damon_sysfs_scheme_filter_memcg_path_attr.attr,
516 &damon_sysfs_scheme_filter_addr_start_attr.attr,
517 &damon_sysfs_scheme_filter_addr_end_attr.attr,
518 &damon_sysfs_scheme_filter_damon_target_idx_attr.attr,
519 NULL,
520};
521ATTRIBUTE_GROUPS(damon_sysfs_scheme_filter);
522
523static const struct kobj_type damon_sysfs_scheme_filter_ktype = {
524 .release = damon_sysfs_scheme_filter_release,
525 .sysfs_ops = &kobj_sysfs_ops,
526 .default_groups = damon_sysfs_scheme_filter_groups,
527};
528
529/*
530 * filters directory
531 */
532
533struct damon_sysfs_scheme_filters {
534 struct kobject kobj;
535 struct damon_sysfs_scheme_filter **filters_arr;
536 int nr;
537};
538
539static struct damon_sysfs_scheme_filters *
540damon_sysfs_scheme_filters_alloc(void)
541{
542 return kzalloc(sizeof(struct damon_sysfs_scheme_filters), GFP_KERNEL);
543}
544
545static void damon_sysfs_scheme_filters_rm_dirs(
546 struct damon_sysfs_scheme_filters *filters)
547{
548 struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr;
549 int i;
550
551 for (i = 0; i < filters->nr; i++)
552 kobject_put(&filters_arr[i]->kobj);
553 filters->nr = 0;
554 kfree(filters_arr);
555 filters->filters_arr = NULL;
556}
557
558static int damon_sysfs_scheme_filters_add_dirs(
559 struct damon_sysfs_scheme_filters *filters, int nr_filters)
560{
561 struct damon_sysfs_scheme_filter **filters_arr, *filter;
562 int err, i;
563
564 damon_sysfs_scheme_filters_rm_dirs(filters);
565 if (!nr_filters)
566 return 0;
567
568 filters_arr = kmalloc_array(nr_filters, sizeof(*filters_arr),
569 GFP_KERNEL | __GFP_NOWARN);
570 if (!filters_arr)
571 return -ENOMEM;
572 filters->filters_arr = filters_arr;
573
574 for (i = 0; i < nr_filters; i++) {
575 filter = damon_sysfs_scheme_filter_alloc();
576 if (!filter) {
577 damon_sysfs_scheme_filters_rm_dirs(filters);
578 return -ENOMEM;
579 }
580
581 err = kobject_init_and_add(&filter->kobj,
582 &damon_sysfs_scheme_filter_ktype,
583 &filters->kobj, "%d", i);
584 if (err) {
585 kobject_put(&filter->kobj);
586 damon_sysfs_scheme_filters_rm_dirs(filters);
587 return err;
588 }
589
590 filters_arr[i] = filter;
591 filters->nr++;
592 }
593 return 0;
594}
595
596static ssize_t nr_filters_show(struct kobject *kobj,
597 struct kobj_attribute *attr, char *buf)
598{
599 struct damon_sysfs_scheme_filters *filters = container_of(kobj,
600 struct damon_sysfs_scheme_filters, kobj);
601
602 return sysfs_emit(buf, "%d\n", filters->nr);
603}
604
605static ssize_t nr_filters_store(struct kobject *kobj,
606 struct kobj_attribute *attr, const char *buf, size_t count)
607{
608 struct damon_sysfs_scheme_filters *filters;
609 int nr, err = kstrtoint(buf, 0, &nr);
610
611 if (err)
612 return err;
613 if (nr < 0)
614 return -EINVAL;
615
616 filters = container_of(kobj, struct damon_sysfs_scheme_filters, kobj);
617
618 if (!mutex_trylock(&damon_sysfs_lock))
619 return -EBUSY;
620 err = damon_sysfs_scheme_filters_add_dirs(filters, nr);
621 mutex_unlock(&damon_sysfs_lock);
622 if (err)
623 return err;
624
625 return count;
626}
627
628static void damon_sysfs_scheme_filters_release(struct kobject *kobj)
629{
630 kfree(container_of(kobj, struct damon_sysfs_scheme_filters, kobj));
631}
632
633static struct kobj_attribute damon_sysfs_scheme_filters_nr_attr =
634 __ATTR_RW_MODE(nr_filters, 0600);
635
636static struct attribute *damon_sysfs_scheme_filters_attrs[] = {
637 &damon_sysfs_scheme_filters_nr_attr.attr,
638 NULL,
639};
640ATTRIBUTE_GROUPS(damon_sysfs_scheme_filters);
641
642static const struct kobj_type damon_sysfs_scheme_filters_ktype = {
643 .release = damon_sysfs_scheme_filters_release,
644 .sysfs_ops = &kobj_sysfs_ops,
645 .default_groups = damon_sysfs_scheme_filters_groups,
646};
647
648/*
649 * watermarks directory
650 */
651
652struct damon_sysfs_watermarks {
653 struct kobject kobj;
654 enum damos_wmark_metric metric;
655 unsigned long interval_us;
656 unsigned long high;
657 unsigned long mid;
658 unsigned long low;
659};
660
661static struct damon_sysfs_watermarks *damon_sysfs_watermarks_alloc(
662 enum damos_wmark_metric metric, unsigned long interval_us,
663 unsigned long high, unsigned long mid, unsigned long low)
664{
665 struct damon_sysfs_watermarks *watermarks = kmalloc(
666 sizeof(*watermarks), GFP_KERNEL);
667
668 if (!watermarks)
669 return NULL;
670 watermarks->kobj = (struct kobject){};
671 watermarks->metric = metric;
672 watermarks->interval_us = interval_us;
673 watermarks->high = high;
674 watermarks->mid = mid;
675 watermarks->low = low;
676 return watermarks;
677}
678
679/* Should match with enum damos_wmark_metric */
680static const char * const damon_sysfs_wmark_metric_strs[] = {
681 "none",
682 "free_mem_rate",
683};
684
685static ssize_t metric_show(struct kobject *kobj, struct kobj_attribute *attr,
686 char *buf)
687{
688 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
689 struct damon_sysfs_watermarks, kobj);
690
691 return sysfs_emit(buf, "%s\n",
692 damon_sysfs_wmark_metric_strs[watermarks->metric]);
693}
694
695static ssize_t metric_store(struct kobject *kobj, struct kobj_attribute *attr,
696 const char *buf, size_t count)
697{
698 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
699 struct damon_sysfs_watermarks, kobj);
700 enum damos_wmark_metric metric;
701
702 for (metric = 0; metric < NR_DAMOS_WMARK_METRICS; metric++) {
703 if (sysfs_streq(buf, damon_sysfs_wmark_metric_strs[metric])) {
704 watermarks->metric = metric;
705 return count;
706 }
707 }
708 return -EINVAL;
709}
710
711static ssize_t interval_us_show(struct kobject *kobj,
712 struct kobj_attribute *attr, char *buf)
713{
714 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
715 struct damon_sysfs_watermarks, kobj);
716
717 return sysfs_emit(buf, "%lu\n", watermarks->interval_us);
718}
719
720static ssize_t interval_us_store(struct kobject *kobj,
721 struct kobj_attribute *attr, const char *buf, size_t count)
722{
723 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
724 struct damon_sysfs_watermarks, kobj);
725 int err = kstrtoul(buf, 0, &watermarks->interval_us);
726
727 return err ? err : count;
728}
729
730static ssize_t high_show(struct kobject *kobj,
731 struct kobj_attribute *attr, char *buf)
732{
733 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
734 struct damon_sysfs_watermarks, kobj);
735
736 return sysfs_emit(buf, "%lu\n", watermarks->high);
737}
738
739static ssize_t high_store(struct kobject *kobj,
740 struct kobj_attribute *attr, const char *buf, size_t count)
741{
742 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
743 struct damon_sysfs_watermarks, kobj);
744 int err = kstrtoul(buf, 0, &watermarks->high);
745
746 return err ? err : count;
747}
748
749static ssize_t mid_show(struct kobject *kobj,
750 struct kobj_attribute *attr, char *buf)
751{
752 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
753 struct damon_sysfs_watermarks, kobj);
754
755 return sysfs_emit(buf, "%lu\n", watermarks->mid);
756}
757
758static ssize_t mid_store(struct kobject *kobj,
759 struct kobj_attribute *attr, const char *buf, size_t count)
760{
761 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
762 struct damon_sysfs_watermarks, kobj);
763 int err = kstrtoul(buf, 0, &watermarks->mid);
764
765 return err ? err : count;
766}
767
768static ssize_t low_show(struct kobject *kobj,
769 struct kobj_attribute *attr, char *buf)
770{
771 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
772 struct damon_sysfs_watermarks, kobj);
773
774 return sysfs_emit(buf, "%lu\n", watermarks->low);
775}
776
777static ssize_t low_store(struct kobject *kobj,
778 struct kobj_attribute *attr, const char *buf, size_t count)
779{
780 struct damon_sysfs_watermarks *watermarks = container_of(kobj,
781 struct damon_sysfs_watermarks, kobj);
782 int err = kstrtoul(buf, 0, &watermarks->low);
783
784 return err ? err : count;
785}
786
787static void damon_sysfs_watermarks_release(struct kobject *kobj)
788{
789 kfree(container_of(kobj, struct damon_sysfs_watermarks, kobj));
790}
791
792static struct kobj_attribute damon_sysfs_watermarks_metric_attr =
793 __ATTR_RW_MODE(metric, 0600);
794
795static struct kobj_attribute damon_sysfs_watermarks_interval_us_attr =
796 __ATTR_RW_MODE(interval_us, 0600);
797
798static struct kobj_attribute damon_sysfs_watermarks_high_attr =
799 __ATTR_RW_MODE(high, 0600);
800
801static struct kobj_attribute damon_sysfs_watermarks_mid_attr =
802 __ATTR_RW_MODE(mid, 0600);
803
804static struct kobj_attribute damon_sysfs_watermarks_low_attr =
805 __ATTR_RW_MODE(low, 0600);
806
807static struct attribute *damon_sysfs_watermarks_attrs[] = {
808 &damon_sysfs_watermarks_metric_attr.attr,
809 &damon_sysfs_watermarks_interval_us_attr.attr,
810 &damon_sysfs_watermarks_high_attr.attr,
811 &damon_sysfs_watermarks_mid_attr.attr,
812 &damon_sysfs_watermarks_low_attr.attr,
813 NULL,
814};
815ATTRIBUTE_GROUPS(damon_sysfs_watermarks);
816
817static const struct kobj_type damon_sysfs_watermarks_ktype = {
818 .release = damon_sysfs_watermarks_release,
819 .sysfs_ops = &kobj_sysfs_ops,
820 .default_groups = damon_sysfs_watermarks_groups,
821};
822
823/*
824 * quota goal directory
825 */
826
827struct damos_sysfs_quota_goal {
828 struct kobject kobj;
829 unsigned long target_value;
830 unsigned long current_value;
831};
832
833static struct damos_sysfs_quota_goal *damos_sysfs_quota_goal_alloc(void)
834{
835 return kzalloc(sizeof(struct damos_sysfs_quota_goal), GFP_KERNEL);
836}
837
838static ssize_t target_value_show(struct kobject *kobj,
839 struct kobj_attribute *attr, char *buf)
840{
841 struct damos_sysfs_quota_goal *goal = container_of(kobj, struct
842 damos_sysfs_quota_goal, kobj);
843
844 return sysfs_emit(buf, "%lu\n", goal->target_value);
845}
846
847static ssize_t target_value_store(struct kobject *kobj,
848 struct kobj_attribute *attr, const char *buf, size_t count)
849{
850 struct damos_sysfs_quota_goal *goal = container_of(kobj, struct
851 damos_sysfs_quota_goal, kobj);
852 int err = kstrtoul(buf, 0, &goal->target_value);
853
854 return err ? err : count;
855}
856
857static ssize_t current_value_show(struct kobject *kobj,
858 struct kobj_attribute *attr, char *buf)
859{
860 struct damos_sysfs_quota_goal *goal = container_of(kobj, struct
861 damos_sysfs_quota_goal, kobj);
862
863 return sysfs_emit(buf, "%lu\n", goal->current_value);
864}
865
866static ssize_t current_value_store(struct kobject *kobj,
867 struct kobj_attribute *attr, const char *buf, size_t count)
868{
869 struct damos_sysfs_quota_goal *goal = container_of(kobj, struct
870 damos_sysfs_quota_goal, kobj);
871 int err = kstrtoul(buf, 0, &goal->current_value);
872
873 /* feed callback should check existence of this file and read value */
874 return err ? err : count;
875}
876
877static void damos_sysfs_quota_goal_release(struct kobject *kobj)
878{
879 /* or, notify this release to the feed callback */
880 kfree(container_of(kobj, struct damos_sysfs_quota_goal, kobj));
881}
882
883static struct kobj_attribute damos_sysfs_quota_goal_target_value_attr =
884 __ATTR_RW_MODE(target_value, 0600);
885
886static struct kobj_attribute damos_sysfs_quota_goal_current_value_attr =
887 __ATTR_RW_MODE(current_value, 0600);
888
889static struct attribute *damos_sysfs_quota_goal_attrs[] = {
890 &damos_sysfs_quota_goal_target_value_attr.attr,
891 &damos_sysfs_quota_goal_current_value_attr.attr,
892 NULL,
893};
894ATTRIBUTE_GROUPS(damos_sysfs_quota_goal);
895
896static const struct kobj_type damos_sysfs_quota_goal_ktype = {
897 .release = damos_sysfs_quota_goal_release,
898 .sysfs_ops = &kobj_sysfs_ops,
899 .default_groups = damos_sysfs_quota_goal_groups,
900};
901
902/*
903 * quota goals directory
904 */
905
906struct damos_sysfs_quota_goals {
907 struct kobject kobj;
908 struct damos_sysfs_quota_goal **goals_arr; /* counted by nr */
909 int nr;
910};
911
912static struct damos_sysfs_quota_goals *damos_sysfs_quota_goals_alloc(void)
913{
914 return kzalloc(sizeof(struct damos_sysfs_quota_goals), GFP_KERNEL);
915}
916
917static void damos_sysfs_quota_goals_rm_dirs(
918 struct damos_sysfs_quota_goals *goals)
919{
920 struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr;
921 int i;
922
923 for (i = 0; i < goals->nr; i++)
924 kobject_put(&goals_arr[i]->kobj);
925 goals->nr = 0;
926 kfree(goals_arr);
927 goals->goals_arr = NULL;
928}
929
930static int damos_sysfs_quota_goals_add_dirs(
931 struct damos_sysfs_quota_goals *goals, int nr_goals)
932{
933 struct damos_sysfs_quota_goal **goals_arr, *goal;
934 int err, i;
935
936 damos_sysfs_quota_goals_rm_dirs(goals);
937 if (!nr_goals)
938 return 0;
939
940 goals_arr = kmalloc_array(nr_goals, sizeof(*goals_arr),
941 GFP_KERNEL | __GFP_NOWARN);
942 if (!goals_arr)
943 return -ENOMEM;
944 goals->goals_arr = goals_arr;
945
946 for (i = 0; i < nr_goals; i++) {
947 goal = damos_sysfs_quota_goal_alloc();
948 if (!goal) {
949 damos_sysfs_quota_goals_rm_dirs(goals);
950 return -ENOMEM;
951 }
952
953 err = kobject_init_and_add(&goal->kobj,
954 &damos_sysfs_quota_goal_ktype, &goals->kobj,
955 "%d", i);
956 if (err) {
957 kobject_put(&goal->kobj);
958 damos_sysfs_quota_goals_rm_dirs(goals);
959 return err;
960 }
961
962 goals_arr[i] = goal;
963 goals->nr++;
964 }
965 return 0;
966}
967
968static ssize_t nr_goals_show(struct kobject *kobj,
969 struct kobj_attribute *attr, char *buf)
970{
971 struct damos_sysfs_quota_goals *goals = container_of(kobj,
972 struct damos_sysfs_quota_goals, kobj);
973
974 return sysfs_emit(buf, "%d\n", goals->nr);
975}
976
977static ssize_t nr_goals_store(struct kobject *kobj,
978 struct kobj_attribute *attr, const char *buf, size_t count)
979{
980 struct damos_sysfs_quota_goals *goals;
981 int nr, err = kstrtoint(buf, 0, &nr);
982
983 if (err)
984 return err;
985 if (nr < 0)
986 return -EINVAL;
987
988 goals = container_of(kobj, struct damos_sysfs_quota_goals, kobj);
989
990 if (!mutex_trylock(&damon_sysfs_lock))
991 return -EBUSY;
992 err = damos_sysfs_quota_goals_add_dirs(goals, nr);
993 mutex_unlock(&damon_sysfs_lock);
994 if (err)
995 return err;
996
997 return count;
998}
999
1000static void damos_sysfs_quota_goals_release(struct kobject *kobj)
1001{
1002 kfree(container_of(kobj, struct damos_sysfs_quota_goals, kobj));
1003}
1004
1005static struct kobj_attribute damos_sysfs_quota_goals_nr_attr =
1006 __ATTR_RW_MODE(nr_goals, 0600);
1007
1008static struct attribute *damos_sysfs_quota_goals_attrs[] = {
1009 &damos_sysfs_quota_goals_nr_attr.attr,
1010 NULL,
1011};
1012ATTRIBUTE_GROUPS(damos_sysfs_quota_goals);
1013
1014static const struct kobj_type damos_sysfs_quota_goals_ktype = {
1015 .release = damos_sysfs_quota_goals_release,
1016 .sysfs_ops = &kobj_sysfs_ops,
1017 .default_groups = damos_sysfs_quota_goals_groups,
1018};
1019
1020/*
1021 * scheme/weights directory
1022 */
1023
1024struct damon_sysfs_weights {
1025 struct kobject kobj;
1026 unsigned int sz;
1027 unsigned int nr_accesses;
1028 unsigned int age;
1029};
1030
1031static struct damon_sysfs_weights *damon_sysfs_weights_alloc(unsigned int sz,
1032 unsigned int nr_accesses, unsigned int age)
1033{
1034 struct damon_sysfs_weights *weights = kmalloc(sizeof(*weights),
1035 GFP_KERNEL);
1036
1037 if (!weights)
1038 return NULL;
1039 weights->kobj = (struct kobject){};
1040 weights->sz = sz;
1041 weights->nr_accesses = nr_accesses;
1042 weights->age = age;
1043 return weights;
1044}
1045
1046static ssize_t sz_permil_show(struct kobject *kobj,
1047 struct kobj_attribute *attr, char *buf)
1048{
1049 struct damon_sysfs_weights *weights = container_of(kobj,
1050 struct damon_sysfs_weights, kobj);
1051
1052 return sysfs_emit(buf, "%u\n", weights->sz);
1053}
1054
1055static ssize_t sz_permil_store(struct kobject *kobj,
1056 struct kobj_attribute *attr, const char *buf, size_t count)
1057{
1058 struct damon_sysfs_weights *weights = container_of(kobj,
1059 struct damon_sysfs_weights, kobj);
1060 int err = kstrtouint(buf, 0, &weights->sz);
1061
1062 return err ? err : count;
1063}
1064
1065static ssize_t nr_accesses_permil_show(struct kobject *kobj,
1066 struct kobj_attribute *attr, char *buf)
1067{
1068 struct damon_sysfs_weights *weights = container_of(kobj,
1069 struct damon_sysfs_weights, kobj);
1070
1071 return sysfs_emit(buf, "%u\n", weights->nr_accesses);
1072}
1073
1074static ssize_t nr_accesses_permil_store(struct kobject *kobj,
1075 struct kobj_attribute *attr, const char *buf, size_t count)
1076{
1077 struct damon_sysfs_weights *weights = container_of(kobj,
1078 struct damon_sysfs_weights, kobj);
1079 int err = kstrtouint(buf, 0, &weights->nr_accesses);
1080
1081 return err ? err : count;
1082}
1083
1084static ssize_t age_permil_show(struct kobject *kobj,
1085 struct kobj_attribute *attr, char *buf)
1086{
1087 struct damon_sysfs_weights *weights = container_of(kobj,
1088 struct damon_sysfs_weights, kobj);
1089
1090 return sysfs_emit(buf, "%u\n", weights->age);
1091}
1092
1093static ssize_t age_permil_store(struct kobject *kobj,
1094 struct kobj_attribute *attr, const char *buf, size_t count)
1095{
1096 struct damon_sysfs_weights *weights = container_of(kobj,
1097 struct damon_sysfs_weights, kobj);
1098 int err = kstrtouint(buf, 0, &weights->age);
1099
1100 return err ? err : count;
1101}
1102
1103static void damon_sysfs_weights_release(struct kobject *kobj)
1104{
1105 kfree(container_of(kobj, struct damon_sysfs_weights, kobj));
1106}
1107
1108static struct kobj_attribute damon_sysfs_weights_sz_attr =
1109 __ATTR_RW_MODE(sz_permil, 0600);
1110
1111static struct kobj_attribute damon_sysfs_weights_nr_accesses_attr =
1112 __ATTR_RW_MODE(nr_accesses_permil, 0600);
1113
1114static struct kobj_attribute damon_sysfs_weights_age_attr =
1115 __ATTR_RW_MODE(age_permil, 0600);
1116
1117static struct attribute *damon_sysfs_weights_attrs[] = {
1118 &damon_sysfs_weights_sz_attr.attr,
1119 &damon_sysfs_weights_nr_accesses_attr.attr,
1120 &damon_sysfs_weights_age_attr.attr,
1121 NULL,
1122};
1123ATTRIBUTE_GROUPS(damon_sysfs_weights);
1124
1125static const struct kobj_type damon_sysfs_weights_ktype = {
1126 .release = damon_sysfs_weights_release,
1127 .sysfs_ops = &kobj_sysfs_ops,
1128 .default_groups = damon_sysfs_weights_groups,
1129};
1130
1131/*
1132 * quotas directory
1133 */
1134
1135struct damon_sysfs_quotas {
1136 struct kobject kobj;
1137 struct damon_sysfs_weights *weights;
1138 struct damos_sysfs_quota_goals *goals;
1139 unsigned long ms;
1140 unsigned long sz;
1141 unsigned long reset_interval_ms;
1142};
1143
1144static struct damon_sysfs_quotas *damon_sysfs_quotas_alloc(void)
1145{
1146 return kzalloc(sizeof(struct damon_sysfs_quotas), GFP_KERNEL);
1147}
1148
1149static int damon_sysfs_quotas_add_dirs(struct damon_sysfs_quotas *quotas)
1150{
1151 struct damon_sysfs_weights *weights;
1152 struct damos_sysfs_quota_goals *goals;
1153 int err;
1154
1155 weights = damon_sysfs_weights_alloc(0, 0, 0);
1156 if (!weights)
1157 return -ENOMEM;
1158
1159 err = kobject_init_and_add(&weights->kobj, &damon_sysfs_weights_ktype,
1160 "as->kobj, "weights");
1161 if (err) {
1162 kobject_put(&weights->kobj);
1163 return err;
1164 }
1165 quotas->weights = weights;
1166
1167 goals = damos_sysfs_quota_goals_alloc();
1168 if (!goals) {
1169 kobject_put(&weights->kobj);
1170 return -ENOMEM;
1171 }
1172 err = kobject_init_and_add(&goals->kobj,
1173 &damos_sysfs_quota_goals_ktype, "as->kobj,
1174 "goals");
1175 if (err) {
1176 kobject_put(&weights->kobj);
1177 kobject_put(&goals->kobj);
1178 } else {
1179 quotas->goals = goals;
1180 }
1181
1182 return err;
1183}
1184
1185static void damon_sysfs_quotas_rm_dirs(struct damon_sysfs_quotas *quotas)
1186{
1187 kobject_put("as->weights->kobj);
1188 damos_sysfs_quota_goals_rm_dirs(quotas->goals);
1189 kobject_put("as->goals->kobj);
1190}
1191
1192static ssize_t ms_show(struct kobject *kobj, struct kobj_attribute *attr,
1193 char *buf)
1194{
1195 struct damon_sysfs_quotas *quotas = container_of(kobj,
1196 struct damon_sysfs_quotas, kobj);
1197
1198 return sysfs_emit(buf, "%lu\n", quotas->ms);
1199}
1200
1201static ssize_t ms_store(struct kobject *kobj, struct kobj_attribute *attr,
1202 const char *buf, size_t count)
1203{
1204 struct damon_sysfs_quotas *quotas = container_of(kobj,
1205 struct damon_sysfs_quotas, kobj);
1206 int err = kstrtoul(buf, 0, "as->ms);
1207
1208 if (err)
1209 return -EINVAL;
1210 return count;
1211}
1212
1213static ssize_t bytes_show(struct kobject *kobj, struct kobj_attribute *attr,
1214 char *buf)
1215{
1216 struct damon_sysfs_quotas *quotas = container_of(kobj,
1217 struct damon_sysfs_quotas, kobj);
1218
1219 return sysfs_emit(buf, "%lu\n", quotas->sz);
1220}
1221
1222static ssize_t bytes_store(struct kobject *kobj,
1223 struct kobj_attribute *attr, const char *buf, size_t count)
1224{
1225 struct damon_sysfs_quotas *quotas = container_of(kobj,
1226 struct damon_sysfs_quotas, kobj);
1227 int err = kstrtoul(buf, 0, "as->sz);
1228
1229 if (err)
1230 return -EINVAL;
1231 return count;
1232}
1233
1234static ssize_t reset_interval_ms_show(struct kobject *kobj,
1235 struct kobj_attribute *attr, char *buf)
1236{
1237 struct damon_sysfs_quotas *quotas = container_of(kobj,
1238 struct damon_sysfs_quotas, kobj);
1239
1240 return sysfs_emit(buf, "%lu\n", quotas->reset_interval_ms);
1241}
1242
1243static ssize_t reset_interval_ms_store(struct kobject *kobj,
1244 struct kobj_attribute *attr, const char *buf, size_t count)
1245{
1246 struct damon_sysfs_quotas *quotas = container_of(kobj,
1247 struct damon_sysfs_quotas, kobj);
1248 int err = kstrtoul(buf, 0, "as->reset_interval_ms);
1249
1250 if (err)
1251 return -EINVAL;
1252 return count;
1253}
1254
1255static void damon_sysfs_quotas_release(struct kobject *kobj)
1256{
1257 kfree(container_of(kobj, struct damon_sysfs_quotas, kobj));
1258}
1259
1260static struct kobj_attribute damon_sysfs_quotas_ms_attr =
1261 __ATTR_RW_MODE(ms, 0600);
1262
1263static struct kobj_attribute damon_sysfs_quotas_sz_attr =
1264 __ATTR_RW_MODE(bytes, 0600);
1265
1266static struct kobj_attribute damon_sysfs_quotas_reset_interval_ms_attr =
1267 __ATTR_RW_MODE(reset_interval_ms, 0600);
1268
1269static struct attribute *damon_sysfs_quotas_attrs[] = {
1270 &damon_sysfs_quotas_ms_attr.attr,
1271 &damon_sysfs_quotas_sz_attr.attr,
1272 &damon_sysfs_quotas_reset_interval_ms_attr.attr,
1273 NULL,
1274};
1275ATTRIBUTE_GROUPS(damon_sysfs_quotas);
1276
1277static const struct kobj_type damon_sysfs_quotas_ktype = {
1278 .release = damon_sysfs_quotas_release,
1279 .sysfs_ops = &kobj_sysfs_ops,
1280 .default_groups = damon_sysfs_quotas_groups,
1281};
1282
1283/*
1284 * access_pattern directory
1285 */
1286
1287struct damon_sysfs_access_pattern {
1288 struct kobject kobj;
1289 struct damon_sysfs_ul_range *sz;
1290 struct damon_sysfs_ul_range *nr_accesses;
1291 struct damon_sysfs_ul_range *age;
1292};
1293
1294static
1295struct damon_sysfs_access_pattern *damon_sysfs_access_pattern_alloc(void)
1296{
1297 struct damon_sysfs_access_pattern *access_pattern =
1298 kmalloc(sizeof(*access_pattern), GFP_KERNEL);
1299
1300 if (!access_pattern)
1301 return NULL;
1302 access_pattern->kobj = (struct kobject){};
1303 return access_pattern;
1304}
1305
1306static int damon_sysfs_access_pattern_add_range_dir(
1307 struct damon_sysfs_access_pattern *access_pattern,
1308 struct damon_sysfs_ul_range **range_dir_ptr,
1309 char *name)
1310{
1311 struct damon_sysfs_ul_range *range = damon_sysfs_ul_range_alloc(0, 0);
1312 int err;
1313
1314 if (!range)
1315 return -ENOMEM;
1316 err = kobject_init_and_add(&range->kobj, &damon_sysfs_ul_range_ktype,
1317 &access_pattern->kobj, name);
1318 if (err)
1319 kobject_put(&range->kobj);
1320 else
1321 *range_dir_ptr = range;
1322 return err;
1323}
1324
1325static int damon_sysfs_access_pattern_add_dirs(
1326 struct damon_sysfs_access_pattern *access_pattern)
1327{
1328 int err;
1329
1330 err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1331 &access_pattern->sz, "sz");
1332 if (err)
1333 goto put_sz_out;
1334
1335 err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1336 &access_pattern->nr_accesses, "nr_accesses");
1337 if (err)
1338 goto put_nr_accesses_sz_out;
1339
1340 err = damon_sysfs_access_pattern_add_range_dir(access_pattern,
1341 &access_pattern->age, "age");
1342 if (err)
1343 goto put_age_nr_accesses_sz_out;
1344 return 0;
1345
1346put_age_nr_accesses_sz_out:
1347 kobject_put(&access_pattern->age->kobj);
1348 access_pattern->age = NULL;
1349put_nr_accesses_sz_out:
1350 kobject_put(&access_pattern->nr_accesses->kobj);
1351 access_pattern->nr_accesses = NULL;
1352put_sz_out:
1353 kobject_put(&access_pattern->sz->kobj);
1354 access_pattern->sz = NULL;
1355 return err;
1356}
1357
1358static void damon_sysfs_access_pattern_rm_dirs(
1359 struct damon_sysfs_access_pattern *access_pattern)
1360{
1361 kobject_put(&access_pattern->sz->kobj);
1362 kobject_put(&access_pattern->nr_accesses->kobj);
1363 kobject_put(&access_pattern->age->kobj);
1364}
1365
1366static void damon_sysfs_access_pattern_release(struct kobject *kobj)
1367{
1368 kfree(container_of(kobj, struct damon_sysfs_access_pattern, kobj));
1369}
1370
1371static struct attribute *damon_sysfs_access_pattern_attrs[] = {
1372 NULL,
1373};
1374ATTRIBUTE_GROUPS(damon_sysfs_access_pattern);
1375
1376static const struct kobj_type damon_sysfs_access_pattern_ktype = {
1377 .release = damon_sysfs_access_pattern_release,
1378 .sysfs_ops = &kobj_sysfs_ops,
1379 .default_groups = damon_sysfs_access_pattern_groups,
1380};
1381
1382/*
1383 * scheme directory
1384 */
1385
1386struct damon_sysfs_scheme {
1387 struct kobject kobj;
1388 enum damos_action action;
1389 struct damon_sysfs_access_pattern *access_pattern;
1390 unsigned long apply_interval_us;
1391 struct damon_sysfs_quotas *quotas;
1392 struct damon_sysfs_watermarks *watermarks;
1393 struct damon_sysfs_scheme_filters *filters;
1394 struct damon_sysfs_stats *stats;
1395 struct damon_sysfs_scheme_regions *tried_regions;
1396};
1397
1398/* This should match with enum damos_action */
1399static const char * const damon_sysfs_damos_action_strs[] = {
1400 "willneed",
1401 "cold",
1402 "pageout",
1403 "hugepage",
1404 "nohugepage",
1405 "lru_prio",
1406 "lru_deprio",
1407 "stat",
1408};
1409
1410static struct damon_sysfs_scheme *damon_sysfs_scheme_alloc(
1411 enum damos_action action, unsigned long apply_interval_us)
1412{
1413 struct damon_sysfs_scheme *scheme = kmalloc(sizeof(*scheme),
1414 GFP_KERNEL);
1415
1416 if (!scheme)
1417 return NULL;
1418 scheme->kobj = (struct kobject){};
1419 scheme->action = action;
1420 scheme->apply_interval_us = apply_interval_us;
1421 return scheme;
1422}
1423
1424static int damon_sysfs_scheme_set_access_pattern(
1425 struct damon_sysfs_scheme *scheme)
1426{
1427 struct damon_sysfs_access_pattern *access_pattern;
1428 int err;
1429
1430 access_pattern = damon_sysfs_access_pattern_alloc();
1431 if (!access_pattern)
1432 return -ENOMEM;
1433 err = kobject_init_and_add(&access_pattern->kobj,
1434 &damon_sysfs_access_pattern_ktype, &scheme->kobj,
1435 "access_pattern");
1436 if (err)
1437 goto out;
1438 err = damon_sysfs_access_pattern_add_dirs(access_pattern);
1439 if (err)
1440 goto out;
1441 scheme->access_pattern = access_pattern;
1442 return 0;
1443
1444out:
1445 kobject_put(&access_pattern->kobj);
1446 return err;
1447}
1448
1449static int damon_sysfs_scheme_set_quotas(struct damon_sysfs_scheme *scheme)
1450{
1451 struct damon_sysfs_quotas *quotas = damon_sysfs_quotas_alloc();
1452 int err;
1453
1454 if (!quotas)
1455 return -ENOMEM;
1456 err = kobject_init_and_add("as->kobj, &damon_sysfs_quotas_ktype,
1457 &scheme->kobj, "quotas");
1458 if (err)
1459 goto out;
1460 err = damon_sysfs_quotas_add_dirs(quotas);
1461 if (err)
1462 goto out;
1463 scheme->quotas = quotas;
1464 return 0;
1465
1466out:
1467 kobject_put("as->kobj);
1468 return err;
1469}
1470
1471static int damon_sysfs_scheme_set_watermarks(struct damon_sysfs_scheme *scheme)
1472{
1473 struct damon_sysfs_watermarks *watermarks =
1474 damon_sysfs_watermarks_alloc(DAMOS_WMARK_NONE, 0, 0, 0, 0);
1475 int err;
1476
1477 if (!watermarks)
1478 return -ENOMEM;
1479 err = kobject_init_and_add(&watermarks->kobj,
1480 &damon_sysfs_watermarks_ktype, &scheme->kobj,
1481 "watermarks");
1482 if (err)
1483 kobject_put(&watermarks->kobj);
1484 else
1485 scheme->watermarks = watermarks;
1486 return err;
1487}
1488
1489static int damon_sysfs_scheme_set_filters(struct damon_sysfs_scheme *scheme)
1490{
1491 struct damon_sysfs_scheme_filters *filters =
1492 damon_sysfs_scheme_filters_alloc();
1493 int err;
1494
1495 if (!filters)
1496 return -ENOMEM;
1497 err = kobject_init_and_add(&filters->kobj,
1498 &damon_sysfs_scheme_filters_ktype, &scheme->kobj,
1499 "filters");
1500 if (err)
1501 kobject_put(&filters->kobj);
1502 else
1503 scheme->filters = filters;
1504 return err;
1505}
1506
1507static int damon_sysfs_scheme_set_stats(struct damon_sysfs_scheme *scheme)
1508{
1509 struct damon_sysfs_stats *stats = damon_sysfs_stats_alloc();
1510 int err;
1511
1512 if (!stats)
1513 return -ENOMEM;
1514 err = kobject_init_and_add(&stats->kobj, &damon_sysfs_stats_ktype,
1515 &scheme->kobj, "stats");
1516 if (err)
1517 kobject_put(&stats->kobj);
1518 else
1519 scheme->stats = stats;
1520 return err;
1521}
1522
1523static int damon_sysfs_scheme_set_tried_regions(
1524 struct damon_sysfs_scheme *scheme)
1525{
1526 struct damon_sysfs_scheme_regions *tried_regions =
1527 damon_sysfs_scheme_regions_alloc();
1528 int err;
1529
1530 if (!tried_regions)
1531 return -ENOMEM;
1532 err = kobject_init_and_add(&tried_regions->kobj,
1533 &damon_sysfs_scheme_regions_ktype, &scheme->kobj,
1534 "tried_regions");
1535 if (err)
1536 kobject_put(&tried_regions->kobj);
1537 else
1538 scheme->tried_regions = tried_regions;
1539 return err;
1540}
1541
1542static int damon_sysfs_scheme_add_dirs(struct damon_sysfs_scheme *scheme)
1543{
1544 int err;
1545
1546 err = damon_sysfs_scheme_set_access_pattern(scheme);
1547 if (err)
1548 return err;
1549 err = damon_sysfs_scheme_set_quotas(scheme);
1550 if (err)
1551 goto put_access_pattern_out;
1552 err = damon_sysfs_scheme_set_watermarks(scheme);
1553 if (err)
1554 goto put_quotas_access_pattern_out;
1555 err = damon_sysfs_scheme_set_filters(scheme);
1556 if (err)
1557 goto put_watermarks_quotas_access_pattern_out;
1558 err = damon_sysfs_scheme_set_stats(scheme);
1559 if (err)
1560 goto put_filters_watermarks_quotas_access_pattern_out;
1561 err = damon_sysfs_scheme_set_tried_regions(scheme);
1562 if (err)
1563 goto put_tried_regions_out;
1564 return 0;
1565
1566put_tried_regions_out:
1567 kobject_put(&scheme->tried_regions->kobj);
1568 scheme->tried_regions = NULL;
1569put_filters_watermarks_quotas_access_pattern_out:
1570 kobject_put(&scheme->filters->kobj);
1571 scheme->filters = NULL;
1572put_watermarks_quotas_access_pattern_out:
1573 kobject_put(&scheme->watermarks->kobj);
1574 scheme->watermarks = NULL;
1575put_quotas_access_pattern_out:
1576 kobject_put(&scheme->quotas->kobj);
1577 scheme->quotas = NULL;
1578put_access_pattern_out:
1579 kobject_put(&scheme->access_pattern->kobj);
1580 scheme->access_pattern = NULL;
1581 return err;
1582}
1583
1584static void damon_sysfs_scheme_rm_dirs(struct damon_sysfs_scheme *scheme)
1585{
1586 damon_sysfs_access_pattern_rm_dirs(scheme->access_pattern);
1587 kobject_put(&scheme->access_pattern->kobj);
1588 damon_sysfs_quotas_rm_dirs(scheme->quotas);
1589 kobject_put(&scheme->quotas->kobj);
1590 kobject_put(&scheme->watermarks->kobj);
1591 damon_sysfs_scheme_filters_rm_dirs(scheme->filters);
1592 kobject_put(&scheme->filters->kobj);
1593 kobject_put(&scheme->stats->kobj);
1594 damon_sysfs_scheme_regions_rm_dirs(scheme->tried_regions);
1595 kobject_put(&scheme->tried_regions->kobj);
1596}
1597
1598static ssize_t action_show(struct kobject *kobj, struct kobj_attribute *attr,
1599 char *buf)
1600{
1601 struct damon_sysfs_scheme *scheme = container_of(kobj,
1602 struct damon_sysfs_scheme, kobj);
1603
1604 return sysfs_emit(buf, "%s\n",
1605 damon_sysfs_damos_action_strs[scheme->action]);
1606}
1607
1608static ssize_t action_store(struct kobject *kobj, struct kobj_attribute *attr,
1609 const char *buf, size_t count)
1610{
1611 struct damon_sysfs_scheme *scheme = container_of(kobj,
1612 struct damon_sysfs_scheme, kobj);
1613 enum damos_action action;
1614
1615 for (action = 0; action < NR_DAMOS_ACTIONS; action++) {
1616 if (sysfs_streq(buf, damon_sysfs_damos_action_strs[action])) {
1617 scheme->action = action;
1618 return count;
1619 }
1620 }
1621 return -EINVAL;
1622}
1623
1624static ssize_t apply_interval_us_show(struct kobject *kobj,
1625 struct kobj_attribute *attr, char *buf)
1626{
1627 struct damon_sysfs_scheme *scheme = container_of(kobj,
1628 struct damon_sysfs_scheme, kobj);
1629
1630 return sysfs_emit(buf, "%lu\n", scheme->apply_interval_us);
1631}
1632
1633static ssize_t apply_interval_us_store(struct kobject *kobj,
1634 struct kobj_attribute *attr, const char *buf, size_t count)
1635{
1636 struct damon_sysfs_scheme *scheme = container_of(kobj,
1637 struct damon_sysfs_scheme, kobj);
1638 int err = kstrtoul(buf, 0, &scheme->apply_interval_us);
1639
1640 return err ? err : count;
1641}
1642
1643static void damon_sysfs_scheme_release(struct kobject *kobj)
1644{
1645 kfree(container_of(kobj, struct damon_sysfs_scheme, kobj));
1646}
1647
1648static struct kobj_attribute damon_sysfs_scheme_action_attr =
1649 __ATTR_RW_MODE(action, 0600);
1650
1651static struct kobj_attribute damon_sysfs_scheme_apply_interval_us_attr =
1652 __ATTR_RW_MODE(apply_interval_us, 0600);
1653
1654static struct attribute *damon_sysfs_scheme_attrs[] = {
1655 &damon_sysfs_scheme_action_attr.attr,
1656 &damon_sysfs_scheme_apply_interval_us_attr.attr,
1657 NULL,
1658};
1659ATTRIBUTE_GROUPS(damon_sysfs_scheme);
1660
1661static const struct kobj_type damon_sysfs_scheme_ktype = {
1662 .release = damon_sysfs_scheme_release,
1663 .sysfs_ops = &kobj_sysfs_ops,
1664 .default_groups = damon_sysfs_scheme_groups,
1665};
1666
1667/*
1668 * schemes directory
1669 */
1670
1671struct damon_sysfs_schemes *damon_sysfs_schemes_alloc(void)
1672{
1673 return kzalloc(sizeof(struct damon_sysfs_schemes), GFP_KERNEL);
1674}
1675
1676void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes)
1677{
1678 struct damon_sysfs_scheme **schemes_arr = schemes->schemes_arr;
1679 int i;
1680
1681 for (i = 0; i < schemes->nr; i++) {
1682 damon_sysfs_scheme_rm_dirs(schemes_arr[i]);
1683 kobject_put(&schemes_arr[i]->kobj);
1684 }
1685 schemes->nr = 0;
1686 kfree(schemes_arr);
1687 schemes->schemes_arr = NULL;
1688}
1689
1690static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes,
1691 int nr_schemes)
1692{
1693 struct damon_sysfs_scheme **schemes_arr, *scheme;
1694 int err, i;
1695
1696 damon_sysfs_schemes_rm_dirs(schemes);
1697 if (!nr_schemes)
1698 return 0;
1699
1700 schemes_arr = kmalloc_array(nr_schemes, sizeof(*schemes_arr),
1701 GFP_KERNEL | __GFP_NOWARN);
1702 if (!schemes_arr)
1703 return -ENOMEM;
1704 schemes->schemes_arr = schemes_arr;
1705
1706 for (i = 0; i < nr_schemes; i++) {
1707 /*
1708 * apply_interval_us as 0 means same to aggregation interval
1709 * (same to before-apply_interval behavior)
1710 */
1711 scheme = damon_sysfs_scheme_alloc(DAMOS_STAT, 0);
1712 if (!scheme) {
1713 damon_sysfs_schemes_rm_dirs(schemes);
1714 return -ENOMEM;
1715 }
1716
1717 err = kobject_init_and_add(&scheme->kobj,
1718 &damon_sysfs_scheme_ktype, &schemes->kobj,
1719 "%d", i);
1720 if (err)
1721 goto out;
1722 err = damon_sysfs_scheme_add_dirs(scheme);
1723 if (err)
1724 goto out;
1725
1726 schemes_arr[i] = scheme;
1727 schemes->nr++;
1728 }
1729 return 0;
1730
1731out:
1732 damon_sysfs_schemes_rm_dirs(schemes);
1733 kobject_put(&scheme->kobj);
1734 return err;
1735}
1736
1737static ssize_t nr_schemes_show(struct kobject *kobj,
1738 struct kobj_attribute *attr, char *buf)
1739{
1740 struct damon_sysfs_schemes *schemes = container_of(kobj,
1741 struct damon_sysfs_schemes, kobj);
1742
1743 return sysfs_emit(buf, "%d\n", schemes->nr);
1744}
1745
1746static ssize_t nr_schemes_store(struct kobject *kobj,
1747 struct kobj_attribute *attr, const char *buf, size_t count)
1748{
1749 struct damon_sysfs_schemes *schemes;
1750 int nr, err = kstrtoint(buf, 0, &nr);
1751
1752 if (err)
1753 return err;
1754 if (nr < 0)
1755 return -EINVAL;
1756
1757 schemes = container_of(kobj, struct damon_sysfs_schemes, kobj);
1758
1759 if (!mutex_trylock(&damon_sysfs_lock))
1760 return -EBUSY;
1761 err = damon_sysfs_schemes_add_dirs(schemes, nr);
1762 mutex_unlock(&damon_sysfs_lock);
1763 if (err)
1764 return err;
1765 return count;
1766}
1767
1768static void damon_sysfs_schemes_release(struct kobject *kobj)
1769{
1770 kfree(container_of(kobj, struct damon_sysfs_schemes, kobj));
1771}
1772
1773static struct kobj_attribute damon_sysfs_schemes_nr_attr =
1774 __ATTR_RW_MODE(nr_schemes, 0600);
1775
1776static struct attribute *damon_sysfs_schemes_attrs[] = {
1777 &damon_sysfs_schemes_nr_attr.attr,
1778 NULL,
1779};
1780ATTRIBUTE_GROUPS(damon_sysfs_schemes);
1781
1782const struct kobj_type damon_sysfs_schemes_ktype = {
1783 .release = damon_sysfs_schemes_release,
1784 .sysfs_ops = &kobj_sysfs_ops,
1785 .default_groups = damon_sysfs_schemes_groups,
1786};
1787
1788static bool damon_sysfs_memcg_path_eq(struct mem_cgroup *memcg,
1789 char *memcg_path_buf, char *path)
1790{
1791#ifdef CONFIG_MEMCG
1792 cgroup_path(memcg->css.cgroup, memcg_path_buf, PATH_MAX);
1793 if (sysfs_streq(memcg_path_buf, path))
1794 return true;
1795#endif /* CONFIG_MEMCG */
1796 return false;
1797}
1798
1799static int damon_sysfs_memcg_path_to_id(char *memcg_path, unsigned short *id)
1800{
1801 struct mem_cgroup *memcg;
1802 char *path;
1803 bool found = false;
1804
1805 if (!memcg_path)
1806 return -EINVAL;
1807
1808 path = kmalloc(sizeof(*path) * PATH_MAX, GFP_KERNEL);
1809 if (!path)
1810 return -ENOMEM;
1811
1812 for (memcg = mem_cgroup_iter(NULL, NULL, NULL); memcg;
1813 memcg = mem_cgroup_iter(NULL, memcg, NULL)) {
1814 /* skip removed memcg */
1815 if (!mem_cgroup_id(memcg))
1816 continue;
1817 if (damon_sysfs_memcg_path_eq(memcg, path, memcg_path)) {
1818 *id = mem_cgroup_id(memcg);
1819 found = true;
1820 break;
1821 }
1822 }
1823
1824 kfree(path);
1825 return found ? 0 : -EINVAL;
1826}
1827
1828static int damon_sysfs_set_scheme_filters(struct damos *scheme,
1829 struct damon_sysfs_scheme_filters *sysfs_filters)
1830{
1831 int i;
1832 struct damos_filter *filter, *next;
1833
1834 damos_for_each_filter_safe(filter, next, scheme)
1835 damos_destroy_filter(filter);
1836
1837 for (i = 0; i < sysfs_filters->nr; i++) {
1838 struct damon_sysfs_scheme_filter *sysfs_filter =
1839 sysfs_filters->filters_arr[i];
1840 struct damos_filter *filter =
1841 damos_new_filter(sysfs_filter->type,
1842 sysfs_filter->matching);
1843 int err;
1844
1845 if (!filter)
1846 return -ENOMEM;
1847 if (filter->type == DAMOS_FILTER_TYPE_MEMCG) {
1848 err = damon_sysfs_memcg_path_to_id(
1849 sysfs_filter->memcg_path,
1850 &filter->memcg_id);
1851 if (err) {
1852 damos_destroy_filter(filter);
1853 return err;
1854 }
1855 } else if (filter->type == DAMOS_FILTER_TYPE_ADDR) {
1856 if (sysfs_filter->addr_range.end <
1857 sysfs_filter->addr_range.start) {
1858 damos_destroy_filter(filter);
1859 return -EINVAL;
1860 }
1861 filter->addr_range = sysfs_filter->addr_range;
1862 } else if (filter->type == DAMOS_FILTER_TYPE_TARGET) {
1863 filter->target_idx = sysfs_filter->target_idx;
1864 }
1865
1866 damos_add_filter(scheme, filter);
1867 }
1868 return 0;
1869}
1870
1871static unsigned long damos_sysfs_get_quota_score(void *arg)
1872{
1873 return (unsigned long)arg;
1874}
1875
1876static void damos_sysfs_set_quota_score(
1877 struct damos_sysfs_quota_goals *sysfs_goals,
1878 struct damos_quota *quota)
1879{
1880 struct damos_sysfs_quota_goal *sysfs_goal;
1881 int i;
1882
1883 quota->get_score = NULL;
1884 quota->get_score_arg = (void *)0;
1885 for (i = 0; i < sysfs_goals->nr; i++) {
1886 sysfs_goal = sysfs_goals->goals_arr[i];
1887 if (!sysfs_goal->target_value)
1888 continue;
1889
1890 /* Higher score makes scheme less aggressive */
1891 quota->get_score_arg = (void *)max(
1892 (unsigned long)quota->get_score_arg,
1893 sysfs_goal->current_value * 10000 /
1894 sysfs_goal->target_value);
1895 quota->get_score = damos_sysfs_get_quota_score;
1896 }
1897}
1898
1899void damos_sysfs_set_quota_scores(struct damon_sysfs_schemes *sysfs_schemes,
1900 struct damon_ctx *ctx)
1901{
1902 struct damos *scheme;
1903 int i = 0;
1904
1905 damon_for_each_scheme(scheme, ctx) {
1906 struct damon_sysfs_scheme *sysfs_scheme;
1907
1908 /* user could have removed the scheme sysfs dir */
1909 if (i >= sysfs_schemes->nr)
1910 break;
1911
1912 sysfs_scheme = sysfs_schemes->schemes_arr[i];
1913 damos_sysfs_set_quota_score(sysfs_scheme->quotas->goals,
1914 &scheme->quota);
1915 i++;
1916 }
1917}
1918
1919static struct damos *damon_sysfs_mk_scheme(
1920 struct damon_sysfs_scheme *sysfs_scheme)
1921{
1922 struct damon_sysfs_access_pattern *access_pattern =
1923 sysfs_scheme->access_pattern;
1924 struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas;
1925 struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights;
1926 struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks;
1927 struct damon_sysfs_scheme_filters *sysfs_filters =
1928 sysfs_scheme->filters;
1929 struct damos *scheme;
1930 int err;
1931
1932 struct damos_access_pattern pattern = {
1933 .min_sz_region = access_pattern->sz->min,
1934 .max_sz_region = access_pattern->sz->max,
1935 .min_nr_accesses = access_pattern->nr_accesses->min,
1936 .max_nr_accesses = access_pattern->nr_accesses->max,
1937 .min_age_region = access_pattern->age->min,
1938 .max_age_region = access_pattern->age->max,
1939 };
1940 struct damos_quota quota = {
1941 .ms = sysfs_quotas->ms,
1942 .sz = sysfs_quotas->sz,
1943 .reset_interval = sysfs_quotas->reset_interval_ms,
1944 .weight_sz = sysfs_weights->sz,
1945 .weight_nr_accesses = sysfs_weights->nr_accesses,
1946 .weight_age = sysfs_weights->age,
1947 };
1948 struct damos_watermarks wmarks = {
1949 .metric = sysfs_wmarks->metric,
1950 .interval = sysfs_wmarks->interval_us,
1951 .high = sysfs_wmarks->high,
1952 .mid = sysfs_wmarks->mid,
1953 .low = sysfs_wmarks->low,
1954 };
1955
1956 damos_sysfs_set_quota_score(sysfs_quotas->goals, "a);
1957
1958 scheme = damon_new_scheme(&pattern, sysfs_scheme->action,
1959 sysfs_scheme->apply_interval_us, "a, &wmarks);
1960 if (!scheme)
1961 return NULL;
1962
1963 err = damon_sysfs_set_scheme_filters(scheme, sysfs_filters);
1964 if (err) {
1965 damon_destroy_scheme(scheme);
1966 return NULL;
1967 }
1968 return scheme;
1969}
1970
1971static void damon_sysfs_update_scheme(struct damos *scheme,
1972 struct damon_sysfs_scheme *sysfs_scheme)
1973{
1974 struct damon_sysfs_access_pattern *access_pattern =
1975 sysfs_scheme->access_pattern;
1976 struct damon_sysfs_quotas *sysfs_quotas = sysfs_scheme->quotas;
1977 struct damon_sysfs_weights *sysfs_weights = sysfs_quotas->weights;
1978 struct damon_sysfs_watermarks *sysfs_wmarks = sysfs_scheme->watermarks;
1979 int err;
1980
1981 scheme->pattern.min_sz_region = access_pattern->sz->min;
1982 scheme->pattern.max_sz_region = access_pattern->sz->max;
1983 scheme->pattern.min_nr_accesses = access_pattern->nr_accesses->min;
1984 scheme->pattern.max_nr_accesses = access_pattern->nr_accesses->max;
1985 scheme->pattern.min_age_region = access_pattern->age->min;
1986 scheme->pattern.max_age_region = access_pattern->age->max;
1987
1988 scheme->action = sysfs_scheme->action;
1989 scheme->apply_interval_us = sysfs_scheme->apply_interval_us;
1990
1991 scheme->quota.ms = sysfs_quotas->ms;
1992 scheme->quota.sz = sysfs_quotas->sz;
1993 scheme->quota.reset_interval = sysfs_quotas->reset_interval_ms;
1994 scheme->quota.weight_sz = sysfs_weights->sz;
1995 scheme->quota.weight_nr_accesses = sysfs_weights->nr_accesses;
1996 scheme->quota.weight_age = sysfs_weights->age;
1997
1998 damos_sysfs_set_quota_score(sysfs_quotas->goals, &scheme->quota);
1999
2000 scheme->wmarks.metric = sysfs_wmarks->metric;
2001 scheme->wmarks.interval = sysfs_wmarks->interval_us;
2002 scheme->wmarks.high = sysfs_wmarks->high;
2003 scheme->wmarks.mid = sysfs_wmarks->mid;
2004 scheme->wmarks.low = sysfs_wmarks->low;
2005
2006 err = damon_sysfs_set_scheme_filters(scheme, sysfs_scheme->filters);
2007 if (err)
2008 damon_destroy_scheme(scheme);
2009}
2010
2011int damon_sysfs_set_schemes(struct damon_ctx *ctx,
2012 struct damon_sysfs_schemes *sysfs_schemes)
2013{
2014 struct damos *scheme, *next;
2015 int i = 0;
2016
2017 damon_for_each_scheme_safe(scheme, next, ctx) {
2018 if (i < sysfs_schemes->nr)
2019 damon_sysfs_update_scheme(scheme,
2020 sysfs_schemes->schemes_arr[i]);
2021 else
2022 damon_destroy_scheme(scheme);
2023 i++;
2024 }
2025
2026 for (; i < sysfs_schemes->nr; i++) {
2027 struct damos *scheme, *next;
2028
2029 scheme = damon_sysfs_mk_scheme(sysfs_schemes->schemes_arr[i]);
2030 if (!scheme) {
2031 damon_for_each_scheme_safe(scheme, next, ctx)
2032 damon_destroy_scheme(scheme);
2033 return -ENOMEM;
2034 }
2035 damon_add_scheme(ctx, scheme);
2036 }
2037 return 0;
2038}
2039
2040void damon_sysfs_schemes_update_stats(
2041 struct damon_sysfs_schemes *sysfs_schemes,
2042 struct damon_ctx *ctx)
2043{
2044 struct damos *scheme;
2045 int schemes_idx = 0;
2046
2047 damon_for_each_scheme(scheme, ctx) {
2048 struct damon_sysfs_stats *sysfs_stats;
2049
2050 /* user could have removed the scheme sysfs dir */
2051 if (schemes_idx >= sysfs_schemes->nr)
2052 break;
2053
2054 sysfs_stats = sysfs_schemes->schemes_arr[schemes_idx++]->stats;
2055 sysfs_stats->nr_tried = scheme->stat.nr_tried;
2056 sysfs_stats->sz_tried = scheme->stat.sz_tried;
2057 sysfs_stats->nr_applied = scheme->stat.nr_applied;
2058 sysfs_stats->sz_applied = scheme->stat.sz_applied;
2059 sysfs_stats->qt_exceeds = scheme->stat.qt_exceeds;
2060 }
2061}
2062
2063/*
2064 * damon_sysfs_schemes that need to update its schemes regions dir. Protected
2065 * by damon_sysfs_lock
2066 */
2067static struct damon_sysfs_schemes *damon_sysfs_schemes_for_damos_callback;
2068static int damon_sysfs_schemes_region_idx;
2069static bool damos_regions_upd_total_bytes_only;
2070
2071/*
2072 * DAMON callback that called before damos apply. While this callback is
2073 * registered, damon_sysfs_lock should be held to ensure the regions
2074 * directories exist.
2075 */
2076static int damon_sysfs_before_damos_apply(struct damon_ctx *ctx,
2077 struct damon_target *t, struct damon_region *r,
2078 struct damos *s)
2079{
2080 struct damos *scheme;
2081 struct damon_sysfs_scheme_regions *sysfs_regions;
2082 struct damon_sysfs_scheme_region *region;
2083 struct damon_sysfs_schemes *sysfs_schemes =
2084 damon_sysfs_schemes_for_damos_callback;
2085 int schemes_idx = 0;
2086
2087 damon_for_each_scheme(scheme, ctx) {
2088 if (scheme == s)
2089 break;
2090 schemes_idx++;
2091 }
2092
2093 /* user could have removed the scheme sysfs dir */
2094 if (schemes_idx >= sysfs_schemes->nr)
2095 return 0;
2096
2097 sysfs_regions = sysfs_schemes->schemes_arr[schemes_idx]->tried_regions;
2098 if (sysfs_regions->upd_status == DAMOS_TRIED_REGIONS_UPD_FINISHED)
2099 return 0;
2100 if (sysfs_regions->upd_status == DAMOS_TRIED_REGIONS_UPD_IDLE)
2101 sysfs_regions->upd_status = DAMOS_TRIED_REGIONS_UPD_STARTED;
2102 sysfs_regions->total_bytes += r->ar.end - r->ar.start;
2103 if (damos_regions_upd_total_bytes_only)
2104 return 0;
2105
2106 region = damon_sysfs_scheme_region_alloc(r);
2107 if (!region)
2108 return 0;
2109 list_add_tail(®ion->list, &sysfs_regions->regions_list);
2110 sysfs_regions->nr_regions++;
2111 if (kobject_init_and_add(®ion->kobj,
2112 &damon_sysfs_scheme_region_ktype,
2113 &sysfs_regions->kobj, "%d",
2114 damon_sysfs_schemes_region_idx++)) {
2115 kobject_put(®ion->kobj);
2116 }
2117 return 0;
2118}
2119
2120/*
2121 * DAMON callback that called after each accesses sampling. While this
2122 * callback is registered, damon_sysfs_lock should be held to ensure the
2123 * regions directories exist.
2124 */
2125static int damon_sysfs_after_sampling(struct damon_ctx *ctx)
2126{
2127 struct damon_sysfs_schemes *sysfs_schemes =
2128 damon_sysfs_schemes_for_damos_callback;
2129 struct damon_sysfs_scheme_regions *sysfs_regions;
2130 int i;
2131
2132 for (i = 0; i < sysfs_schemes->nr; i++) {
2133 sysfs_regions = sysfs_schemes->schemes_arr[i]->tried_regions;
2134 if (sysfs_regions->upd_status ==
2135 DAMOS_TRIED_REGIONS_UPD_STARTED ||
2136 time_after(jiffies,
2137 sysfs_regions->upd_timeout_jiffies))
2138 sysfs_regions->upd_status =
2139 DAMOS_TRIED_REGIONS_UPD_FINISHED;
2140 }
2141
2142 return 0;
2143}
2144
2145/* Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock */
2146int damon_sysfs_schemes_clear_regions(
2147 struct damon_sysfs_schemes *sysfs_schemes,
2148 struct damon_ctx *ctx)
2149{
2150 struct damos *scheme;
2151 int schemes_idx = 0;
2152
2153 damon_for_each_scheme(scheme, ctx) {
2154 struct damon_sysfs_scheme *sysfs_scheme;
2155
2156 /* user could have removed the scheme sysfs dir */
2157 if (schemes_idx >= sysfs_schemes->nr)
2158 break;
2159
2160 sysfs_scheme = sysfs_schemes->schemes_arr[schemes_idx++];
2161 damon_sysfs_scheme_regions_rm_dirs(
2162 sysfs_scheme->tried_regions);
2163 sysfs_scheme->tried_regions->total_bytes = 0;
2164 }
2165 return 0;
2166}
2167
2168static struct damos *damos_sysfs_nth_scheme(int n, struct damon_ctx *ctx)
2169{
2170 struct damos *scheme;
2171 int i = 0;
2172
2173 damon_for_each_scheme(scheme, ctx) {
2174 if (i == n)
2175 return scheme;
2176 i++;
2177 }
2178 return NULL;
2179}
2180
2181static void damos_tried_regions_init_upd_status(
2182 struct damon_sysfs_schemes *sysfs_schemes,
2183 struct damon_ctx *ctx)
2184{
2185 int i;
2186 struct damos *scheme;
2187 struct damon_sysfs_scheme_regions *sysfs_regions;
2188
2189 for (i = 0; i < sysfs_schemes->nr; i++) {
2190 sysfs_regions = sysfs_schemes->schemes_arr[i]->tried_regions;
2191 scheme = damos_sysfs_nth_scheme(i, ctx);
2192 if (!scheme) {
2193 sysfs_regions->upd_status =
2194 DAMOS_TRIED_REGIONS_UPD_FINISHED;
2195 continue;
2196 }
2197 sysfs_regions->upd_status = DAMOS_TRIED_REGIONS_UPD_IDLE;
2198 sysfs_regions->upd_timeout_jiffies = jiffies +
2199 2 * usecs_to_jiffies(scheme->apply_interval_us ?
2200 scheme->apply_interval_us :
2201 ctx->attrs.aggr_interval);
2202 }
2203}
2204
2205/* Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock */
2206int damon_sysfs_schemes_update_regions_start(
2207 struct damon_sysfs_schemes *sysfs_schemes,
2208 struct damon_ctx *ctx, bool total_bytes_only)
2209{
2210 damon_sysfs_schemes_clear_regions(sysfs_schemes, ctx);
2211 damon_sysfs_schemes_for_damos_callback = sysfs_schemes;
2212 damos_tried_regions_init_upd_status(sysfs_schemes, ctx);
2213 damos_regions_upd_total_bytes_only = total_bytes_only;
2214 ctx->callback.before_damos_apply = damon_sysfs_before_damos_apply;
2215 ctx->callback.after_sampling = damon_sysfs_after_sampling;
2216 return 0;
2217}
2218
2219bool damos_sysfs_regions_upd_done(void)
2220{
2221 struct damon_sysfs_schemes *sysfs_schemes =
2222 damon_sysfs_schemes_for_damos_callback;
2223 struct damon_sysfs_scheme_regions *sysfs_regions;
2224 int i;
2225
2226 for (i = 0; i < sysfs_schemes->nr; i++) {
2227 sysfs_regions = sysfs_schemes->schemes_arr[i]->tried_regions;
2228 if (sysfs_regions->upd_status !=
2229 DAMOS_TRIED_REGIONS_UPD_FINISHED)
2230 return false;
2231 }
2232 return true;
2233}
2234
2235/*
2236 * Called from damon_sysfs_cmd_request_callback under damon_sysfs_lock. Caller
2237 * should unlock damon_sysfs_lock which held before
2238 * damon_sysfs_schemes_update_regions_start()
2239 */
2240int damon_sysfs_schemes_update_regions_stop(struct damon_ctx *ctx)
2241{
2242 damon_sysfs_schemes_for_damos_callback = NULL;
2243 ctx->callback.before_damos_apply = NULL;
2244 ctx->callback.after_sampling = NULL;
2245 damon_sysfs_schemes_region_idx = 0;
2246 return 0;
2247}