Loading...
1/*
2 *
3 * Copyright IBM Corporation, 2012
4 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of version 2.1 of the GNU Lesser General Public License
8 * as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it would be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13 *
14 */
15
16#include <linux/cgroup.h>
17#include <linux/page_counter.h>
18#include <linux/slab.h>
19#include <linux/hugetlb.h>
20#include <linux/hugetlb_cgroup.h>
21
22struct hugetlb_cgroup {
23 struct cgroup_subsys_state css;
24 /*
25 * the counter to account for hugepages from hugetlb.
26 */
27 struct page_counter hugepage[HUGE_MAX_HSTATE];
28};
29
30#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
31#define MEMFILE_IDX(val) (((val) >> 16) & 0xffff)
32#define MEMFILE_ATTR(val) ((val) & 0xffff)
33
34static struct hugetlb_cgroup *root_h_cgroup __read_mostly;
35
36static inline
37struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s)
38{
39 return s ? container_of(s, struct hugetlb_cgroup, css) : NULL;
40}
41
42static inline
43struct hugetlb_cgroup *hugetlb_cgroup_from_task(struct task_struct *task)
44{
45 return hugetlb_cgroup_from_css(task_css(task, hugetlb_cgrp_id));
46}
47
48static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg)
49{
50 return (h_cg == root_h_cgroup);
51}
52
53static inline struct hugetlb_cgroup *
54parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg)
55{
56 return hugetlb_cgroup_from_css(h_cg->css.parent);
57}
58
59static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg)
60{
61 int idx;
62
63 for (idx = 0; idx < hugetlb_max_hstate; idx++) {
64 if (page_counter_read(&h_cg->hugepage[idx]))
65 return true;
66 }
67 return false;
68}
69
70static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup,
71 struct hugetlb_cgroup *parent_h_cgroup)
72{
73 int idx;
74
75 for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) {
76 struct page_counter *counter = &h_cgroup->hugepage[idx];
77 struct page_counter *parent = NULL;
78 unsigned long limit;
79 int ret;
80
81 if (parent_h_cgroup)
82 parent = &parent_h_cgroup->hugepage[idx];
83 page_counter_init(counter, parent);
84
85 limit = round_down(PAGE_COUNTER_MAX,
86 1 << huge_page_order(&hstates[idx]));
87 ret = page_counter_set_max(counter, limit);
88 VM_BUG_ON(ret);
89 }
90}
91
92static struct cgroup_subsys_state *
93hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
94{
95 struct hugetlb_cgroup *parent_h_cgroup = hugetlb_cgroup_from_css(parent_css);
96 struct hugetlb_cgroup *h_cgroup;
97
98 h_cgroup = kzalloc(sizeof(*h_cgroup), GFP_KERNEL);
99 if (!h_cgroup)
100 return ERR_PTR(-ENOMEM);
101
102 if (!parent_h_cgroup)
103 root_h_cgroup = h_cgroup;
104
105 hugetlb_cgroup_init(h_cgroup, parent_h_cgroup);
106 return &h_cgroup->css;
107}
108
109static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css)
110{
111 struct hugetlb_cgroup *h_cgroup;
112
113 h_cgroup = hugetlb_cgroup_from_css(css);
114 kfree(h_cgroup);
115}
116
117
118/*
119 * Should be called with hugetlb_lock held.
120 * Since we are holding hugetlb_lock, pages cannot get moved from
121 * active list or uncharged from the cgroup, So no need to get
122 * page reference and test for page active here. This function
123 * cannot fail.
124 */
125static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg,
126 struct page *page)
127{
128 unsigned int nr_pages;
129 struct page_counter *counter;
130 struct hugetlb_cgroup *page_hcg;
131 struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg);
132
133 page_hcg = hugetlb_cgroup_from_page(page);
134 /*
135 * We can have pages in active list without any cgroup
136 * ie, hugepage with less than 3 pages. We can safely
137 * ignore those pages.
138 */
139 if (!page_hcg || page_hcg != h_cg)
140 goto out;
141
142 nr_pages = compound_nr(page);
143 if (!parent) {
144 parent = root_h_cgroup;
145 /* root has no limit */
146 page_counter_charge(&parent->hugepage[idx], nr_pages);
147 }
148 counter = &h_cg->hugepage[idx];
149 /* Take the pages off the local counter */
150 page_counter_cancel(counter, nr_pages);
151
152 set_hugetlb_cgroup(page, parent);
153out:
154 return;
155}
156
157/*
158 * Force the hugetlb cgroup to empty the hugetlb resources by moving them to
159 * the parent cgroup.
160 */
161static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css)
162{
163 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
164 struct hstate *h;
165 struct page *page;
166 int idx = 0;
167
168 do {
169 for_each_hstate(h) {
170 spin_lock(&hugetlb_lock);
171 list_for_each_entry(page, &h->hugepage_activelist, lru)
172 hugetlb_cgroup_move_parent(idx, h_cg, page);
173
174 spin_unlock(&hugetlb_lock);
175 idx++;
176 }
177 cond_resched();
178 } while (hugetlb_cgroup_have_usage(h_cg));
179}
180
181int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
182 struct hugetlb_cgroup **ptr)
183{
184 int ret = 0;
185 struct page_counter *counter;
186 struct hugetlb_cgroup *h_cg = NULL;
187
188 if (hugetlb_cgroup_disabled())
189 goto done;
190 /*
191 * We don't charge any cgroup if the compound page have less
192 * than 3 pages.
193 */
194 if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER)
195 goto done;
196again:
197 rcu_read_lock();
198 h_cg = hugetlb_cgroup_from_task(current);
199 if (!css_tryget(&h_cg->css)) {
200 rcu_read_unlock();
201 goto again;
202 }
203 rcu_read_unlock();
204
205 if (!page_counter_try_charge(&h_cg->hugepage[idx], nr_pages, &counter))
206 ret = -ENOMEM;
207 css_put(&h_cg->css);
208done:
209 *ptr = h_cg;
210 return ret;
211}
212
213/* Should be called with hugetlb_lock held */
214void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
215 struct hugetlb_cgroup *h_cg,
216 struct page *page)
217{
218 if (hugetlb_cgroup_disabled() || !h_cg)
219 return;
220
221 set_hugetlb_cgroup(page, h_cg);
222 return;
223}
224
225/*
226 * Should be called with hugetlb_lock held
227 */
228void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
229 struct page *page)
230{
231 struct hugetlb_cgroup *h_cg;
232
233 if (hugetlb_cgroup_disabled())
234 return;
235 lockdep_assert_held(&hugetlb_lock);
236 h_cg = hugetlb_cgroup_from_page(page);
237 if (unlikely(!h_cg))
238 return;
239 set_hugetlb_cgroup(page, NULL);
240 page_counter_uncharge(&h_cg->hugepage[idx], nr_pages);
241 return;
242}
243
244void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
245 struct hugetlb_cgroup *h_cg)
246{
247 if (hugetlb_cgroup_disabled() || !h_cg)
248 return;
249
250 if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER)
251 return;
252
253 page_counter_uncharge(&h_cg->hugepage[idx], nr_pages);
254 return;
255}
256
257enum {
258 RES_USAGE,
259 RES_LIMIT,
260 RES_MAX_USAGE,
261 RES_FAILCNT,
262};
263
264static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css,
265 struct cftype *cft)
266{
267 struct page_counter *counter;
268 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
269
270 counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];
271
272 switch (MEMFILE_ATTR(cft->private)) {
273 case RES_USAGE:
274 return (u64)page_counter_read(counter) * PAGE_SIZE;
275 case RES_LIMIT:
276 return (u64)counter->max * PAGE_SIZE;
277 case RES_MAX_USAGE:
278 return (u64)counter->watermark * PAGE_SIZE;
279 case RES_FAILCNT:
280 return counter->failcnt;
281 default:
282 BUG();
283 }
284}
285
286static DEFINE_MUTEX(hugetlb_limit_mutex);
287
288static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
289 char *buf, size_t nbytes, loff_t off)
290{
291 int ret, idx;
292 unsigned long nr_pages;
293 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
294
295 if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */
296 return -EINVAL;
297
298 buf = strstrip(buf);
299 ret = page_counter_memparse(buf, "-1", &nr_pages);
300 if (ret)
301 return ret;
302
303 idx = MEMFILE_IDX(of_cft(of)->private);
304 nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx]));
305
306 switch (MEMFILE_ATTR(of_cft(of)->private)) {
307 case RES_LIMIT:
308 mutex_lock(&hugetlb_limit_mutex);
309 ret = page_counter_set_max(&h_cg->hugepage[idx], nr_pages);
310 mutex_unlock(&hugetlb_limit_mutex);
311 break;
312 default:
313 ret = -EINVAL;
314 break;
315 }
316 return ret ?: nbytes;
317}
318
319static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of,
320 char *buf, size_t nbytes, loff_t off)
321{
322 int ret = 0;
323 struct page_counter *counter;
324 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
325
326 counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];
327
328 switch (MEMFILE_ATTR(of_cft(of)->private)) {
329 case RES_MAX_USAGE:
330 page_counter_reset_watermark(counter);
331 break;
332 case RES_FAILCNT:
333 counter->failcnt = 0;
334 break;
335 default:
336 ret = -EINVAL;
337 break;
338 }
339 return ret ?: nbytes;
340}
341
342static char *mem_fmt(char *buf, int size, unsigned long hsize)
343{
344 if (hsize >= (1UL << 30))
345 snprintf(buf, size, "%luGB", hsize >> 30);
346 else if (hsize >= (1UL << 20))
347 snprintf(buf, size, "%luMB", hsize >> 20);
348 else
349 snprintf(buf, size, "%luKB", hsize >> 10);
350 return buf;
351}
352
353static void __init __hugetlb_cgroup_file_init(int idx)
354{
355 char buf[32];
356 struct cftype *cft;
357 struct hstate *h = &hstates[idx];
358
359 /* format the size */
360 mem_fmt(buf, 32, huge_page_size(h));
361
362 /* Add the limit file */
363 cft = &h->cgroup_files[0];
364 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf);
365 cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);
366 cft->read_u64 = hugetlb_cgroup_read_u64;
367 cft->write = hugetlb_cgroup_write;
368
369 /* Add the usage file */
370 cft = &h->cgroup_files[1];
371 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf);
372 cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);
373 cft->read_u64 = hugetlb_cgroup_read_u64;
374
375 /* Add the MAX usage file */
376 cft = &h->cgroup_files[2];
377 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf);
378 cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE);
379 cft->write = hugetlb_cgroup_reset;
380 cft->read_u64 = hugetlb_cgroup_read_u64;
381
382 /* Add the failcntfile */
383 cft = &h->cgroup_files[3];
384 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf);
385 cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT);
386 cft->write = hugetlb_cgroup_reset;
387 cft->read_u64 = hugetlb_cgroup_read_u64;
388
389 /* NULL terminate the last cft */
390 cft = &h->cgroup_files[4];
391 memset(cft, 0, sizeof(*cft));
392
393 WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys,
394 h->cgroup_files));
395}
396
397void __init hugetlb_cgroup_file_init(void)
398{
399 struct hstate *h;
400
401 for_each_hstate(h) {
402 /*
403 * Add cgroup control files only if the huge page consists
404 * of more than two normal pages. This is because we use
405 * page[2].private for storing cgroup details.
406 */
407 if (huge_page_order(h) >= HUGETLB_CGROUP_MIN_ORDER)
408 __hugetlb_cgroup_file_init(hstate_index(h));
409 }
410}
411
412/*
413 * hugetlb_lock will make sure a parallel cgroup rmdir won't happen
414 * when we migrate hugepages
415 */
416void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage)
417{
418 struct hugetlb_cgroup *h_cg;
419 struct hstate *h = page_hstate(oldhpage);
420
421 if (hugetlb_cgroup_disabled())
422 return;
423
424 VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage);
425 spin_lock(&hugetlb_lock);
426 h_cg = hugetlb_cgroup_from_page(oldhpage);
427 set_hugetlb_cgroup(oldhpage, NULL);
428
429 /* move the h_cg details to new cgroup */
430 set_hugetlb_cgroup(newhpage, h_cg);
431 list_move(&newhpage->lru, &h->hugepage_activelist);
432 spin_unlock(&hugetlb_lock);
433 return;
434}
435
436struct cgroup_subsys hugetlb_cgrp_subsys = {
437 .css_alloc = hugetlb_cgroup_css_alloc,
438 .css_offline = hugetlb_cgroup_css_offline,
439 .css_free = hugetlb_cgroup_css_free,
440};
1/*
2 *
3 * Copyright IBM Corporation, 2012
4 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
5 *
6 * Cgroup v2
7 * Copyright (C) 2019 Red Hat, Inc.
8 * Author: Giuseppe Scrivano <gscrivan@redhat.com>
9 *
10 * This program is free software; you can redistribute it and/or modify it
11 * under the terms of version 2.1 of the GNU Lesser General Public License
12 * as published by the Free Software Foundation.
13 *
14 * This program is distributed in the hope that it would be useful, but
15 * WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
17 *
18 */
19
20#include <linux/cgroup.h>
21#include <linux/page_counter.h>
22#include <linux/slab.h>
23#include <linux/hugetlb.h>
24#include <linux/hugetlb_cgroup.h>
25
26#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
27#define MEMFILE_IDX(val) (((val) >> 16) & 0xffff)
28#define MEMFILE_ATTR(val) ((val) & 0xffff)
29
30#define hugetlb_cgroup_from_counter(counter, idx) \
31 container_of(counter, struct hugetlb_cgroup, hugepage[idx])
32
33static struct hugetlb_cgroup *root_h_cgroup __read_mostly;
34
35static inline struct page_counter *
36__hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx,
37 bool rsvd)
38{
39 if (rsvd)
40 return &h_cg->rsvd_hugepage[idx];
41 return &h_cg->hugepage[idx];
42}
43
44static inline struct page_counter *
45hugetlb_cgroup_counter_from_cgroup(struct hugetlb_cgroup *h_cg, int idx)
46{
47 return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, false);
48}
49
50static inline struct page_counter *
51hugetlb_cgroup_counter_from_cgroup_rsvd(struct hugetlb_cgroup *h_cg, int idx)
52{
53 return __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, true);
54}
55
56static inline
57struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s)
58{
59 return s ? container_of(s, struct hugetlb_cgroup, css) : NULL;
60}
61
62static inline
63struct hugetlb_cgroup *hugetlb_cgroup_from_task(struct task_struct *task)
64{
65 return hugetlb_cgroup_from_css(task_css(task, hugetlb_cgrp_id));
66}
67
68static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg)
69{
70 return (h_cg == root_h_cgroup);
71}
72
73static inline struct hugetlb_cgroup *
74parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg)
75{
76 return hugetlb_cgroup_from_css(h_cg->css.parent);
77}
78
79static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg)
80{
81 int idx;
82
83 for (idx = 0; idx < hugetlb_max_hstate; idx++) {
84 if (page_counter_read(
85 hugetlb_cgroup_counter_from_cgroup(h_cg, idx)) ||
86 page_counter_read(hugetlb_cgroup_counter_from_cgroup_rsvd(
87 h_cg, idx))) {
88 return true;
89 }
90 }
91 return false;
92}
93
94static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup,
95 struct hugetlb_cgroup *parent_h_cgroup)
96{
97 int idx;
98
99 for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) {
100 struct page_counter *fault_parent = NULL;
101 struct page_counter *rsvd_parent = NULL;
102 unsigned long limit;
103 int ret;
104
105 if (parent_h_cgroup) {
106 fault_parent = hugetlb_cgroup_counter_from_cgroup(
107 parent_h_cgroup, idx);
108 rsvd_parent = hugetlb_cgroup_counter_from_cgroup_rsvd(
109 parent_h_cgroup, idx);
110 }
111 page_counter_init(hugetlb_cgroup_counter_from_cgroup(h_cgroup,
112 idx),
113 fault_parent);
114 page_counter_init(
115 hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx),
116 rsvd_parent);
117
118 limit = round_down(PAGE_COUNTER_MAX,
119 1 << huge_page_order(&hstates[idx]));
120
121 ret = page_counter_set_max(
122 hugetlb_cgroup_counter_from_cgroup(h_cgroup, idx),
123 limit);
124 VM_BUG_ON(ret);
125 ret = page_counter_set_max(
126 hugetlb_cgroup_counter_from_cgroup_rsvd(h_cgroup, idx),
127 limit);
128 VM_BUG_ON(ret);
129 }
130}
131
132static struct cgroup_subsys_state *
133hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
134{
135 struct hugetlb_cgroup *parent_h_cgroup = hugetlb_cgroup_from_css(parent_css);
136 struct hugetlb_cgroup *h_cgroup;
137
138 h_cgroup = kzalloc(sizeof(*h_cgroup), GFP_KERNEL);
139 if (!h_cgroup)
140 return ERR_PTR(-ENOMEM);
141
142 if (!parent_h_cgroup)
143 root_h_cgroup = h_cgroup;
144
145 hugetlb_cgroup_init(h_cgroup, parent_h_cgroup);
146 return &h_cgroup->css;
147}
148
149static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css)
150{
151 struct hugetlb_cgroup *h_cgroup;
152
153 h_cgroup = hugetlb_cgroup_from_css(css);
154 kfree(h_cgroup);
155}
156
157/*
158 * Should be called with hugetlb_lock held.
159 * Since we are holding hugetlb_lock, pages cannot get moved from
160 * active list or uncharged from the cgroup, So no need to get
161 * page reference and test for page active here. This function
162 * cannot fail.
163 */
164static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg,
165 struct page *page)
166{
167 unsigned int nr_pages;
168 struct page_counter *counter;
169 struct hugetlb_cgroup *page_hcg;
170 struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg);
171
172 page_hcg = hugetlb_cgroup_from_page(page);
173 /*
174 * We can have pages in active list without any cgroup
175 * ie, hugepage with less than 3 pages. We can safely
176 * ignore those pages.
177 */
178 if (!page_hcg || page_hcg != h_cg)
179 goto out;
180
181 nr_pages = compound_nr(page);
182 if (!parent) {
183 parent = root_h_cgroup;
184 /* root has no limit */
185 page_counter_charge(&parent->hugepage[idx], nr_pages);
186 }
187 counter = &h_cg->hugepage[idx];
188 /* Take the pages off the local counter */
189 page_counter_cancel(counter, nr_pages);
190
191 set_hugetlb_cgroup(page, parent);
192out:
193 return;
194}
195
196/*
197 * Force the hugetlb cgroup to empty the hugetlb resources by moving them to
198 * the parent cgroup.
199 */
200static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css)
201{
202 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
203 struct hstate *h;
204 struct page *page;
205 int idx = 0;
206
207 do {
208 for_each_hstate(h) {
209 spin_lock(&hugetlb_lock);
210 list_for_each_entry(page, &h->hugepage_activelist, lru)
211 hugetlb_cgroup_move_parent(idx, h_cg, page);
212
213 spin_unlock(&hugetlb_lock);
214 idx++;
215 }
216 cond_resched();
217 } while (hugetlb_cgroup_have_usage(h_cg));
218}
219
220static inline void hugetlb_event(struct hugetlb_cgroup *hugetlb, int idx,
221 enum hugetlb_memory_event event)
222{
223 atomic_long_inc(&hugetlb->events_local[idx][event]);
224 cgroup_file_notify(&hugetlb->events_local_file[idx]);
225
226 do {
227 atomic_long_inc(&hugetlb->events[idx][event]);
228 cgroup_file_notify(&hugetlb->events_file[idx]);
229 } while ((hugetlb = parent_hugetlb_cgroup(hugetlb)) &&
230 !hugetlb_cgroup_is_root(hugetlb));
231}
232
233static int __hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
234 struct hugetlb_cgroup **ptr,
235 bool rsvd)
236{
237 int ret = 0;
238 struct page_counter *counter;
239 struct hugetlb_cgroup *h_cg = NULL;
240
241 if (hugetlb_cgroup_disabled())
242 goto done;
243 /*
244 * We don't charge any cgroup if the compound page have less
245 * than 3 pages.
246 */
247 if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER)
248 goto done;
249again:
250 rcu_read_lock();
251 h_cg = hugetlb_cgroup_from_task(current);
252 if (!css_tryget(&h_cg->css)) {
253 rcu_read_unlock();
254 goto again;
255 }
256 rcu_read_unlock();
257
258 if (!page_counter_try_charge(
259 __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),
260 nr_pages, &counter)) {
261 ret = -ENOMEM;
262 hugetlb_event(h_cg, idx, HUGETLB_MAX);
263 css_put(&h_cg->css);
264 goto done;
265 }
266 /* Reservations take a reference to the css because they do not get
267 * reparented.
268 */
269 if (!rsvd)
270 css_put(&h_cg->css);
271done:
272 *ptr = h_cg;
273 return ret;
274}
275
276int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages,
277 struct hugetlb_cgroup **ptr)
278{
279 return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, false);
280}
281
282int hugetlb_cgroup_charge_cgroup_rsvd(int idx, unsigned long nr_pages,
283 struct hugetlb_cgroup **ptr)
284{
285 return __hugetlb_cgroup_charge_cgroup(idx, nr_pages, ptr, true);
286}
287
288/* Should be called with hugetlb_lock held */
289static void __hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
290 struct hugetlb_cgroup *h_cg,
291 struct page *page, bool rsvd)
292{
293 if (hugetlb_cgroup_disabled() || !h_cg)
294 return;
295
296 __set_hugetlb_cgroup(page, h_cg, rsvd);
297 return;
298}
299
300void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages,
301 struct hugetlb_cgroup *h_cg,
302 struct page *page)
303{
304 __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, false);
305}
306
307void hugetlb_cgroup_commit_charge_rsvd(int idx, unsigned long nr_pages,
308 struct hugetlb_cgroup *h_cg,
309 struct page *page)
310{
311 __hugetlb_cgroup_commit_charge(idx, nr_pages, h_cg, page, true);
312}
313
314/*
315 * Should be called with hugetlb_lock held
316 */
317static void __hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
318 struct page *page, bool rsvd)
319{
320 struct hugetlb_cgroup *h_cg;
321
322 if (hugetlb_cgroup_disabled())
323 return;
324 lockdep_assert_held(&hugetlb_lock);
325 h_cg = __hugetlb_cgroup_from_page(page, rsvd);
326 if (unlikely(!h_cg))
327 return;
328 __set_hugetlb_cgroup(page, NULL, rsvd);
329
330 page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,
331 rsvd),
332 nr_pages);
333
334 if (rsvd)
335 css_put(&h_cg->css);
336
337 return;
338}
339
340void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages,
341 struct page *page)
342{
343 __hugetlb_cgroup_uncharge_page(idx, nr_pages, page, false);
344}
345
346void hugetlb_cgroup_uncharge_page_rsvd(int idx, unsigned long nr_pages,
347 struct page *page)
348{
349 __hugetlb_cgroup_uncharge_page(idx, nr_pages, page, true);
350}
351
352static void __hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
353 struct hugetlb_cgroup *h_cg,
354 bool rsvd)
355{
356 if (hugetlb_cgroup_disabled() || !h_cg)
357 return;
358
359 if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER)
360 return;
361
362 page_counter_uncharge(__hugetlb_cgroup_counter_from_cgroup(h_cg, idx,
363 rsvd),
364 nr_pages);
365
366 if (rsvd)
367 css_put(&h_cg->css);
368}
369
370void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages,
371 struct hugetlb_cgroup *h_cg)
372{
373 __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, false);
374}
375
376void hugetlb_cgroup_uncharge_cgroup_rsvd(int idx, unsigned long nr_pages,
377 struct hugetlb_cgroup *h_cg)
378{
379 __hugetlb_cgroup_uncharge_cgroup(idx, nr_pages, h_cg, true);
380}
381
382void hugetlb_cgroup_uncharge_counter(struct resv_map *resv, unsigned long start,
383 unsigned long end)
384{
385 if (hugetlb_cgroup_disabled() || !resv || !resv->reservation_counter ||
386 !resv->css)
387 return;
388
389 page_counter_uncharge(resv->reservation_counter,
390 (end - start) * resv->pages_per_hpage);
391 css_put(resv->css);
392}
393
394void hugetlb_cgroup_uncharge_file_region(struct resv_map *resv,
395 struct file_region *rg,
396 unsigned long nr_pages)
397{
398 if (hugetlb_cgroup_disabled() || !resv || !rg || !nr_pages)
399 return;
400
401 if (rg->reservation_counter && resv->pages_per_hpage && nr_pages > 0 &&
402 !resv->reservation_counter) {
403 page_counter_uncharge(rg->reservation_counter,
404 nr_pages * resv->pages_per_hpage);
405 css_put(rg->css);
406 }
407}
408
409enum {
410 RES_USAGE,
411 RES_RSVD_USAGE,
412 RES_LIMIT,
413 RES_RSVD_LIMIT,
414 RES_MAX_USAGE,
415 RES_RSVD_MAX_USAGE,
416 RES_FAILCNT,
417 RES_RSVD_FAILCNT,
418};
419
420static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css,
421 struct cftype *cft)
422{
423 struct page_counter *counter;
424 struct page_counter *rsvd_counter;
425 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css);
426
427 counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)];
428 rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(cft->private)];
429
430 switch (MEMFILE_ATTR(cft->private)) {
431 case RES_USAGE:
432 return (u64)page_counter_read(counter) * PAGE_SIZE;
433 case RES_RSVD_USAGE:
434 return (u64)page_counter_read(rsvd_counter) * PAGE_SIZE;
435 case RES_LIMIT:
436 return (u64)counter->max * PAGE_SIZE;
437 case RES_RSVD_LIMIT:
438 return (u64)rsvd_counter->max * PAGE_SIZE;
439 case RES_MAX_USAGE:
440 return (u64)counter->watermark * PAGE_SIZE;
441 case RES_RSVD_MAX_USAGE:
442 return (u64)rsvd_counter->watermark * PAGE_SIZE;
443 case RES_FAILCNT:
444 return counter->failcnt;
445 case RES_RSVD_FAILCNT:
446 return rsvd_counter->failcnt;
447 default:
448 BUG();
449 }
450}
451
452static int hugetlb_cgroup_read_u64_max(struct seq_file *seq, void *v)
453{
454 int idx;
455 u64 val;
456 struct cftype *cft = seq_cft(seq);
457 unsigned long limit;
458 struct page_counter *counter;
459 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));
460
461 idx = MEMFILE_IDX(cft->private);
462 counter = &h_cg->hugepage[idx];
463
464 limit = round_down(PAGE_COUNTER_MAX,
465 1 << huge_page_order(&hstates[idx]));
466
467 switch (MEMFILE_ATTR(cft->private)) {
468 case RES_RSVD_USAGE:
469 counter = &h_cg->rsvd_hugepage[idx];
470 fallthrough;
471 case RES_USAGE:
472 val = (u64)page_counter_read(counter);
473 seq_printf(seq, "%llu\n", val * PAGE_SIZE);
474 break;
475 case RES_RSVD_LIMIT:
476 counter = &h_cg->rsvd_hugepage[idx];
477 fallthrough;
478 case RES_LIMIT:
479 val = (u64)counter->max;
480 if (val == limit)
481 seq_puts(seq, "max\n");
482 else
483 seq_printf(seq, "%llu\n", val * PAGE_SIZE);
484 break;
485 default:
486 BUG();
487 }
488
489 return 0;
490}
491
492static DEFINE_MUTEX(hugetlb_limit_mutex);
493
494static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of,
495 char *buf, size_t nbytes, loff_t off,
496 const char *max)
497{
498 int ret, idx;
499 unsigned long nr_pages;
500 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
501 bool rsvd = false;
502
503 if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */
504 return -EINVAL;
505
506 buf = strstrip(buf);
507 ret = page_counter_memparse(buf, max, &nr_pages);
508 if (ret)
509 return ret;
510
511 idx = MEMFILE_IDX(of_cft(of)->private);
512 nr_pages = round_down(nr_pages, 1 << huge_page_order(&hstates[idx]));
513
514 switch (MEMFILE_ATTR(of_cft(of)->private)) {
515 case RES_RSVD_LIMIT:
516 rsvd = true;
517 fallthrough;
518 case RES_LIMIT:
519 mutex_lock(&hugetlb_limit_mutex);
520 ret = page_counter_set_max(
521 __hugetlb_cgroup_counter_from_cgroup(h_cg, idx, rsvd),
522 nr_pages);
523 mutex_unlock(&hugetlb_limit_mutex);
524 break;
525 default:
526 ret = -EINVAL;
527 break;
528 }
529 return ret ?: nbytes;
530}
531
532static ssize_t hugetlb_cgroup_write_legacy(struct kernfs_open_file *of,
533 char *buf, size_t nbytes, loff_t off)
534{
535 return hugetlb_cgroup_write(of, buf, nbytes, off, "-1");
536}
537
538static ssize_t hugetlb_cgroup_write_dfl(struct kernfs_open_file *of,
539 char *buf, size_t nbytes, loff_t off)
540{
541 return hugetlb_cgroup_write(of, buf, nbytes, off, "max");
542}
543
544static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of,
545 char *buf, size_t nbytes, loff_t off)
546{
547 int ret = 0;
548 struct page_counter *counter, *rsvd_counter;
549 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of));
550
551 counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)];
552 rsvd_counter = &h_cg->rsvd_hugepage[MEMFILE_IDX(of_cft(of)->private)];
553
554 switch (MEMFILE_ATTR(of_cft(of)->private)) {
555 case RES_MAX_USAGE:
556 page_counter_reset_watermark(counter);
557 break;
558 case RES_RSVD_MAX_USAGE:
559 page_counter_reset_watermark(rsvd_counter);
560 break;
561 case RES_FAILCNT:
562 counter->failcnt = 0;
563 break;
564 case RES_RSVD_FAILCNT:
565 rsvd_counter->failcnt = 0;
566 break;
567 default:
568 ret = -EINVAL;
569 break;
570 }
571 return ret ?: nbytes;
572}
573
574static char *mem_fmt(char *buf, int size, unsigned long hsize)
575{
576 if (hsize >= (1UL << 30))
577 snprintf(buf, size, "%luGB", hsize >> 30);
578 else if (hsize >= (1UL << 20))
579 snprintf(buf, size, "%luMB", hsize >> 20);
580 else
581 snprintf(buf, size, "%luKB", hsize >> 10);
582 return buf;
583}
584
585static int __hugetlb_events_show(struct seq_file *seq, bool local)
586{
587 int idx;
588 long max;
589 struct cftype *cft = seq_cft(seq);
590 struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(seq_css(seq));
591
592 idx = MEMFILE_IDX(cft->private);
593
594 if (local)
595 max = atomic_long_read(&h_cg->events_local[idx][HUGETLB_MAX]);
596 else
597 max = atomic_long_read(&h_cg->events[idx][HUGETLB_MAX]);
598
599 seq_printf(seq, "max %lu\n", max);
600
601 return 0;
602}
603
604static int hugetlb_events_show(struct seq_file *seq, void *v)
605{
606 return __hugetlb_events_show(seq, false);
607}
608
609static int hugetlb_events_local_show(struct seq_file *seq, void *v)
610{
611 return __hugetlb_events_show(seq, true);
612}
613
614static void __init __hugetlb_cgroup_file_dfl_init(int idx)
615{
616 char buf[32];
617 struct cftype *cft;
618 struct hstate *h = &hstates[idx];
619
620 /* format the size */
621 mem_fmt(buf, sizeof(buf), huge_page_size(h));
622
623 /* Add the limit file */
624 cft = &h->cgroup_files_dfl[0];
625 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max", buf);
626 cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);
627 cft->seq_show = hugetlb_cgroup_read_u64_max;
628 cft->write = hugetlb_cgroup_write_dfl;
629 cft->flags = CFTYPE_NOT_ON_ROOT;
630
631 /* Add the reservation limit file */
632 cft = &h->cgroup_files_dfl[1];
633 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.max", buf);
634 cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT);
635 cft->seq_show = hugetlb_cgroup_read_u64_max;
636 cft->write = hugetlb_cgroup_write_dfl;
637 cft->flags = CFTYPE_NOT_ON_ROOT;
638
639 /* Add the current usage file */
640 cft = &h->cgroup_files_dfl[2];
641 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.current", buf);
642 cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);
643 cft->seq_show = hugetlb_cgroup_read_u64_max;
644 cft->flags = CFTYPE_NOT_ON_ROOT;
645
646 /* Add the current reservation usage file */
647 cft = &h->cgroup_files_dfl[3];
648 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.current", buf);
649 cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE);
650 cft->seq_show = hugetlb_cgroup_read_u64_max;
651 cft->flags = CFTYPE_NOT_ON_ROOT;
652
653 /* Add the events file */
654 cft = &h->cgroup_files_dfl[4];
655 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events", buf);
656 cft->private = MEMFILE_PRIVATE(idx, 0);
657 cft->seq_show = hugetlb_events_show;
658 cft->file_offset = offsetof(struct hugetlb_cgroup, events_file[idx]);
659 cft->flags = CFTYPE_NOT_ON_ROOT;
660
661 /* Add the events.local file */
662 cft = &h->cgroup_files_dfl[5];
663 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.events.local", buf);
664 cft->private = MEMFILE_PRIVATE(idx, 0);
665 cft->seq_show = hugetlb_events_local_show;
666 cft->file_offset = offsetof(struct hugetlb_cgroup,
667 events_local_file[idx]);
668 cft->flags = CFTYPE_NOT_ON_ROOT;
669
670 /* NULL terminate the last cft */
671 cft = &h->cgroup_files_dfl[6];
672 memset(cft, 0, sizeof(*cft));
673
674 WARN_ON(cgroup_add_dfl_cftypes(&hugetlb_cgrp_subsys,
675 h->cgroup_files_dfl));
676}
677
678static void __init __hugetlb_cgroup_file_legacy_init(int idx)
679{
680 char buf[32];
681 struct cftype *cft;
682 struct hstate *h = &hstates[idx];
683
684 /* format the size */
685 mem_fmt(buf, sizeof(buf), huge_page_size(h));
686
687 /* Add the limit file */
688 cft = &h->cgroup_files_legacy[0];
689 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf);
690 cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT);
691 cft->read_u64 = hugetlb_cgroup_read_u64;
692 cft->write = hugetlb_cgroup_write_legacy;
693
694 /* Add the reservation limit file */
695 cft = &h->cgroup_files_legacy[1];
696 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.limit_in_bytes", buf);
697 cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_LIMIT);
698 cft->read_u64 = hugetlb_cgroup_read_u64;
699 cft->write = hugetlb_cgroup_write_legacy;
700
701 /* Add the usage file */
702 cft = &h->cgroup_files_legacy[2];
703 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf);
704 cft->private = MEMFILE_PRIVATE(idx, RES_USAGE);
705 cft->read_u64 = hugetlb_cgroup_read_u64;
706
707 /* Add the reservation usage file */
708 cft = &h->cgroup_files_legacy[3];
709 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.usage_in_bytes", buf);
710 cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_USAGE);
711 cft->read_u64 = hugetlb_cgroup_read_u64;
712
713 /* Add the MAX usage file */
714 cft = &h->cgroup_files_legacy[4];
715 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf);
716 cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE);
717 cft->write = hugetlb_cgroup_reset;
718 cft->read_u64 = hugetlb_cgroup_read_u64;
719
720 /* Add the MAX reservation usage file */
721 cft = &h->cgroup_files_legacy[5];
722 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.max_usage_in_bytes", buf);
723 cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_MAX_USAGE);
724 cft->write = hugetlb_cgroup_reset;
725 cft->read_u64 = hugetlb_cgroup_read_u64;
726
727 /* Add the failcntfile */
728 cft = &h->cgroup_files_legacy[6];
729 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf);
730 cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT);
731 cft->write = hugetlb_cgroup_reset;
732 cft->read_u64 = hugetlb_cgroup_read_u64;
733
734 /* Add the reservation failcntfile */
735 cft = &h->cgroup_files_legacy[7];
736 snprintf(cft->name, MAX_CFTYPE_NAME, "%s.rsvd.failcnt", buf);
737 cft->private = MEMFILE_PRIVATE(idx, RES_RSVD_FAILCNT);
738 cft->write = hugetlb_cgroup_reset;
739 cft->read_u64 = hugetlb_cgroup_read_u64;
740
741 /* NULL terminate the last cft */
742 cft = &h->cgroup_files_legacy[8];
743 memset(cft, 0, sizeof(*cft));
744
745 WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys,
746 h->cgroup_files_legacy));
747}
748
749static void __init __hugetlb_cgroup_file_init(int idx)
750{
751 __hugetlb_cgroup_file_dfl_init(idx);
752 __hugetlb_cgroup_file_legacy_init(idx);
753}
754
755void __init hugetlb_cgroup_file_init(void)
756{
757 struct hstate *h;
758
759 for_each_hstate(h) {
760 /*
761 * Add cgroup control files only if the huge page consists
762 * of more than two normal pages. This is because we use
763 * page[2].private for storing cgroup details.
764 */
765 if (huge_page_order(h) >= HUGETLB_CGROUP_MIN_ORDER)
766 __hugetlb_cgroup_file_init(hstate_index(h));
767 }
768}
769
770/*
771 * hugetlb_lock will make sure a parallel cgroup rmdir won't happen
772 * when we migrate hugepages
773 */
774void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage)
775{
776 struct hugetlb_cgroup *h_cg;
777 struct hugetlb_cgroup *h_cg_rsvd;
778 struct hstate *h = page_hstate(oldhpage);
779
780 if (hugetlb_cgroup_disabled())
781 return;
782
783 VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage);
784 spin_lock(&hugetlb_lock);
785 h_cg = hugetlb_cgroup_from_page(oldhpage);
786 h_cg_rsvd = hugetlb_cgroup_from_page_rsvd(oldhpage);
787 set_hugetlb_cgroup(oldhpage, NULL);
788 set_hugetlb_cgroup_rsvd(oldhpage, NULL);
789
790 /* move the h_cg details to new cgroup */
791 set_hugetlb_cgroup(newhpage, h_cg);
792 set_hugetlb_cgroup_rsvd(newhpage, h_cg_rsvd);
793 list_move(&newhpage->lru, &h->hugepage_activelist);
794 spin_unlock(&hugetlb_lock);
795 return;
796}
797
798static struct cftype hugetlb_files[] = {
799 {} /* terminate */
800};
801
802struct cgroup_subsys hugetlb_cgrp_subsys = {
803 .css_alloc = hugetlb_cgroup_css_alloc,
804 .css_offline = hugetlb_cgroup_css_offline,
805 .css_free = hugetlb_cgroup_css_free,
806 .dfl_cftypes = hugetlb_files,
807 .legacy_cftypes = hugetlb_files,
808};