Loading...
1#include <linux/swap_cgroup.h>
2#include <linux/vmalloc.h>
3#include <linux/mm.h>
4
5#include <linux/swapops.h> /* depends on mm.h include */
6
7static DEFINE_MUTEX(swap_cgroup_mutex);
8struct swap_cgroup_ctrl {
9 struct page **map;
10 unsigned long length;
11 spinlock_t lock;
12};
13
14static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
15
16struct swap_cgroup {
17 unsigned short id;
18};
19#define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
20
21/*
22 * SwapCgroup implements "lookup" and "exchange" operations.
23 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
24 * against SwapCache. At swap_free(), this is accessed directly from swap.
25 *
26 * This means,
27 * - we have no race in "exchange" when we're accessed via SwapCache because
28 * SwapCache(and its swp_entry) is under lock.
29 * - When called via swap_free(), there is no user of this entry and no race.
30 * Then, we don't need lock around "exchange".
31 *
32 * TODO: we can push these buffers out to HIGHMEM.
33 */
34
35/*
36 * allocate buffer for swap_cgroup.
37 */
38static int swap_cgroup_prepare(int type)
39{
40 struct page *page;
41 struct swap_cgroup_ctrl *ctrl;
42 unsigned long idx, max;
43
44 ctrl = &swap_cgroup_ctrl[type];
45
46 for (idx = 0; idx < ctrl->length; idx++) {
47 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
48 if (!page)
49 goto not_enough_page;
50 ctrl->map[idx] = page;
51 }
52 return 0;
53not_enough_page:
54 max = idx;
55 for (idx = 0; idx < max; idx++)
56 __free_page(ctrl->map[idx]);
57
58 return -ENOMEM;
59}
60
61static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
62 struct swap_cgroup_ctrl **ctrlp)
63{
64 pgoff_t offset = swp_offset(ent);
65 struct swap_cgroup_ctrl *ctrl;
66 struct page *mappage;
67 struct swap_cgroup *sc;
68
69 ctrl = &swap_cgroup_ctrl[swp_type(ent)];
70 if (ctrlp)
71 *ctrlp = ctrl;
72
73 mappage = ctrl->map[offset / SC_PER_PAGE];
74 sc = page_address(mappage);
75 return sc + offset % SC_PER_PAGE;
76}
77
78/**
79 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
80 * @ent: swap entry to be cmpxchged
81 * @old: old id
82 * @new: new id
83 *
84 * Returns old id at success, 0 at failure.
85 * (There is no mem_cgroup using 0 as its id)
86 */
87unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
88 unsigned short old, unsigned short new)
89{
90 struct swap_cgroup_ctrl *ctrl;
91 struct swap_cgroup *sc;
92 unsigned long flags;
93 unsigned short retval;
94
95 sc = lookup_swap_cgroup(ent, &ctrl);
96
97 spin_lock_irqsave(&ctrl->lock, flags);
98 retval = sc->id;
99 if (retval == old)
100 sc->id = new;
101 else
102 retval = 0;
103 spin_unlock_irqrestore(&ctrl->lock, flags);
104 return retval;
105}
106
107/**
108 * swap_cgroup_record - record mem_cgroup for this swp_entry.
109 * @ent: swap entry to be recorded into
110 * @id: mem_cgroup to be recorded
111 *
112 * Returns old value at success, 0 at failure.
113 * (Of course, old value can be 0.)
114 */
115unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
116{
117 struct swap_cgroup_ctrl *ctrl;
118 struct swap_cgroup *sc;
119 unsigned short old;
120 unsigned long flags;
121
122 sc = lookup_swap_cgroup(ent, &ctrl);
123
124 spin_lock_irqsave(&ctrl->lock, flags);
125 old = sc->id;
126 sc->id = id;
127 spin_unlock_irqrestore(&ctrl->lock, flags);
128
129 return old;
130}
131
132/**
133 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
134 * @ent: swap entry to be looked up.
135 *
136 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
137 */
138unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
139{
140 return lookup_swap_cgroup(ent, NULL)->id;
141}
142
143int swap_cgroup_swapon(int type, unsigned long max_pages)
144{
145 void *array;
146 unsigned long array_size;
147 unsigned long length;
148 struct swap_cgroup_ctrl *ctrl;
149
150 if (!do_swap_account)
151 return 0;
152
153 length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
154 array_size = length * sizeof(void *);
155
156 array = vzalloc(array_size);
157 if (!array)
158 goto nomem;
159
160 ctrl = &swap_cgroup_ctrl[type];
161 mutex_lock(&swap_cgroup_mutex);
162 ctrl->length = length;
163 ctrl->map = array;
164 spin_lock_init(&ctrl->lock);
165 if (swap_cgroup_prepare(type)) {
166 /* memory shortage */
167 ctrl->map = NULL;
168 ctrl->length = 0;
169 mutex_unlock(&swap_cgroup_mutex);
170 vfree(array);
171 goto nomem;
172 }
173 mutex_unlock(&swap_cgroup_mutex);
174
175 return 0;
176nomem:
177 pr_info("couldn't allocate enough memory for swap_cgroup\n");
178 pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
179 return -ENOMEM;
180}
181
182void swap_cgroup_swapoff(int type)
183{
184 struct page **map;
185 unsigned long i, length;
186 struct swap_cgroup_ctrl *ctrl;
187
188 if (!do_swap_account)
189 return;
190
191 mutex_lock(&swap_cgroup_mutex);
192 ctrl = &swap_cgroup_ctrl[type];
193 map = ctrl->map;
194 length = ctrl->length;
195 ctrl->map = NULL;
196 ctrl->length = 0;
197 mutex_unlock(&swap_cgroup_mutex);
198
199 if (map) {
200 for (i = 0; i < length; i++) {
201 struct page *page = map[i];
202 if (page)
203 __free_page(page);
204 }
205 vfree(map);
206 }
207}
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/swap_cgroup.h>
3#include <linux/vmalloc.h>
4#include <linux/mm.h>
5
6#include <linux/swapops.h> /* depends on mm.h include */
7
8static DEFINE_MUTEX(swap_cgroup_mutex);
9struct swap_cgroup_ctrl {
10 struct page **map;
11 unsigned long length;
12 spinlock_t lock;
13};
14
15static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
16
17struct swap_cgroup {
18 unsigned short id;
19};
20#define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup))
21
22/*
23 * SwapCgroup implements "lookup" and "exchange" operations.
24 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
25 * against SwapCache. At swap_free(), this is accessed directly from swap.
26 *
27 * This means,
28 * - we have no race in "exchange" when we're accessed via SwapCache because
29 * SwapCache(and its swp_entry) is under lock.
30 * - When called via swap_free(), there is no user of this entry and no race.
31 * Then, we don't need lock around "exchange".
32 *
33 * TODO: we can push these buffers out to HIGHMEM.
34 */
35
36/*
37 * allocate buffer for swap_cgroup.
38 */
39static int swap_cgroup_prepare(int type)
40{
41 struct page *page;
42 struct swap_cgroup_ctrl *ctrl;
43 unsigned long idx, max;
44
45 ctrl = &swap_cgroup_ctrl[type];
46
47 for (idx = 0; idx < ctrl->length; idx++) {
48 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
49 if (!page)
50 goto not_enough_page;
51 ctrl->map[idx] = page;
52
53 if (!(idx % SWAP_CLUSTER_MAX))
54 cond_resched();
55 }
56 return 0;
57not_enough_page:
58 max = idx;
59 for (idx = 0; idx < max; idx++)
60 __free_page(ctrl->map[idx]);
61
62 return -ENOMEM;
63}
64
65static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl,
66 pgoff_t offset)
67{
68 struct page *mappage;
69 struct swap_cgroup *sc;
70
71 mappage = ctrl->map[offset / SC_PER_PAGE];
72 sc = page_address(mappage);
73 return sc + offset % SC_PER_PAGE;
74}
75
76static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
77 struct swap_cgroup_ctrl **ctrlp)
78{
79 pgoff_t offset = swp_offset(ent);
80 struct swap_cgroup_ctrl *ctrl;
81
82 ctrl = &swap_cgroup_ctrl[swp_type(ent)];
83 if (ctrlp)
84 *ctrlp = ctrl;
85 return __lookup_swap_cgroup(ctrl, offset);
86}
87
88/**
89 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
90 * @ent: swap entry to be cmpxchged
91 * @old: old id
92 * @new: new id
93 *
94 * Returns old id at success, 0 at failure.
95 * (There is no mem_cgroup using 0 as its id)
96 */
97unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
98 unsigned short old, unsigned short new)
99{
100 struct swap_cgroup_ctrl *ctrl;
101 struct swap_cgroup *sc;
102 unsigned long flags;
103 unsigned short retval;
104
105 sc = lookup_swap_cgroup(ent, &ctrl);
106
107 spin_lock_irqsave(&ctrl->lock, flags);
108 retval = sc->id;
109 if (retval == old)
110 sc->id = new;
111 else
112 retval = 0;
113 spin_unlock_irqrestore(&ctrl->lock, flags);
114 return retval;
115}
116
117/**
118 * swap_cgroup_record - record mem_cgroup for a set of swap entries
119 * @ent: the first swap entry to be recorded into
120 * @id: mem_cgroup to be recorded
121 * @nr_ents: number of swap entries to be recorded
122 *
123 * Returns old value at success, 0 at failure.
124 * (Of course, old value can be 0.)
125 */
126unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id,
127 unsigned int nr_ents)
128{
129 struct swap_cgroup_ctrl *ctrl;
130 struct swap_cgroup *sc;
131 unsigned short old;
132 unsigned long flags;
133 pgoff_t offset = swp_offset(ent);
134 pgoff_t end = offset + nr_ents;
135
136 sc = lookup_swap_cgroup(ent, &ctrl);
137
138 spin_lock_irqsave(&ctrl->lock, flags);
139 old = sc->id;
140 for (;;) {
141 VM_BUG_ON(sc->id != old);
142 sc->id = id;
143 offset++;
144 if (offset == end)
145 break;
146 if (offset % SC_PER_PAGE)
147 sc++;
148 else
149 sc = __lookup_swap_cgroup(ctrl, offset);
150 }
151 spin_unlock_irqrestore(&ctrl->lock, flags);
152
153 return old;
154}
155
156/**
157 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
158 * @ent: swap entry to be looked up.
159 *
160 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
161 */
162unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
163{
164 return lookup_swap_cgroup(ent, NULL)->id;
165}
166
167int swap_cgroup_swapon(int type, unsigned long max_pages)
168{
169 void *array;
170 unsigned long length;
171 struct swap_cgroup_ctrl *ctrl;
172
173 if (mem_cgroup_disabled())
174 return 0;
175
176 length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
177
178 array = vcalloc(length, sizeof(void *));
179 if (!array)
180 goto nomem;
181
182 ctrl = &swap_cgroup_ctrl[type];
183 mutex_lock(&swap_cgroup_mutex);
184 ctrl->length = length;
185 ctrl->map = array;
186 spin_lock_init(&ctrl->lock);
187 if (swap_cgroup_prepare(type)) {
188 /* memory shortage */
189 ctrl->map = NULL;
190 ctrl->length = 0;
191 mutex_unlock(&swap_cgroup_mutex);
192 vfree(array);
193 goto nomem;
194 }
195 mutex_unlock(&swap_cgroup_mutex);
196
197 return 0;
198nomem:
199 pr_info("couldn't allocate enough memory for swap_cgroup\n");
200 pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
201 return -ENOMEM;
202}
203
204void swap_cgroup_swapoff(int type)
205{
206 struct page **map;
207 unsigned long i, length;
208 struct swap_cgroup_ctrl *ctrl;
209
210 if (mem_cgroup_disabled())
211 return;
212
213 mutex_lock(&swap_cgroup_mutex);
214 ctrl = &swap_cgroup_ctrl[type];
215 map = ctrl->map;
216 length = ctrl->length;
217 ctrl->map = NULL;
218 ctrl->length = 0;
219 mutex_unlock(&swap_cgroup_mutex);
220
221 if (map) {
222 for (i = 0; i < length; i++) {
223 struct page *page = map[i];
224 if (page)
225 __free_page(page);
226 if (!(i % SWAP_CLUSTER_MAX))
227 cond_resched();
228 }
229 vfree(map);
230 }
231}