1#include <linux/swap_cgroup.h>
  2#include <linux/vmalloc.h>
  3#include <linux/mm.h>
  4
  5#include <linux/swapops.h> /* depends on mm.h include */
  6
  7static DEFINE_MUTEX(swap_cgroup_mutex);
  8struct swap_cgroup_ctrl {
  9	struct page **map;
 10	unsigned long length;
 11	spinlock_t	lock;
 12};
 13
 14static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
 15
 16struct swap_cgroup {
 17	unsigned short		id;
 18};
 19#define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))
 20
 21/*
 22 * SwapCgroup implements "lookup" and "exchange" operations.
 23 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
 24 * against SwapCache. At swap_free(), this is accessed directly from swap.
 25 *
 26 * This means,
 27 *  - we have no race in "exchange" when we're accessed via SwapCache because
 28 *    SwapCache(and its swp_entry) is under lock.
 29 *  - When called via swap_free(), there is no user of this entry and no race.
 30 * Then, we don't need lock around "exchange".
 31 *
 32 * TODO: we can push these buffers out to HIGHMEM.
 33 */
 34
 35/*
 36 * allocate buffer for swap_cgroup.
 37 */
 38static int swap_cgroup_prepare(int type)
 39{
 40	struct page *page;
 41	struct swap_cgroup_ctrl *ctrl;
 42	unsigned long idx, max;
 43
 44	ctrl = &swap_cgroup_ctrl[type];
 45
 46	for (idx = 0; idx < ctrl->length; idx++) {
 47		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 48		if (!page)
 49			goto not_enough_page;
 50		ctrl->map[idx] = page;
 51	}
 52	return 0;
 53not_enough_page:
 54	max = idx;
 55	for (idx = 0; idx < max; idx++)
 56		__free_page(ctrl->map[idx]);
 57
 58	return -ENOMEM;
 59}
 60
 61static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
 62					struct swap_cgroup_ctrl **ctrlp)
 63{
 64	pgoff_t offset = swp_offset(ent);
 65	struct swap_cgroup_ctrl *ctrl;
 66	struct page *mappage;
 67	struct swap_cgroup *sc;
 68
 69	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
 70	if (ctrlp)
 71		*ctrlp = ctrl;
 72
 73	mappage = ctrl->map[offset / SC_PER_PAGE];
 74	sc = page_address(mappage);
 75	return sc + offset % SC_PER_PAGE;
 76}
 77
 78/**
 79 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
 80 * @ent: swap entry to be cmpxchged
 81 * @old: old id
 82 * @new: new id
 83 *
 84 * Returns old id at success, 0 at failure.
 85 * (There is no mem_cgroup using 0 as its id)
 86 */
 87unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
 88					unsigned short old, unsigned short new)
 89{
 90	struct swap_cgroup_ctrl *ctrl;
 91	struct swap_cgroup *sc;
 92	unsigned long flags;
 93	unsigned short retval;
 94
 95	sc = lookup_swap_cgroup(ent, &ctrl);
 96
 97	spin_lock_irqsave(&ctrl->lock, flags);
 98	retval = sc->id;
 99	if (retval == old)
100		sc->id = new;
101	else
102		retval = 0;
103	spin_unlock_irqrestore(&ctrl->lock, flags);
104	return retval;
105}
106
107/**
108 * swap_cgroup_record - record mem_cgroup for this swp_entry.
109 * @ent: swap entry to be recorded into
110 * @id: mem_cgroup to be recorded
111 *
112 * Returns old value at success, 0 at failure.
113 * (Of course, old value can be 0.)
114 */
115unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
116{
117	struct swap_cgroup_ctrl *ctrl;
118	struct swap_cgroup *sc;
119	unsigned short old;
120	unsigned long flags;
121
122	sc = lookup_swap_cgroup(ent, &ctrl);
123
124	spin_lock_irqsave(&ctrl->lock, flags);
125	old = sc->id;
126	sc->id = id;
127	spin_unlock_irqrestore(&ctrl->lock, flags);
128
129	return old;
130}
131
132/**
133 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
134 * @ent: swap entry to be looked up.
135 *
136 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
137 */
138unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
139{
140	return lookup_swap_cgroup(ent, NULL)->id;
141}
142
143int swap_cgroup_swapon(int type, unsigned long max_pages)
144{
145	void *array;
146	unsigned long array_size;
147	unsigned long length;
148	struct swap_cgroup_ctrl *ctrl;
149
150	if (!do_swap_account)
151		return 0;
152
153	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
154	array_size = length * sizeof(void *);
155
156	array = vzalloc(array_size);
157	if (!array)
158		goto nomem;
159
160	ctrl = &swap_cgroup_ctrl[type];
161	mutex_lock(&swap_cgroup_mutex);
162	ctrl->length = length;
163	ctrl->map = array;
164	spin_lock_init(&ctrl->lock);
165	if (swap_cgroup_prepare(type)) {
166		/* memory shortage */
167		ctrl->map = NULL;
168		ctrl->length = 0;
169		mutex_unlock(&swap_cgroup_mutex);
170		vfree(array);
171		goto nomem;
172	}
173	mutex_unlock(&swap_cgroup_mutex);
174
175	return 0;
176nomem:
177	pr_info("couldn't allocate enough memory for swap_cgroup\n");
178	pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n");
179	return -ENOMEM;
180}
181
182void swap_cgroup_swapoff(int type)
183{
184	struct page **map;
185	unsigned long i, length;
186	struct swap_cgroup_ctrl *ctrl;
187
188	if (!do_swap_account)
189		return;
190
191	mutex_lock(&swap_cgroup_mutex);
192	ctrl = &swap_cgroup_ctrl[type];
193	map = ctrl->map;
194	length = ctrl->length;
195	ctrl->map = NULL;
196	ctrl->length = 0;
197	mutex_unlock(&swap_cgroup_mutex);
198
199	if (map) {
200		for (i = 0; i < length; i++) {
201			struct page *page = map[i];
202			if (page)
203				__free_page(page);
204		}
205		vfree(map);
206	}
207}