1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Manage cache of swap slots to be used for and returned from
  4 * swap.
  5 *
  6 * Copyright(c) 2016 Intel Corporation.
  7 *
  8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
  9 *
 10 * We allocate the swap slots from the global pool and put
 11 * it into local per cpu caches.  This has the advantage
 12 * of no needing to acquire the swap_info lock every time
 13 * we need a new slot.
 14 *
 15 * There is also opportunity to simply return the slot
 16 * to local caches without needing to acquire swap_info
 17 * lock.  We do not reuse the returned slots directly but
 18 * move them back to the global pool in a batch.  This
 19 * allows the slots to coaellesce and reduce fragmentation.
 20 *
 21 * The swap entry allocated is marked with SWAP_HAS_CACHE
 22 * flag in map_count that prevents it from being allocated
 23 * again from the global pool.
 24 *
 25 * The swap slots cache is protected by a mutex instead of
 26 * a spin lock as when we search for slots with scan_swap_map,
 27 * we can possibly sleep.
 28 */
 29
 30#include <linux/swap_slots.h>
 31#include <linux/cpu.h>
 32#include <linux/cpumask.h>
 33#include <linux/vmalloc.h>
 34#include <linux/mutex.h>
 35#include <linux/mm.h>
 36
 37static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
 38static bool	swap_slot_cache_active;
 39bool	swap_slot_cache_enabled;
 40static bool	swap_slot_cache_initialized;
 41static DEFINE_MUTEX(swap_slots_cache_mutex);
 42/* Serialize swap slots cache enable/disable operations */
 43static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
 44
 45static void __drain_swap_slots_cache(unsigned int type);
 46static void deactivate_swap_slots_cache(void);
 47static void reactivate_swap_slots_cache(void);
 48
 49#define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
 50#define SLOTS_CACHE 0x1
 51#define SLOTS_CACHE_RET 0x2
 52
 53static void deactivate_swap_slots_cache(void)
 54{
 55	mutex_lock(&swap_slots_cache_mutex);
 56	swap_slot_cache_active = false;
 57	__drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
 58	mutex_unlock(&swap_slots_cache_mutex);
 59}
 60
 61static void reactivate_swap_slots_cache(void)
 62{
 63	mutex_lock(&swap_slots_cache_mutex);
 64	swap_slot_cache_active = true;
 65	mutex_unlock(&swap_slots_cache_mutex);
 66}
 67
 68/* Must not be called with cpu hot plug lock */
 69void disable_swap_slots_cache_lock(void)
 70{
 71	mutex_lock(&swap_slots_cache_enable_mutex);
 72	swap_slot_cache_enabled = false;
 73	if (swap_slot_cache_initialized) {
 74		/* serialize with cpu hotplug operations */
 75		get_online_cpus();
 76		__drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
 77		put_online_cpus();
 78	}
 79}
 80
 81static void __reenable_swap_slots_cache(void)
 82{
 83	swap_slot_cache_enabled = has_usable_swap();
 84}
 85
 86void reenable_swap_slots_cache_unlock(void)
 87{
 88	__reenable_swap_slots_cache();
 89	mutex_unlock(&swap_slots_cache_enable_mutex);
 90}
 91
 92static bool check_cache_active(void)
 93{
 94	long pages;
 95
 96	if (!swap_slot_cache_enabled)
 97		return false;
 98
 99	pages = get_nr_swap_pages();
100	if (!swap_slot_cache_active) {
101		if (pages > num_online_cpus() *
102		    THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
103			reactivate_swap_slots_cache();
104		goto out;
105	}
106
107	/* if global pool of slot caches too low, deactivate cache */
108	if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
109		deactivate_swap_slots_cache();
110out:
111	return swap_slot_cache_active;
112}
113
114static int alloc_swap_slot_cache(unsigned int cpu)
115{
116	struct swap_slots_cache *cache;
117	swp_entry_t *slots, *slots_ret;
118
119	/*
120	 * Do allocation outside swap_slots_cache_mutex
121	 * as kvzalloc could trigger reclaim and get_swap_page,
122	 * which can lock swap_slots_cache_mutex.
123	 */
124	slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
125			 GFP_KERNEL);
126	if (!slots)
127		return -ENOMEM;
128
129	slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
130			     GFP_KERNEL);
131	if (!slots_ret) {
132		kvfree(slots);
133		return -ENOMEM;
134	}
135
136	mutex_lock(&swap_slots_cache_mutex);
137	cache = &per_cpu(swp_slots, cpu);
138	if (cache->slots || cache->slots_ret) {
139		/* cache already allocated */
140		mutex_unlock(&swap_slots_cache_mutex);
141
142		kvfree(slots);
143		kvfree(slots_ret);
144
145		return 0;
146	}
147
148	if (!cache->lock_initialized) {
149		mutex_init(&cache->alloc_lock);
150		spin_lock_init(&cache->free_lock);
151		cache->lock_initialized = true;
152	}
153	cache->nr = 0;
154	cache->cur = 0;
155	cache->n_ret = 0;
156	/*
157	 * We initialized alloc_lock and free_lock earlier.  We use
158	 * !cache->slots or !cache->slots_ret to know if it is safe to acquire
159	 * the corresponding lock and use the cache.  Memory barrier below
160	 * ensures the assumption.
161	 */
162	mb();
163	cache->slots = slots;
164	cache->slots_ret = slots_ret;
165	mutex_unlock(&swap_slots_cache_mutex);
166	return 0;
167}
168
169static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
170				  bool free_slots)
171{
172	struct swap_slots_cache *cache;
173	swp_entry_t *slots = NULL;
174
175	cache = &per_cpu(swp_slots, cpu);
176	if ((type & SLOTS_CACHE) && cache->slots) {
177		mutex_lock(&cache->alloc_lock);
178		swapcache_free_entries(cache->slots + cache->cur, cache->nr);
179		cache->cur = 0;
180		cache->nr = 0;
181		if (free_slots && cache->slots) {
182			kvfree(cache->slots);
183			cache->slots = NULL;
184		}
185		mutex_unlock(&cache->alloc_lock);
186	}
187	if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
188		spin_lock_irq(&cache->free_lock);
189		swapcache_free_entries(cache->slots_ret, cache->n_ret);
190		cache->n_ret = 0;
191		if (free_slots && cache->slots_ret) {
192			slots = cache->slots_ret;
193			cache->slots_ret = NULL;
194		}
195		spin_unlock_irq(&cache->free_lock);
196		if (slots)
197			kvfree(slots);
198	}
199}
200
201static void __drain_swap_slots_cache(unsigned int type)
202{
203	unsigned int cpu;
204
205	/*
206	 * This function is called during
207	 *	1) swapoff, when we have to make sure no
208	 *	   left over slots are in cache when we remove
209	 *	   a swap device;
210	 *      2) disabling of swap slot cache, when we run low
211	 *	   on swap slots when allocating memory and need
212	 *	   to return swap slots to global pool.
213	 *
214	 * We cannot acquire cpu hot plug lock here as
215	 * this function can be invoked in the cpu
216	 * hot plug path:
217	 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
218	 *   -> memory allocation -> direct reclaim -> get_swap_page
219	 *   -> drain_swap_slots_cache
220	 *
221	 * Hence the loop over current online cpu below could miss cpu that
222	 * is being brought online but not yet marked as online.
223	 * That is okay as we do not schedule and run anything on a
224	 * cpu before it has been marked online. Hence, we will not
225	 * fill any swap slots in slots cache of such cpu.
226	 * There are no slots on such cpu that need to be drained.
227	 */
228	for_each_online_cpu(cpu)
229		drain_slots_cache_cpu(cpu, type, false);
230}
231
232static int free_slot_cache(unsigned int cpu)
233{
234	mutex_lock(&swap_slots_cache_mutex);
235	drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
236	mutex_unlock(&swap_slots_cache_mutex);
237	return 0;
238}
239
240int enable_swap_slots_cache(void)
241{
242	mutex_lock(&swap_slots_cache_enable_mutex);
243	if (!swap_slot_cache_initialized) {
244		int ret;
245
246		ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
247					alloc_swap_slot_cache, free_slot_cache);
248		if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
249				       "without swap slots cache.\n", __func__))
250			goto out_unlock;
251
252		swap_slot_cache_initialized = true;
253	}
254
255	__reenable_swap_slots_cache();
256out_unlock:
257	mutex_unlock(&swap_slots_cache_enable_mutex);
258	return 0;
259}
260
261/* called with swap slot cache's alloc lock held */
262static int refill_swap_slots_cache(struct swap_slots_cache *cache)
263{
264	if (!use_swap_slot_cache || cache->nr)
265		return 0;
266
267	cache->cur = 0;
268	if (swap_slot_cache_active)
269		cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
270					   cache->slots, 1);
271
272	return cache->nr;
273}
274
275int free_swap_slot(swp_entry_t entry)
276{
277	struct swap_slots_cache *cache;
278
279	cache = raw_cpu_ptr(&swp_slots);
280	if (likely(use_swap_slot_cache && cache->slots_ret)) {
281		spin_lock_irq(&cache->free_lock);
282		/* Swap slots cache may be deactivated before acquiring lock */
283		if (!use_swap_slot_cache || !cache->slots_ret) {
284			spin_unlock_irq(&cache->free_lock);
285			goto direct_free;
286		}
287		if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
288			/*
289			 * Return slots to global pool.
290			 * The current swap_map value is SWAP_HAS_CACHE.
291			 * Set it to 0 to indicate it is available for
292			 * allocation in global pool
293			 */
294			swapcache_free_entries(cache->slots_ret, cache->n_ret);
295			cache->n_ret = 0;
296		}
297		cache->slots_ret[cache->n_ret++] = entry;
298		spin_unlock_irq(&cache->free_lock);
299	} else {
300direct_free:
301		swapcache_free_entries(&entry, 1);
302	}
303
304	return 0;
305}
306
307swp_entry_t get_swap_page(struct page *page)
308{
309	swp_entry_t entry;
310	struct swap_slots_cache *cache;
311
312	entry.val = 0;
313
314	if (PageTransHuge(page)) {
315		if (IS_ENABLED(CONFIG_THP_SWAP))
316			get_swap_pages(1, &entry, HPAGE_PMD_NR);
317		goto out;
318	}
319
320	/*
321	 * Preemption is allowed here, because we may sleep
322	 * in refill_swap_slots_cache().  But it is safe, because
323	 * accesses to the per-CPU data structure are protected by the
324	 * mutex cache->alloc_lock.
325	 *
326	 * The alloc path here does not touch cache->slots_ret
327	 * so cache->free_lock is not taken.
328	 */
329	cache = raw_cpu_ptr(&swp_slots);
330
331	if (likely(check_cache_active() && cache->slots)) {
332		mutex_lock(&cache->alloc_lock);
333		if (cache->slots) {
334repeat:
335			if (cache->nr) {
336				entry = cache->slots[cache->cur];
337				cache->slots[cache->cur++].val = 0;
338				cache->nr--;
339			} else if (refill_swap_slots_cache(cache)) {
340				goto repeat;
341			}
342		}
343		mutex_unlock(&cache->alloc_lock);
344		if (entry.val)
345			goto out;
346	}
347
348	get_swap_pages(1, &entry, 1);
349out:
350	if (mem_cgroup_try_charge_swap(page, entry)) {
351		put_swap_page(page, entry);
352		entry.val = 0;
353	}
354	return entry;
355}