Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Manage cache of swap slots to be used for and returned from
4 * swap.
5 *
6 * Copyright(c) 2016 Intel Corporation.
7 *
8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
9 *
10 * We allocate the swap slots from the global pool and put
11 * it into local per cpu caches. This has the advantage
12 * of no needing to acquire the swap_info lock every time
13 * we need a new slot.
14 *
15 * There is also opportunity to simply return the slot
16 * to local caches without needing to acquire swap_info
17 * lock. We do not reuse the returned slots directly but
18 * move them back to the global pool in a batch. This
19 * allows the slots to coaellesce and reduce fragmentation.
20 *
21 * The swap entry allocated is marked with SWAP_HAS_CACHE
22 * flag in map_count that prevents it from being allocated
23 * again from the global pool.
24 *
25 * The swap slots cache is protected by a mutex instead of
26 * a spin lock as when we search for slots with scan_swap_map,
27 * we can possibly sleep.
28 */
29
30#include <linux/swap_slots.h>
31#include <linux/cpu.h>
32#include <linux/cpumask.h>
33#include <linux/vmalloc.h>
34#include <linux/mutex.h>
35#include <linux/mm.h>
36
37static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
38static bool swap_slot_cache_active;
39bool swap_slot_cache_enabled;
40static bool swap_slot_cache_initialized;
41static DEFINE_MUTEX(swap_slots_cache_mutex);
42/* Serialize swap slots cache enable/disable operations */
43static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
44
45static void __drain_swap_slots_cache(unsigned int type);
46static void deactivate_swap_slots_cache(void);
47static void reactivate_swap_slots_cache(void);
48
49#define use_swap_slot_cache (swap_slot_cache_active && \
50 swap_slot_cache_enabled && swap_slot_cache_initialized)
51#define SLOTS_CACHE 0x1
52#define SLOTS_CACHE_RET 0x2
53
54static void deactivate_swap_slots_cache(void)
55{
56 mutex_lock(&swap_slots_cache_mutex);
57 swap_slot_cache_active = false;
58 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
59 mutex_unlock(&swap_slots_cache_mutex);
60}
61
62static void reactivate_swap_slots_cache(void)
63{
64 mutex_lock(&swap_slots_cache_mutex);
65 swap_slot_cache_active = true;
66 mutex_unlock(&swap_slots_cache_mutex);
67}
68
69/* Must not be called with cpu hot plug lock */
70void disable_swap_slots_cache_lock(void)
71{
72 mutex_lock(&swap_slots_cache_enable_mutex);
73 swap_slot_cache_enabled = false;
74 if (swap_slot_cache_initialized) {
75 /* serialize with cpu hotplug operations */
76 get_online_cpus();
77 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
78 put_online_cpus();
79 }
80}
81
82static void __reenable_swap_slots_cache(void)
83{
84 swap_slot_cache_enabled = has_usable_swap();
85}
86
87void reenable_swap_slots_cache_unlock(void)
88{
89 __reenable_swap_slots_cache();
90 mutex_unlock(&swap_slots_cache_enable_mutex);
91}
92
93static bool check_cache_active(void)
94{
95 long pages;
96
97 if (!swap_slot_cache_enabled || !swap_slot_cache_initialized)
98 return false;
99
100 pages = get_nr_swap_pages();
101 if (!swap_slot_cache_active) {
102 if (pages > num_online_cpus() *
103 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
104 reactivate_swap_slots_cache();
105 goto out;
106 }
107
108 /* if global pool of slot caches too low, deactivate cache */
109 if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
110 deactivate_swap_slots_cache();
111out:
112 return swap_slot_cache_active;
113}
114
115static int alloc_swap_slot_cache(unsigned int cpu)
116{
117 struct swap_slots_cache *cache;
118 swp_entry_t *slots, *slots_ret;
119
120 /*
121 * Do allocation outside swap_slots_cache_mutex
122 * as kvzalloc could trigger reclaim and get_swap_page,
123 * which can lock swap_slots_cache_mutex.
124 */
125 slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
126 GFP_KERNEL);
127 if (!slots)
128 return -ENOMEM;
129
130 slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
131 GFP_KERNEL);
132 if (!slots_ret) {
133 kvfree(slots);
134 return -ENOMEM;
135 }
136
137 mutex_lock(&swap_slots_cache_mutex);
138 cache = &per_cpu(swp_slots, cpu);
139 if (cache->slots || cache->slots_ret)
140 /* cache already allocated */
141 goto out;
142 if (!cache->lock_initialized) {
143 mutex_init(&cache->alloc_lock);
144 spin_lock_init(&cache->free_lock);
145 cache->lock_initialized = true;
146 }
147 cache->nr = 0;
148 cache->cur = 0;
149 cache->n_ret = 0;
150 /*
151 * We initialized alloc_lock and free_lock earlier. We use
152 * !cache->slots or !cache->slots_ret to know if it is safe to acquire
153 * the corresponding lock and use the cache. Memory barrier below
154 * ensures the assumption.
155 */
156 mb();
157 cache->slots = slots;
158 slots = NULL;
159 cache->slots_ret = slots_ret;
160 slots_ret = NULL;
161out:
162 mutex_unlock(&swap_slots_cache_mutex);
163 if (slots)
164 kvfree(slots);
165 if (slots_ret)
166 kvfree(slots_ret);
167 return 0;
168}
169
170static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
171 bool free_slots)
172{
173 struct swap_slots_cache *cache;
174 swp_entry_t *slots = NULL;
175
176 cache = &per_cpu(swp_slots, cpu);
177 if ((type & SLOTS_CACHE) && cache->slots) {
178 mutex_lock(&cache->alloc_lock);
179 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
180 cache->cur = 0;
181 cache->nr = 0;
182 if (free_slots && cache->slots) {
183 kvfree(cache->slots);
184 cache->slots = NULL;
185 }
186 mutex_unlock(&cache->alloc_lock);
187 }
188 if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
189 spin_lock_irq(&cache->free_lock);
190 swapcache_free_entries(cache->slots_ret, cache->n_ret);
191 cache->n_ret = 0;
192 if (free_slots && cache->slots_ret) {
193 slots = cache->slots_ret;
194 cache->slots_ret = NULL;
195 }
196 spin_unlock_irq(&cache->free_lock);
197 if (slots)
198 kvfree(slots);
199 }
200}
201
202static void __drain_swap_slots_cache(unsigned int type)
203{
204 unsigned int cpu;
205
206 /*
207 * This function is called during
208 * 1) swapoff, when we have to make sure no
209 * left over slots are in cache when we remove
210 * a swap device;
211 * 2) disabling of swap slot cache, when we run low
212 * on swap slots when allocating memory and need
213 * to return swap slots to global pool.
214 *
215 * We cannot acquire cpu hot plug lock here as
216 * this function can be invoked in the cpu
217 * hot plug path:
218 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
219 * -> memory allocation -> direct reclaim -> get_swap_page
220 * -> drain_swap_slots_cache
221 *
222 * Hence the loop over current online cpu below could miss cpu that
223 * is being brought online but not yet marked as online.
224 * That is okay as we do not schedule and run anything on a
225 * cpu before it has been marked online. Hence, we will not
226 * fill any swap slots in slots cache of such cpu.
227 * There are no slots on such cpu that need to be drained.
228 */
229 for_each_online_cpu(cpu)
230 drain_slots_cache_cpu(cpu, type, false);
231}
232
233static int free_slot_cache(unsigned int cpu)
234{
235 mutex_lock(&swap_slots_cache_mutex);
236 drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
237 mutex_unlock(&swap_slots_cache_mutex);
238 return 0;
239}
240
241int enable_swap_slots_cache(void)
242{
243 int ret = 0;
244
245 mutex_lock(&swap_slots_cache_enable_mutex);
246 if (swap_slot_cache_initialized) {
247 __reenable_swap_slots_cache();
248 goto out_unlock;
249 }
250
251 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
252 alloc_swap_slot_cache, free_slot_cache);
253 if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
254 "without swap slots cache.\n", __func__))
255 goto out_unlock;
256
257 swap_slot_cache_initialized = true;
258 __reenable_swap_slots_cache();
259out_unlock:
260 mutex_unlock(&swap_slots_cache_enable_mutex);
261 return 0;
262}
263
264/* called with swap slot cache's alloc lock held */
265static int refill_swap_slots_cache(struct swap_slots_cache *cache)
266{
267 if (!use_swap_slot_cache || cache->nr)
268 return 0;
269
270 cache->cur = 0;
271 if (swap_slot_cache_active)
272 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
273 cache->slots, 1);
274
275 return cache->nr;
276}
277
278int free_swap_slot(swp_entry_t entry)
279{
280 struct swap_slots_cache *cache;
281
282 cache = raw_cpu_ptr(&swp_slots);
283 if (likely(use_swap_slot_cache && cache->slots_ret)) {
284 spin_lock_irq(&cache->free_lock);
285 /* Swap slots cache may be deactivated before acquiring lock */
286 if (!use_swap_slot_cache || !cache->slots_ret) {
287 spin_unlock_irq(&cache->free_lock);
288 goto direct_free;
289 }
290 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
291 /*
292 * Return slots to global pool.
293 * The current swap_map value is SWAP_HAS_CACHE.
294 * Set it to 0 to indicate it is available for
295 * allocation in global pool
296 */
297 swapcache_free_entries(cache->slots_ret, cache->n_ret);
298 cache->n_ret = 0;
299 }
300 cache->slots_ret[cache->n_ret++] = entry;
301 spin_unlock_irq(&cache->free_lock);
302 } else {
303direct_free:
304 swapcache_free_entries(&entry, 1);
305 }
306
307 return 0;
308}
309
310swp_entry_t get_swap_page(struct page *page)
311{
312 swp_entry_t entry, *pentry;
313 struct swap_slots_cache *cache;
314
315 entry.val = 0;
316
317 if (PageTransHuge(page)) {
318 if (IS_ENABLED(CONFIG_THP_SWAP))
319 get_swap_pages(1, &entry, HPAGE_PMD_NR);
320 goto out;
321 }
322
323 /*
324 * Preemption is allowed here, because we may sleep
325 * in refill_swap_slots_cache(). But it is safe, because
326 * accesses to the per-CPU data structure are protected by the
327 * mutex cache->alloc_lock.
328 *
329 * The alloc path here does not touch cache->slots_ret
330 * so cache->free_lock is not taken.
331 */
332 cache = raw_cpu_ptr(&swp_slots);
333
334 if (likely(check_cache_active() && cache->slots)) {
335 mutex_lock(&cache->alloc_lock);
336 if (cache->slots) {
337repeat:
338 if (cache->nr) {
339 pentry = &cache->slots[cache->cur++];
340 entry = *pentry;
341 pentry->val = 0;
342 cache->nr--;
343 } else {
344 if (refill_swap_slots_cache(cache))
345 goto repeat;
346 }
347 }
348 mutex_unlock(&cache->alloc_lock);
349 if (entry.val)
350 goto out;
351 }
352
353 get_swap_pages(1, &entry, 1);
354out:
355 if (mem_cgroup_try_charge_swap(page, entry)) {
356 put_swap_page(page, entry);
357 entry.val = 0;
358 }
359 return entry;
360}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Manage cache of swap slots to be used for and returned from
4 * swap.
5 *
6 * Copyright(c) 2016 Intel Corporation.
7 *
8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
9 *
10 * We allocate the swap slots from the global pool and put
11 * it into local per cpu caches. This has the advantage
12 * of no needing to acquire the swap_info lock every time
13 * we need a new slot.
14 *
15 * There is also opportunity to simply return the slot
16 * to local caches without needing to acquire swap_info
17 * lock. We do not reuse the returned slots directly but
18 * move them back to the global pool in a batch. This
19 * allows the slots to coaellesce and reduce fragmentation.
20 *
21 * The swap entry allocated is marked with SWAP_HAS_CACHE
22 * flag in map_count that prevents it from being allocated
23 * again from the global pool.
24 *
25 * The swap slots cache is protected by a mutex instead of
26 * a spin lock as when we search for slots with scan_swap_map,
27 * we can possibly sleep.
28 */
29
30#include <linux/swap_slots.h>
31#include <linux/cpu.h>
32#include <linux/cpumask.h>
33#include <linux/vmalloc.h>
34#include <linux/mutex.h>
35#include <linux/mm.h>
36
37static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
38static bool swap_slot_cache_active;
39bool swap_slot_cache_enabled;
40static bool swap_slot_cache_initialized;
41static DEFINE_MUTEX(swap_slots_cache_mutex);
42/* Serialize swap slots cache enable/disable operations */
43static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
44
45static void __drain_swap_slots_cache(unsigned int type);
46static void deactivate_swap_slots_cache(void);
47static void reactivate_swap_slots_cache(void);
48
49#define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
50#define SLOTS_CACHE 0x1
51#define SLOTS_CACHE_RET 0x2
52
53static void deactivate_swap_slots_cache(void)
54{
55 mutex_lock(&swap_slots_cache_mutex);
56 swap_slot_cache_active = false;
57 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
58 mutex_unlock(&swap_slots_cache_mutex);
59}
60
61static void reactivate_swap_slots_cache(void)
62{
63 mutex_lock(&swap_slots_cache_mutex);
64 swap_slot_cache_active = true;
65 mutex_unlock(&swap_slots_cache_mutex);
66}
67
68/* Must not be called with cpu hot plug lock */
69void disable_swap_slots_cache_lock(void)
70{
71 mutex_lock(&swap_slots_cache_enable_mutex);
72 swap_slot_cache_enabled = false;
73 if (swap_slot_cache_initialized) {
74 /* serialize with cpu hotplug operations */
75 get_online_cpus();
76 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
77 put_online_cpus();
78 }
79}
80
81static void __reenable_swap_slots_cache(void)
82{
83 swap_slot_cache_enabled = has_usable_swap();
84}
85
86void reenable_swap_slots_cache_unlock(void)
87{
88 __reenable_swap_slots_cache();
89 mutex_unlock(&swap_slots_cache_enable_mutex);
90}
91
92static bool check_cache_active(void)
93{
94 long pages;
95
96 if (!swap_slot_cache_enabled)
97 return false;
98
99 pages = get_nr_swap_pages();
100 if (!swap_slot_cache_active) {
101 if (pages > num_online_cpus() *
102 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
103 reactivate_swap_slots_cache();
104 goto out;
105 }
106
107 /* if global pool of slot caches too low, deactivate cache */
108 if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
109 deactivate_swap_slots_cache();
110out:
111 return swap_slot_cache_active;
112}
113
114static int alloc_swap_slot_cache(unsigned int cpu)
115{
116 struct swap_slots_cache *cache;
117 swp_entry_t *slots, *slots_ret;
118
119 /*
120 * Do allocation outside swap_slots_cache_mutex
121 * as kvzalloc could trigger reclaim and get_swap_page,
122 * which can lock swap_slots_cache_mutex.
123 */
124 slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
125 GFP_KERNEL);
126 if (!slots)
127 return -ENOMEM;
128
129 slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
130 GFP_KERNEL);
131 if (!slots_ret) {
132 kvfree(slots);
133 return -ENOMEM;
134 }
135
136 mutex_lock(&swap_slots_cache_mutex);
137 cache = &per_cpu(swp_slots, cpu);
138 if (cache->slots || cache->slots_ret) {
139 /* cache already allocated */
140 mutex_unlock(&swap_slots_cache_mutex);
141
142 kvfree(slots);
143 kvfree(slots_ret);
144
145 return 0;
146 }
147
148 if (!cache->lock_initialized) {
149 mutex_init(&cache->alloc_lock);
150 spin_lock_init(&cache->free_lock);
151 cache->lock_initialized = true;
152 }
153 cache->nr = 0;
154 cache->cur = 0;
155 cache->n_ret = 0;
156 /*
157 * We initialized alloc_lock and free_lock earlier. We use
158 * !cache->slots or !cache->slots_ret to know if it is safe to acquire
159 * the corresponding lock and use the cache. Memory barrier below
160 * ensures the assumption.
161 */
162 mb();
163 cache->slots = slots;
164 cache->slots_ret = slots_ret;
165 mutex_unlock(&swap_slots_cache_mutex);
166 return 0;
167}
168
169static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
170 bool free_slots)
171{
172 struct swap_slots_cache *cache;
173 swp_entry_t *slots = NULL;
174
175 cache = &per_cpu(swp_slots, cpu);
176 if ((type & SLOTS_CACHE) && cache->slots) {
177 mutex_lock(&cache->alloc_lock);
178 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
179 cache->cur = 0;
180 cache->nr = 0;
181 if (free_slots && cache->slots) {
182 kvfree(cache->slots);
183 cache->slots = NULL;
184 }
185 mutex_unlock(&cache->alloc_lock);
186 }
187 if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
188 spin_lock_irq(&cache->free_lock);
189 swapcache_free_entries(cache->slots_ret, cache->n_ret);
190 cache->n_ret = 0;
191 if (free_slots && cache->slots_ret) {
192 slots = cache->slots_ret;
193 cache->slots_ret = NULL;
194 }
195 spin_unlock_irq(&cache->free_lock);
196 if (slots)
197 kvfree(slots);
198 }
199}
200
201static void __drain_swap_slots_cache(unsigned int type)
202{
203 unsigned int cpu;
204
205 /*
206 * This function is called during
207 * 1) swapoff, when we have to make sure no
208 * left over slots are in cache when we remove
209 * a swap device;
210 * 2) disabling of swap slot cache, when we run low
211 * on swap slots when allocating memory and need
212 * to return swap slots to global pool.
213 *
214 * We cannot acquire cpu hot plug lock here as
215 * this function can be invoked in the cpu
216 * hot plug path:
217 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
218 * -> memory allocation -> direct reclaim -> get_swap_page
219 * -> drain_swap_slots_cache
220 *
221 * Hence the loop over current online cpu below could miss cpu that
222 * is being brought online but not yet marked as online.
223 * That is okay as we do not schedule and run anything on a
224 * cpu before it has been marked online. Hence, we will not
225 * fill any swap slots in slots cache of such cpu.
226 * There are no slots on such cpu that need to be drained.
227 */
228 for_each_online_cpu(cpu)
229 drain_slots_cache_cpu(cpu, type, false);
230}
231
232static int free_slot_cache(unsigned int cpu)
233{
234 mutex_lock(&swap_slots_cache_mutex);
235 drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
236 mutex_unlock(&swap_slots_cache_mutex);
237 return 0;
238}
239
240int enable_swap_slots_cache(void)
241{
242 mutex_lock(&swap_slots_cache_enable_mutex);
243 if (!swap_slot_cache_initialized) {
244 int ret;
245
246 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
247 alloc_swap_slot_cache, free_slot_cache);
248 if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
249 "without swap slots cache.\n", __func__))
250 goto out_unlock;
251
252 swap_slot_cache_initialized = true;
253 }
254
255 __reenable_swap_slots_cache();
256out_unlock:
257 mutex_unlock(&swap_slots_cache_enable_mutex);
258 return 0;
259}
260
261/* called with swap slot cache's alloc lock held */
262static int refill_swap_slots_cache(struct swap_slots_cache *cache)
263{
264 if (!use_swap_slot_cache || cache->nr)
265 return 0;
266
267 cache->cur = 0;
268 if (swap_slot_cache_active)
269 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
270 cache->slots, 1);
271
272 return cache->nr;
273}
274
275int free_swap_slot(swp_entry_t entry)
276{
277 struct swap_slots_cache *cache;
278
279 cache = raw_cpu_ptr(&swp_slots);
280 if (likely(use_swap_slot_cache && cache->slots_ret)) {
281 spin_lock_irq(&cache->free_lock);
282 /* Swap slots cache may be deactivated before acquiring lock */
283 if (!use_swap_slot_cache || !cache->slots_ret) {
284 spin_unlock_irq(&cache->free_lock);
285 goto direct_free;
286 }
287 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
288 /*
289 * Return slots to global pool.
290 * The current swap_map value is SWAP_HAS_CACHE.
291 * Set it to 0 to indicate it is available for
292 * allocation in global pool
293 */
294 swapcache_free_entries(cache->slots_ret, cache->n_ret);
295 cache->n_ret = 0;
296 }
297 cache->slots_ret[cache->n_ret++] = entry;
298 spin_unlock_irq(&cache->free_lock);
299 } else {
300direct_free:
301 swapcache_free_entries(&entry, 1);
302 }
303
304 return 0;
305}
306
307swp_entry_t get_swap_page(struct page *page)
308{
309 swp_entry_t entry;
310 struct swap_slots_cache *cache;
311
312 entry.val = 0;
313
314 if (PageTransHuge(page)) {
315 if (IS_ENABLED(CONFIG_THP_SWAP))
316 get_swap_pages(1, &entry, HPAGE_PMD_NR);
317 goto out;
318 }
319
320 /*
321 * Preemption is allowed here, because we may sleep
322 * in refill_swap_slots_cache(). But it is safe, because
323 * accesses to the per-CPU data structure are protected by the
324 * mutex cache->alloc_lock.
325 *
326 * The alloc path here does not touch cache->slots_ret
327 * so cache->free_lock is not taken.
328 */
329 cache = raw_cpu_ptr(&swp_slots);
330
331 if (likely(check_cache_active() && cache->slots)) {
332 mutex_lock(&cache->alloc_lock);
333 if (cache->slots) {
334repeat:
335 if (cache->nr) {
336 entry = cache->slots[cache->cur];
337 cache->slots[cache->cur++].val = 0;
338 cache->nr--;
339 } else if (refill_swap_slots_cache(cache)) {
340 goto repeat;
341 }
342 }
343 mutex_unlock(&cache->alloc_lock);
344 if (entry.val)
345 goto out;
346 }
347
348 get_swap_pages(1, &entry, 1);
349out:
350 if (mem_cgroup_try_charge_swap(page, entry)) {
351 put_swap_page(page, entry);
352 entry.val = 0;
353 }
354 return entry;
355}