Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * zswap.c - zswap driver file
4 *
5 * zswap is a cache that takes pages that are in the process
6 * of being swapped out and attempts to compress and store them in a
7 * RAM-based memory pool. This can result in a significant I/O reduction on
8 * the swap device and, in the case where decompressing from RAM is faster
9 * than reading from the swap device, can also improve workload performance.
10 *
11 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com>
12*/
13
14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16#include <linux/module.h>
17#include <linux/cpu.h>
18#include <linux/highmem.h>
19#include <linux/slab.h>
20#include <linux/spinlock.h>
21#include <linux/types.h>
22#include <linux/atomic.h>
23#include <linux/swap.h>
24#include <linux/crypto.h>
25#include <linux/scatterlist.h>
26#include <linux/mempolicy.h>
27#include <linux/mempool.h>
28#include <linux/zpool.h>
29#include <crypto/acompress.h>
30#include <linux/zswap.h>
31#include <linux/mm_types.h>
32#include <linux/page-flags.h>
33#include <linux/swapops.h>
34#include <linux/writeback.h>
35#include <linux/pagemap.h>
36#include <linux/workqueue.h>
37#include <linux/list_lru.h>
38
39#include "swap.h"
40#include "internal.h"
41
42/*********************************
43* statistics
44**********************************/
45/* The number of compressed pages currently stored in zswap */
46atomic_long_t zswap_stored_pages = ATOMIC_INIT(0);
47
48/*
49 * The statistics below are not protected from concurrent access for
50 * performance reasons so they may not be a 100% accurate. However,
51 * they do provide useful information on roughly how many times a
52 * certain event is occurring.
53*/
54
55/* Pool limit was hit (see zswap_max_pool_percent) */
56static u64 zswap_pool_limit_hit;
57/* Pages written back when pool limit was reached */
58static u64 zswap_written_back_pages;
59/* Store failed due to a reclaim failure after pool limit was reached */
60static u64 zswap_reject_reclaim_fail;
61/* Store failed due to compression algorithm failure */
62static u64 zswap_reject_compress_fail;
63/* Compressed page was too big for the allocator to (optimally) store */
64static u64 zswap_reject_compress_poor;
65/* Store failed because underlying allocator could not get memory */
66static u64 zswap_reject_alloc_fail;
67/* Store failed because the entry metadata could not be allocated (rare) */
68static u64 zswap_reject_kmemcache_fail;
69
70/* Shrinker work queue */
71static struct workqueue_struct *shrink_wq;
72/* Pool limit was hit, we need to calm down */
73static bool zswap_pool_reached_full;
74
75/*********************************
76* tunables
77**********************************/
78
79#define ZSWAP_PARAM_UNSET ""
80
81static int zswap_setup(void);
82
83/* Enable/disable zswap */
84static DEFINE_STATIC_KEY_MAYBE(CONFIG_ZSWAP_DEFAULT_ON, zswap_ever_enabled);
85static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON);
86static int zswap_enabled_param_set(const char *,
87 const struct kernel_param *);
88static const struct kernel_param_ops zswap_enabled_param_ops = {
89 .set = zswap_enabled_param_set,
90 .get = param_get_bool,
91};
92module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
93
94/* Crypto compressor to use */
95static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
96static int zswap_compressor_param_set(const char *,
97 const struct kernel_param *);
98static const struct kernel_param_ops zswap_compressor_param_ops = {
99 .set = zswap_compressor_param_set,
100 .get = param_get_charp,
101 .free = param_free_charp,
102};
103module_param_cb(compressor, &zswap_compressor_param_ops,
104 &zswap_compressor, 0644);
105
106/* Compressed storage zpool to use */
107static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
108static int zswap_zpool_param_set(const char *, const struct kernel_param *);
109static const struct kernel_param_ops zswap_zpool_param_ops = {
110 .set = zswap_zpool_param_set,
111 .get = param_get_charp,
112 .free = param_free_charp,
113};
114module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
115
116/* The maximum percentage of memory that the compressed pool can occupy */
117static unsigned int zswap_max_pool_percent = 20;
118module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
119
120/* The threshold for accepting new pages after the max_pool_percent was hit */
121static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */
122module_param_named(accept_threshold_percent, zswap_accept_thr_percent,
123 uint, 0644);
124
125/* Enable/disable memory pressure-based shrinker. */
126static bool zswap_shrinker_enabled = IS_ENABLED(
127 CONFIG_ZSWAP_SHRINKER_DEFAULT_ON);
128module_param_named(shrinker_enabled, zswap_shrinker_enabled, bool, 0644);
129
130bool zswap_is_enabled(void)
131{
132 return zswap_enabled;
133}
134
135bool zswap_never_enabled(void)
136{
137 return !static_branch_maybe(CONFIG_ZSWAP_DEFAULT_ON, &zswap_ever_enabled);
138}
139
140/*********************************
141* data structures
142**********************************/
143
144struct crypto_acomp_ctx {
145 struct crypto_acomp *acomp;
146 struct acomp_req *req;
147 struct crypto_wait wait;
148 u8 *buffer;
149 struct mutex mutex;
150 bool is_sleepable;
151};
152
153/*
154 * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock.
155 * The only case where lru_lock is not acquired while holding tree.lock is
156 * when a zswap_entry is taken off the lru for writeback, in that case it
157 * needs to be verified that it's still valid in the tree.
158 */
159struct zswap_pool {
160 struct zpool *zpool;
161 struct crypto_acomp_ctx __percpu *acomp_ctx;
162 struct percpu_ref ref;
163 struct list_head list;
164 struct work_struct release_work;
165 struct hlist_node node;
166 char tfm_name[CRYPTO_MAX_ALG_NAME];
167};
168
169/* Global LRU lists shared by all zswap pools. */
170static struct list_lru zswap_list_lru;
171
172/* The lock protects zswap_next_shrink updates. */
173static DEFINE_SPINLOCK(zswap_shrink_lock);
174static struct mem_cgroup *zswap_next_shrink;
175static struct work_struct zswap_shrink_work;
176static struct shrinker *zswap_shrinker;
177
178/*
179 * struct zswap_entry
180 *
181 * This structure contains the metadata for tracking a single compressed
182 * page within zswap.
183 *
184 * swpentry - associated swap entry, the offset indexes into the red-black tree
185 * length - the length in bytes of the compressed page data. Needed during
186 * decompression.
187 * referenced - true if the entry recently entered the zswap pool. Unset by the
188 * writeback logic. The entry is only reclaimed by the writeback
189 * logic if referenced is unset. See comments in the shrinker
190 * section for context.
191 * pool - the zswap_pool the entry's data is in
192 * handle - zpool allocation handle that stores the compressed page data
193 * objcg - the obj_cgroup that the compressed memory is charged to
194 * lru - handle to the pool's lru used to evict pages.
195 */
196struct zswap_entry {
197 swp_entry_t swpentry;
198 unsigned int length;
199 bool referenced;
200 struct zswap_pool *pool;
201 unsigned long handle;
202 struct obj_cgroup *objcg;
203 struct list_head lru;
204};
205
206static struct xarray *zswap_trees[MAX_SWAPFILES];
207static unsigned int nr_zswap_trees[MAX_SWAPFILES];
208
209/* RCU-protected iteration */
210static LIST_HEAD(zswap_pools);
211/* protects zswap_pools list modification */
212static DEFINE_SPINLOCK(zswap_pools_lock);
213/* pool counter to provide unique names to zpool */
214static atomic_t zswap_pools_count = ATOMIC_INIT(0);
215
216enum zswap_init_type {
217 ZSWAP_UNINIT,
218 ZSWAP_INIT_SUCCEED,
219 ZSWAP_INIT_FAILED
220};
221
222static enum zswap_init_type zswap_init_state;
223
224/* used to ensure the integrity of initialization */
225static DEFINE_MUTEX(zswap_init_lock);
226
227/* init completed, but couldn't create the initial pool */
228static bool zswap_has_pool;
229
230/*********************************
231* helpers and fwd declarations
232**********************************/
233
234static inline struct xarray *swap_zswap_tree(swp_entry_t swp)
235{
236 return &zswap_trees[swp_type(swp)][swp_offset(swp)
237 >> SWAP_ADDRESS_SPACE_SHIFT];
238}
239
240#define zswap_pool_debug(msg, p) \
241 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
242 zpool_get_type((p)->zpool))
243
244/*********************************
245* pool functions
246**********************************/
247static void __zswap_pool_empty(struct percpu_ref *ref);
248
249static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
250{
251 struct zswap_pool *pool;
252 char name[38]; /* 'zswap' + 32 char (max) num + \0 */
253 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
254 int ret, cpu;
255
256 if (!zswap_has_pool) {
257 /* if either are unset, pool initialization failed, and we
258 * need both params to be set correctly before trying to
259 * create a pool.
260 */
261 if (!strcmp(type, ZSWAP_PARAM_UNSET))
262 return NULL;
263 if (!strcmp(compressor, ZSWAP_PARAM_UNSET))
264 return NULL;
265 }
266
267 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
268 if (!pool)
269 return NULL;
270
271 /* unique name for each pool specifically required by zsmalloc */
272 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
273 pool->zpool = zpool_create_pool(type, name, gfp);
274 if (!pool->zpool) {
275 pr_err("%s zpool not available\n", type);
276 goto error;
277 }
278 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
279
280 strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
281
282 pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx);
283 if (!pool->acomp_ctx) {
284 pr_err("percpu alloc failed\n");
285 goto error;
286 }
287
288 for_each_possible_cpu(cpu)
289 mutex_init(&per_cpu_ptr(pool->acomp_ctx, cpu)->mutex);
290
291 ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
292 &pool->node);
293 if (ret)
294 goto error;
295
296 /* being the current pool takes 1 ref; this func expects the
297 * caller to always add the new pool as the current pool
298 */
299 ret = percpu_ref_init(&pool->ref, __zswap_pool_empty,
300 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
301 if (ret)
302 goto ref_fail;
303 INIT_LIST_HEAD(&pool->list);
304
305 zswap_pool_debug("created", pool);
306
307 return pool;
308
309ref_fail:
310 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
311error:
312 if (pool->acomp_ctx)
313 free_percpu(pool->acomp_ctx);
314 if (pool->zpool)
315 zpool_destroy_pool(pool->zpool);
316 kfree(pool);
317 return NULL;
318}
319
320static struct zswap_pool *__zswap_pool_create_fallback(void)
321{
322 bool has_comp, has_zpool;
323
324 has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
325 if (!has_comp && strcmp(zswap_compressor,
326 CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) {
327 pr_err("compressor %s not available, using default %s\n",
328 zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT);
329 param_free_charp(&zswap_compressor);
330 zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
331 has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
332 }
333 if (!has_comp) {
334 pr_err("default compressor %s not available\n",
335 zswap_compressor);
336 param_free_charp(&zswap_compressor);
337 zswap_compressor = ZSWAP_PARAM_UNSET;
338 }
339
340 has_zpool = zpool_has_pool(zswap_zpool_type);
341 if (!has_zpool && strcmp(zswap_zpool_type,
342 CONFIG_ZSWAP_ZPOOL_DEFAULT)) {
343 pr_err("zpool %s not available, using default %s\n",
344 zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT);
345 param_free_charp(&zswap_zpool_type);
346 zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
347 has_zpool = zpool_has_pool(zswap_zpool_type);
348 }
349 if (!has_zpool) {
350 pr_err("default zpool %s not available\n",
351 zswap_zpool_type);
352 param_free_charp(&zswap_zpool_type);
353 zswap_zpool_type = ZSWAP_PARAM_UNSET;
354 }
355
356 if (!has_comp || !has_zpool)
357 return NULL;
358
359 return zswap_pool_create(zswap_zpool_type, zswap_compressor);
360}
361
362static void zswap_pool_destroy(struct zswap_pool *pool)
363{
364 zswap_pool_debug("destroying", pool);
365
366 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
367 free_percpu(pool->acomp_ctx);
368
369 zpool_destroy_pool(pool->zpool);
370 kfree(pool);
371}
372
373static void __zswap_pool_release(struct work_struct *work)
374{
375 struct zswap_pool *pool = container_of(work, typeof(*pool),
376 release_work);
377
378 synchronize_rcu();
379
380 /* nobody should have been able to get a ref... */
381 WARN_ON(!percpu_ref_is_zero(&pool->ref));
382 percpu_ref_exit(&pool->ref);
383
384 /* pool is now off zswap_pools list and has no references. */
385 zswap_pool_destroy(pool);
386}
387
388static struct zswap_pool *zswap_pool_current(void);
389
390static void __zswap_pool_empty(struct percpu_ref *ref)
391{
392 struct zswap_pool *pool;
393
394 pool = container_of(ref, typeof(*pool), ref);
395
396 spin_lock_bh(&zswap_pools_lock);
397
398 WARN_ON(pool == zswap_pool_current());
399
400 list_del_rcu(&pool->list);
401
402 INIT_WORK(&pool->release_work, __zswap_pool_release);
403 schedule_work(&pool->release_work);
404
405 spin_unlock_bh(&zswap_pools_lock);
406}
407
408static int __must_check zswap_pool_tryget(struct zswap_pool *pool)
409{
410 if (!pool)
411 return 0;
412
413 return percpu_ref_tryget(&pool->ref);
414}
415
416/* The caller must already have a reference. */
417static void zswap_pool_get(struct zswap_pool *pool)
418{
419 percpu_ref_get(&pool->ref);
420}
421
422static void zswap_pool_put(struct zswap_pool *pool)
423{
424 percpu_ref_put(&pool->ref);
425}
426
427static struct zswap_pool *__zswap_pool_current(void)
428{
429 struct zswap_pool *pool;
430
431 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
432 WARN_ONCE(!pool && zswap_has_pool,
433 "%s: no page storage pool!\n", __func__);
434
435 return pool;
436}
437
438static struct zswap_pool *zswap_pool_current(void)
439{
440 assert_spin_locked(&zswap_pools_lock);
441
442 return __zswap_pool_current();
443}
444
445static struct zswap_pool *zswap_pool_current_get(void)
446{
447 struct zswap_pool *pool;
448
449 rcu_read_lock();
450
451 pool = __zswap_pool_current();
452 if (!zswap_pool_tryget(pool))
453 pool = NULL;
454
455 rcu_read_unlock();
456
457 return pool;
458}
459
460/* type and compressor must be null-terminated */
461static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
462{
463 struct zswap_pool *pool;
464
465 assert_spin_locked(&zswap_pools_lock);
466
467 list_for_each_entry_rcu(pool, &zswap_pools, list) {
468 if (strcmp(pool->tfm_name, compressor))
469 continue;
470 if (strcmp(zpool_get_type(pool->zpool), type))
471 continue;
472 /* if we can't get it, it's about to be destroyed */
473 if (!zswap_pool_tryget(pool))
474 continue;
475 return pool;
476 }
477
478 return NULL;
479}
480
481static unsigned long zswap_max_pages(void)
482{
483 return totalram_pages() * zswap_max_pool_percent / 100;
484}
485
486static unsigned long zswap_accept_thr_pages(void)
487{
488 return zswap_max_pages() * zswap_accept_thr_percent / 100;
489}
490
491unsigned long zswap_total_pages(void)
492{
493 struct zswap_pool *pool;
494 unsigned long total = 0;
495
496 rcu_read_lock();
497 list_for_each_entry_rcu(pool, &zswap_pools, list)
498 total += zpool_get_total_pages(pool->zpool);
499 rcu_read_unlock();
500
501 return total;
502}
503
504static bool zswap_check_limits(void)
505{
506 unsigned long cur_pages = zswap_total_pages();
507 unsigned long max_pages = zswap_max_pages();
508
509 if (cur_pages >= max_pages) {
510 zswap_pool_limit_hit++;
511 zswap_pool_reached_full = true;
512 } else if (zswap_pool_reached_full &&
513 cur_pages <= zswap_accept_thr_pages()) {
514 zswap_pool_reached_full = false;
515 }
516 return zswap_pool_reached_full;
517}
518
519/*********************************
520* param callbacks
521**********************************/
522
523static bool zswap_pool_changed(const char *s, const struct kernel_param *kp)
524{
525 /* no change required */
526 if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)
527 return false;
528 return true;
529}
530
531/* val must be a null-terminated string */
532static int __zswap_param_set(const char *val, const struct kernel_param *kp,
533 char *type, char *compressor)
534{
535 struct zswap_pool *pool, *put_pool = NULL;
536 char *s = strstrip((char *)val);
537 int ret = 0;
538 bool new_pool = false;
539
540 mutex_lock(&zswap_init_lock);
541 switch (zswap_init_state) {
542 case ZSWAP_UNINIT:
543 /* if this is load-time (pre-init) param setting,
544 * don't create a pool; that's done during init.
545 */
546 ret = param_set_charp(s, kp);
547 break;
548 case ZSWAP_INIT_SUCCEED:
549 new_pool = zswap_pool_changed(s, kp);
550 break;
551 case ZSWAP_INIT_FAILED:
552 pr_err("can't set param, initialization failed\n");
553 ret = -ENODEV;
554 }
555 mutex_unlock(&zswap_init_lock);
556
557 /* no need to create a new pool, return directly */
558 if (!new_pool)
559 return ret;
560
561 if (!type) {
562 if (!zpool_has_pool(s)) {
563 pr_err("zpool %s not available\n", s);
564 return -ENOENT;
565 }
566 type = s;
567 } else if (!compressor) {
568 if (!crypto_has_acomp(s, 0, 0)) {
569 pr_err("compressor %s not available\n", s);
570 return -ENOENT;
571 }
572 compressor = s;
573 } else {
574 WARN_ON(1);
575 return -EINVAL;
576 }
577
578 spin_lock_bh(&zswap_pools_lock);
579
580 pool = zswap_pool_find_get(type, compressor);
581 if (pool) {
582 zswap_pool_debug("using existing", pool);
583 WARN_ON(pool == zswap_pool_current());
584 list_del_rcu(&pool->list);
585 }
586
587 spin_unlock_bh(&zswap_pools_lock);
588
589 if (!pool)
590 pool = zswap_pool_create(type, compressor);
591 else {
592 /*
593 * Restore the initial ref dropped by percpu_ref_kill()
594 * when the pool was decommissioned and switch it again
595 * to percpu mode.
596 */
597 percpu_ref_resurrect(&pool->ref);
598
599 /* Drop the ref from zswap_pool_find_get(). */
600 zswap_pool_put(pool);
601 }
602
603 if (pool)
604 ret = param_set_charp(s, kp);
605 else
606 ret = -EINVAL;
607
608 spin_lock_bh(&zswap_pools_lock);
609
610 if (!ret) {
611 put_pool = zswap_pool_current();
612 list_add_rcu(&pool->list, &zswap_pools);
613 zswap_has_pool = true;
614 } else if (pool) {
615 /* add the possibly pre-existing pool to the end of the pools
616 * list; if it's new (and empty) then it'll be removed and
617 * destroyed by the put after we drop the lock
618 */
619 list_add_tail_rcu(&pool->list, &zswap_pools);
620 put_pool = pool;
621 }
622
623 spin_unlock_bh(&zswap_pools_lock);
624
625 if (!zswap_has_pool && !pool) {
626 /* if initial pool creation failed, and this pool creation also
627 * failed, maybe both compressor and zpool params were bad.
628 * Allow changing this param, so pool creation will succeed
629 * when the other param is changed. We already verified this
630 * param is ok in the zpool_has_pool() or crypto_has_acomp()
631 * checks above.
632 */
633 ret = param_set_charp(s, kp);
634 }
635
636 /* drop the ref from either the old current pool,
637 * or the new pool we failed to add
638 */
639 if (put_pool)
640 percpu_ref_kill(&put_pool->ref);
641
642 return ret;
643}
644
645static int zswap_compressor_param_set(const char *val,
646 const struct kernel_param *kp)
647{
648 return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
649}
650
651static int zswap_zpool_param_set(const char *val,
652 const struct kernel_param *kp)
653{
654 return __zswap_param_set(val, kp, NULL, zswap_compressor);
655}
656
657static int zswap_enabled_param_set(const char *val,
658 const struct kernel_param *kp)
659{
660 int ret = -ENODEV;
661
662 /* if this is load-time (pre-init) param setting, only set param. */
663 if (system_state != SYSTEM_RUNNING)
664 return param_set_bool(val, kp);
665
666 mutex_lock(&zswap_init_lock);
667 switch (zswap_init_state) {
668 case ZSWAP_UNINIT:
669 if (zswap_setup())
670 break;
671 fallthrough;
672 case ZSWAP_INIT_SUCCEED:
673 if (!zswap_has_pool)
674 pr_err("can't enable, no pool configured\n");
675 else
676 ret = param_set_bool(val, kp);
677 break;
678 case ZSWAP_INIT_FAILED:
679 pr_err("can't enable, initialization failed\n");
680 }
681 mutex_unlock(&zswap_init_lock);
682
683 return ret;
684}
685
686/*********************************
687* lru functions
688**********************************/
689
690/* should be called under RCU */
691#ifdef CONFIG_MEMCG
692static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
693{
694 return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL;
695}
696#else
697static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
698{
699 return NULL;
700}
701#endif
702
703static inline int entry_to_nid(struct zswap_entry *entry)
704{
705 return page_to_nid(virt_to_page(entry));
706}
707
708static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry)
709{
710 int nid = entry_to_nid(entry);
711 struct mem_cgroup *memcg;
712
713 /*
714 * Note that it is safe to use rcu_read_lock() here, even in the face of
715 * concurrent memcg offlining:
716 *
717 * 1. list_lru_add() is called before list_lru_one is dead. The
718 * new entry will be reparented to memcg's parent's list_lru.
719 * 2. list_lru_add() is called after list_lru_one is dead. The
720 * new entry will be added directly to memcg's parent's list_lru.
721 *
722 * Similar reasoning holds for list_lru_del().
723 */
724 rcu_read_lock();
725 memcg = mem_cgroup_from_entry(entry);
726 /* will always succeed */
727 list_lru_add(list_lru, &entry->lru, nid, memcg);
728 rcu_read_unlock();
729}
730
731static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry)
732{
733 int nid = entry_to_nid(entry);
734 struct mem_cgroup *memcg;
735
736 rcu_read_lock();
737 memcg = mem_cgroup_from_entry(entry);
738 /* will always succeed */
739 list_lru_del(list_lru, &entry->lru, nid, memcg);
740 rcu_read_unlock();
741}
742
743void zswap_lruvec_state_init(struct lruvec *lruvec)
744{
745 atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0);
746}
747
748void zswap_folio_swapin(struct folio *folio)
749{
750 struct lruvec *lruvec;
751
752 if (folio) {
753 lruvec = folio_lruvec(folio);
754 atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins);
755 }
756}
757
758/*
759 * This function should be called when a memcg is being offlined.
760 *
761 * Since the global shrinker shrink_worker() may hold a reference
762 * of the memcg, we must check and release the reference in
763 * zswap_next_shrink.
764 *
765 * shrink_worker() must handle the case where this function releases
766 * the reference of memcg being shrunk.
767 */
768void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg)
769{
770 /* lock out zswap shrinker walking memcg tree */
771 spin_lock(&zswap_shrink_lock);
772 if (zswap_next_shrink == memcg) {
773 do {
774 zswap_next_shrink = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
775 } while (zswap_next_shrink && !mem_cgroup_online(zswap_next_shrink));
776 }
777 spin_unlock(&zswap_shrink_lock);
778}
779
780/*********************************
781* zswap entry functions
782**********************************/
783static struct kmem_cache *zswap_entry_cache;
784
785static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid)
786{
787 struct zswap_entry *entry;
788 entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid);
789 if (!entry)
790 return NULL;
791 return entry;
792}
793
794static void zswap_entry_cache_free(struct zswap_entry *entry)
795{
796 kmem_cache_free(zswap_entry_cache, entry);
797}
798
799/*
800 * Carries out the common pattern of freeing and entry's zpool allocation,
801 * freeing the entry itself, and decrementing the number of stored pages.
802 */
803static void zswap_entry_free(struct zswap_entry *entry)
804{
805 zswap_lru_del(&zswap_list_lru, entry);
806 zpool_free(entry->pool->zpool, entry->handle);
807 zswap_pool_put(entry->pool);
808 if (entry->objcg) {
809 obj_cgroup_uncharge_zswap(entry->objcg, entry->length);
810 obj_cgroup_put(entry->objcg);
811 }
812 zswap_entry_cache_free(entry);
813 atomic_long_dec(&zswap_stored_pages);
814}
815
816/*********************************
817* compressed storage functions
818**********************************/
819static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
820{
821 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
822 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
823 struct crypto_acomp *acomp = NULL;
824 struct acomp_req *req = NULL;
825 u8 *buffer = NULL;
826 int ret;
827
828 buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
829 if (!buffer) {
830 ret = -ENOMEM;
831 goto fail;
832 }
833
834 acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));
835 if (IS_ERR(acomp)) {
836 pr_err("could not alloc crypto acomp %s : %ld\n",
837 pool->tfm_name, PTR_ERR(acomp));
838 ret = PTR_ERR(acomp);
839 goto fail;
840 }
841
842 req = acomp_request_alloc(acomp);
843 if (!req) {
844 pr_err("could not alloc crypto acomp_request %s\n",
845 pool->tfm_name);
846 ret = -ENOMEM;
847 goto fail;
848 }
849
850 /*
851 * Only hold the mutex after completing allocations, otherwise we may
852 * recurse into zswap through reclaim and attempt to hold the mutex
853 * again resulting in a deadlock.
854 */
855 mutex_lock(&acomp_ctx->mutex);
856 crypto_init_wait(&acomp_ctx->wait);
857
858 /*
859 * if the backend of acomp is async zip, crypto_req_done() will wakeup
860 * crypto_wait_req(); if the backend of acomp is scomp, the callback
861 * won't be called, crypto_wait_req() will return without blocking.
862 */
863 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
864 crypto_req_done, &acomp_ctx->wait);
865
866 acomp_ctx->buffer = buffer;
867 acomp_ctx->acomp = acomp;
868 acomp_ctx->is_sleepable = acomp_is_async(acomp);
869 acomp_ctx->req = req;
870 mutex_unlock(&acomp_ctx->mutex);
871 return 0;
872
873fail:
874 if (acomp)
875 crypto_free_acomp(acomp);
876 kfree(buffer);
877 return ret;
878}
879
880static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
881{
882 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
883 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
884
885 mutex_lock(&acomp_ctx->mutex);
886 if (!IS_ERR_OR_NULL(acomp_ctx)) {
887 if (!IS_ERR_OR_NULL(acomp_ctx->req))
888 acomp_request_free(acomp_ctx->req);
889 acomp_ctx->req = NULL;
890 if (!IS_ERR_OR_NULL(acomp_ctx->acomp))
891 crypto_free_acomp(acomp_ctx->acomp);
892 kfree(acomp_ctx->buffer);
893 }
894 mutex_unlock(&acomp_ctx->mutex);
895
896 return 0;
897}
898
899static struct crypto_acomp_ctx *acomp_ctx_get_cpu_lock(struct zswap_pool *pool)
900{
901 struct crypto_acomp_ctx *acomp_ctx;
902
903 for (;;) {
904 acomp_ctx = raw_cpu_ptr(pool->acomp_ctx);
905 mutex_lock(&acomp_ctx->mutex);
906 if (likely(acomp_ctx->req))
907 return acomp_ctx;
908 /*
909 * It is possible that we were migrated to a different CPU after
910 * getting the per-CPU ctx but before the mutex was acquired. If
911 * the old CPU got offlined, zswap_cpu_comp_dead() could have
912 * already freed ctx->req (among other things) and set it to
913 * NULL. Just try again on the new CPU that we ended up on.
914 */
915 mutex_unlock(&acomp_ctx->mutex);
916 }
917}
918
919static void acomp_ctx_put_unlock(struct crypto_acomp_ctx *acomp_ctx)
920{
921 mutex_unlock(&acomp_ctx->mutex);
922}
923
924static bool zswap_compress(struct page *page, struct zswap_entry *entry,
925 struct zswap_pool *pool)
926{
927 struct crypto_acomp_ctx *acomp_ctx;
928 struct scatterlist input, output;
929 int comp_ret = 0, alloc_ret = 0;
930 unsigned int dlen = PAGE_SIZE;
931 unsigned long handle;
932 struct zpool *zpool;
933 char *buf;
934 gfp_t gfp;
935 u8 *dst;
936
937 acomp_ctx = acomp_ctx_get_cpu_lock(pool);
938 dst = acomp_ctx->buffer;
939 sg_init_table(&input, 1);
940 sg_set_page(&input, page, PAGE_SIZE, 0);
941
942 /*
943 * We need PAGE_SIZE * 2 here since there maybe over-compression case,
944 * and hardware-accelerators may won't check the dst buffer size, so
945 * giving the dst buffer with enough length to avoid buffer overflow.
946 */
947 sg_init_one(&output, dst, PAGE_SIZE * 2);
948 acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
949
950 /*
951 * it maybe looks a little bit silly that we send an asynchronous request,
952 * then wait for its completion synchronously. This makes the process look
953 * synchronous in fact.
954 * Theoretically, acomp supports users send multiple acomp requests in one
955 * acomp instance, then get those requests done simultaneously. but in this
956 * case, zswap actually does store and load page by page, there is no
957 * existing method to send the second page before the first page is done
958 * in one thread doing zwap.
959 * but in different threads running on different cpu, we have different
960 * acomp instance, so multiple threads can do (de)compression in parallel.
961 */
962 comp_ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);
963 dlen = acomp_ctx->req->dlen;
964 if (comp_ret)
965 goto unlock;
966
967 zpool = pool->zpool;
968 gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
969 if (zpool_malloc_support_movable(zpool))
970 gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;
971 alloc_ret = zpool_malloc(zpool, dlen, gfp, &handle);
972 if (alloc_ret)
973 goto unlock;
974
975 buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);
976 memcpy(buf, dst, dlen);
977 zpool_unmap_handle(zpool, handle);
978
979 entry->handle = handle;
980 entry->length = dlen;
981
982unlock:
983 if (comp_ret == -ENOSPC || alloc_ret == -ENOSPC)
984 zswap_reject_compress_poor++;
985 else if (comp_ret)
986 zswap_reject_compress_fail++;
987 else if (alloc_ret)
988 zswap_reject_alloc_fail++;
989
990 acomp_ctx_put_unlock(acomp_ctx);
991 return comp_ret == 0 && alloc_ret == 0;
992}
993
994static void zswap_decompress(struct zswap_entry *entry, struct folio *folio)
995{
996 struct zpool *zpool = entry->pool->zpool;
997 struct scatterlist input, output;
998 struct crypto_acomp_ctx *acomp_ctx;
999 u8 *src;
1000
1001 acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool);
1002 src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
1003 /*
1004 * If zpool_map_handle is atomic, we cannot reliably utilize its mapped buffer
1005 * to do crypto_acomp_decompress() which might sleep. In such cases, we must
1006 * resort to copying the buffer to a temporary one.
1007 * Meanwhile, zpool_map_handle() might return a non-linearly mapped buffer,
1008 * such as a kmap address of high memory or even ever a vmap address.
1009 * However, sg_init_one is only equipped to handle linearly mapped low memory.
1010 * In such cases, we also must copy the buffer to a temporary and lowmem one.
1011 */
1012 if ((acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) ||
1013 !virt_addr_valid(src)) {
1014 memcpy(acomp_ctx->buffer, src, entry->length);
1015 src = acomp_ctx->buffer;
1016 zpool_unmap_handle(zpool, entry->handle);
1017 }
1018
1019 sg_init_one(&input, src, entry->length);
1020 sg_init_table(&output, 1);
1021 sg_set_folio(&output, folio, PAGE_SIZE, 0);
1022 acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);
1023 BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));
1024 BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
1025
1026 if (src != acomp_ctx->buffer)
1027 zpool_unmap_handle(zpool, entry->handle);
1028 acomp_ctx_put_unlock(acomp_ctx);
1029}
1030
1031/*********************************
1032* writeback code
1033**********************************/
1034/*
1035 * Attempts to free an entry by adding a folio to the swap cache,
1036 * decompressing the entry data into the folio, and issuing a
1037 * bio write to write the folio back to the swap device.
1038 *
1039 * This can be thought of as a "resumed writeback" of the folio
1040 * to the swap device. We are basically resuming the same swap
1041 * writeback path that was intercepted with the zswap_store()
1042 * in the first place. After the folio has been decompressed into
1043 * the swap cache, the compressed version stored by zswap can be
1044 * freed.
1045 */
1046static int zswap_writeback_entry(struct zswap_entry *entry,
1047 swp_entry_t swpentry)
1048{
1049 struct xarray *tree;
1050 pgoff_t offset = swp_offset(swpentry);
1051 struct folio *folio;
1052 struct mempolicy *mpol;
1053 bool folio_was_allocated;
1054 struct writeback_control wbc = {
1055 .sync_mode = WB_SYNC_NONE,
1056 };
1057
1058 /* try to allocate swap cache folio */
1059 mpol = get_task_policy(current);
1060 folio = __read_swap_cache_async(swpentry, GFP_KERNEL, mpol,
1061 NO_INTERLEAVE_INDEX, &folio_was_allocated, true);
1062 if (!folio)
1063 return -ENOMEM;
1064
1065 /*
1066 * Found an existing folio, we raced with swapin or concurrent
1067 * shrinker. We generally writeback cold folios from zswap, and
1068 * swapin means the folio just became hot, so skip this folio.
1069 * For unlikely concurrent shrinker case, it will be unlinked
1070 * and freed when invalidated by the concurrent shrinker anyway.
1071 */
1072 if (!folio_was_allocated) {
1073 folio_put(folio);
1074 return -EEXIST;
1075 }
1076
1077 /*
1078 * folio is locked, and the swapcache is now secured against
1079 * concurrent swapping to and from the slot, and concurrent
1080 * swapoff so we can safely dereference the zswap tree here.
1081 * Verify that the swap entry hasn't been invalidated and recycled
1082 * behind our backs, to avoid overwriting a new swap folio with
1083 * old compressed data. Only when this is successful can the entry
1084 * be dereferenced.
1085 */
1086 tree = swap_zswap_tree(swpentry);
1087 if (entry != xa_cmpxchg(tree, offset, entry, NULL, GFP_KERNEL)) {
1088 delete_from_swap_cache(folio);
1089 folio_unlock(folio);
1090 folio_put(folio);
1091 return -ENOMEM;
1092 }
1093
1094 zswap_decompress(entry, folio);
1095
1096 count_vm_event(ZSWPWB);
1097 if (entry->objcg)
1098 count_objcg_events(entry->objcg, ZSWPWB, 1);
1099
1100 zswap_entry_free(entry);
1101
1102 /* folio is up to date */
1103 folio_mark_uptodate(folio);
1104
1105 /* move it to the tail of the inactive list after end_writeback */
1106 folio_set_reclaim(folio);
1107
1108 /* start writeback */
1109 __swap_writepage(folio, &wbc);
1110 folio_put(folio);
1111
1112 return 0;
1113}
1114
1115/*********************************
1116* shrinker functions
1117**********************************/
1118/*
1119 * The dynamic shrinker is modulated by the following factors:
1120 *
1121 * 1. Each zswap entry has a referenced bit, which the shrinker unsets (giving
1122 * the entry a second chance) before rotating it in the LRU list. If the
1123 * entry is considered again by the shrinker, with its referenced bit unset,
1124 * it is written back. The writeback rate as a result is dynamically
1125 * adjusted by the pool activities - if the pool is dominated by new entries
1126 * (i.e lots of recent zswapouts), these entries will be protected and
1127 * the writeback rate will slow down. On the other hand, if the pool has a
1128 * lot of stagnant entries, these entries will be reclaimed immediately,
1129 * effectively increasing the writeback rate.
1130 *
1131 * 2. Swapins counter: If we observe swapins, it is a sign that we are
1132 * overshrinking and should slow down. We maintain a swapins counter, which
1133 * is consumed and subtract from the number of eligible objects on the LRU
1134 * in zswap_shrinker_count().
1135 *
1136 * 3. Compression ratio. The better the workload compresses, the less gains we
1137 * can expect from writeback. We scale down the number of objects available
1138 * for reclaim by this ratio.
1139 */
1140static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_one *l,
1141 void *arg)
1142{
1143 struct zswap_entry *entry = container_of(item, struct zswap_entry, lru);
1144 bool *encountered_page_in_swapcache = (bool *)arg;
1145 swp_entry_t swpentry;
1146 enum lru_status ret = LRU_REMOVED_RETRY;
1147 int writeback_result;
1148
1149 /*
1150 * Second chance algorithm: if the entry has its referenced bit set, give it
1151 * a second chance. Only clear the referenced bit and rotate it in the
1152 * zswap's LRU list.
1153 */
1154 if (entry->referenced) {
1155 entry->referenced = false;
1156 return LRU_ROTATE;
1157 }
1158
1159 /*
1160 * As soon as we drop the LRU lock, the entry can be freed by
1161 * a concurrent invalidation. This means the following:
1162 *
1163 * 1. We extract the swp_entry_t to the stack, allowing
1164 * zswap_writeback_entry() to pin the swap entry and
1165 * then validate the zwap entry against that swap entry's
1166 * tree using pointer value comparison. Only when that
1167 * is successful can the entry be dereferenced.
1168 *
1169 * 2. Usually, objects are taken off the LRU for reclaim. In
1170 * this case this isn't possible, because if reclaim fails
1171 * for whatever reason, we have no means of knowing if the
1172 * entry is alive to put it back on the LRU.
1173 *
1174 * So rotate it before dropping the lock. If the entry is
1175 * written back or invalidated, the free path will unlink
1176 * it. For failures, rotation is the right thing as well.
1177 *
1178 * Temporary failures, where the same entry should be tried
1179 * again immediately, almost never happen for this shrinker.
1180 * We don't do any trylocking; -ENOMEM comes closest,
1181 * but that's extremely rare and doesn't happen spuriously
1182 * either. Don't bother distinguishing this case.
1183 */
1184 list_move_tail(item, &l->list);
1185
1186 /*
1187 * Once the lru lock is dropped, the entry might get freed. The
1188 * swpentry is copied to the stack, and entry isn't deref'd again
1189 * until the entry is verified to still be alive in the tree.
1190 */
1191 swpentry = entry->swpentry;
1192
1193 /*
1194 * It's safe to drop the lock here because we return either
1195 * LRU_REMOVED_RETRY or LRU_RETRY.
1196 */
1197 spin_unlock(&l->lock);
1198
1199 writeback_result = zswap_writeback_entry(entry, swpentry);
1200
1201 if (writeback_result) {
1202 zswap_reject_reclaim_fail++;
1203 ret = LRU_RETRY;
1204
1205 /*
1206 * Encountering a page already in swap cache is a sign that we are shrinking
1207 * into the warmer region. We should terminate shrinking (if we're in the dynamic
1208 * shrinker context).
1209 */
1210 if (writeback_result == -EEXIST && encountered_page_in_swapcache) {
1211 ret = LRU_STOP;
1212 *encountered_page_in_swapcache = true;
1213 }
1214 } else {
1215 zswap_written_back_pages++;
1216 }
1217
1218 return ret;
1219}
1220
1221static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
1222 struct shrink_control *sc)
1223{
1224 unsigned long shrink_ret;
1225 bool encountered_page_in_swapcache = false;
1226
1227 if (!zswap_shrinker_enabled ||
1228 !mem_cgroup_zswap_writeback_enabled(sc->memcg)) {
1229 sc->nr_scanned = 0;
1230 return SHRINK_STOP;
1231 }
1232
1233 shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb,
1234 &encountered_page_in_swapcache);
1235
1236 if (encountered_page_in_swapcache)
1237 return SHRINK_STOP;
1238
1239 return shrink_ret ? shrink_ret : SHRINK_STOP;
1240}
1241
1242static unsigned long zswap_shrinker_count(struct shrinker *shrinker,
1243 struct shrink_control *sc)
1244{
1245 struct mem_cgroup *memcg = sc->memcg;
1246 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid));
1247 atomic_long_t *nr_disk_swapins =
1248 &lruvec->zswap_lruvec_state.nr_disk_swapins;
1249 unsigned long nr_backing, nr_stored, nr_freeable, nr_disk_swapins_cur,
1250 nr_remain;
1251
1252 if (!zswap_shrinker_enabled || !mem_cgroup_zswap_writeback_enabled(memcg))
1253 return 0;
1254
1255 /*
1256 * The shrinker resumes swap writeback, which will enter block
1257 * and may enter fs. XXX: Harmonize with vmscan.c __GFP_FS
1258 * rules (may_enter_fs()), which apply on a per-folio basis.
1259 */
1260 if (!gfp_has_io_fs(sc->gfp_mask))
1261 return 0;
1262
1263 /*
1264 * For memcg, use the cgroup-wide ZSWAP stats since we don't
1265 * have them per-node and thus per-lruvec. Careful if memcg is
1266 * runtime-disabled: we can get sc->memcg == NULL, which is ok
1267 * for the lruvec, but not for memcg_page_state().
1268 *
1269 * Without memcg, use the zswap pool-wide metrics.
1270 */
1271 if (!mem_cgroup_disabled()) {
1272 mem_cgroup_flush_stats(memcg);
1273 nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT;
1274 nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED);
1275 } else {
1276 nr_backing = zswap_total_pages();
1277 nr_stored = atomic_long_read(&zswap_stored_pages);
1278 }
1279
1280 if (!nr_stored)
1281 return 0;
1282
1283 nr_freeable = list_lru_shrink_count(&zswap_list_lru, sc);
1284 if (!nr_freeable)
1285 return 0;
1286
1287 /*
1288 * Subtract from the lru size the number of pages that are recently swapped
1289 * in from disk. The idea is that had we protect the zswap's LRU by this
1290 * amount of pages, these disk swapins would not have happened.
1291 */
1292 nr_disk_swapins_cur = atomic_long_read(nr_disk_swapins);
1293 do {
1294 if (nr_freeable >= nr_disk_swapins_cur)
1295 nr_remain = 0;
1296 else
1297 nr_remain = nr_disk_swapins_cur - nr_freeable;
1298 } while (!atomic_long_try_cmpxchg(
1299 nr_disk_swapins, &nr_disk_swapins_cur, nr_remain));
1300
1301 nr_freeable -= nr_disk_swapins_cur - nr_remain;
1302 if (!nr_freeable)
1303 return 0;
1304
1305 /*
1306 * Scale the number of freeable pages by the memory saving factor.
1307 * This ensures that the better zswap compresses memory, the fewer
1308 * pages we will evict to swap (as it will otherwise incur IO for
1309 * relatively small memory saving).
1310 */
1311 return mult_frac(nr_freeable, nr_backing, nr_stored);
1312}
1313
1314static struct shrinker *zswap_alloc_shrinker(void)
1315{
1316 struct shrinker *shrinker;
1317
1318 shrinker =
1319 shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, "mm-zswap");
1320 if (!shrinker)
1321 return NULL;
1322
1323 shrinker->scan_objects = zswap_shrinker_scan;
1324 shrinker->count_objects = zswap_shrinker_count;
1325 shrinker->batch = 0;
1326 shrinker->seeks = DEFAULT_SEEKS;
1327 return shrinker;
1328}
1329
1330static int shrink_memcg(struct mem_cgroup *memcg)
1331{
1332 int nid, shrunk = 0, scanned = 0;
1333
1334 if (!mem_cgroup_zswap_writeback_enabled(memcg))
1335 return -ENOENT;
1336
1337 /*
1338 * Skip zombies because their LRUs are reparented and we would be
1339 * reclaiming from the parent instead of the dead memcg.
1340 */
1341 if (memcg && !mem_cgroup_online(memcg))
1342 return -ENOENT;
1343
1344 for_each_node_state(nid, N_NORMAL_MEMORY) {
1345 unsigned long nr_to_walk = 1;
1346
1347 shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg,
1348 &shrink_memcg_cb, NULL, &nr_to_walk);
1349 scanned += 1 - nr_to_walk;
1350 }
1351
1352 if (!scanned)
1353 return -ENOENT;
1354
1355 return shrunk ? 0 : -EAGAIN;
1356}
1357
1358static void shrink_worker(struct work_struct *w)
1359{
1360 struct mem_cgroup *memcg;
1361 int ret, failures = 0, attempts = 0;
1362 unsigned long thr;
1363
1364 /* Reclaim down to the accept threshold */
1365 thr = zswap_accept_thr_pages();
1366
1367 /*
1368 * Global reclaim will select cgroup in a round-robin fashion from all
1369 * online memcgs, but memcgs that have no pages in zswap and
1370 * writeback-disabled memcgs (memory.zswap.writeback=0) are not
1371 * candidates for shrinking.
1372 *
1373 * Shrinking will be aborted if we encounter the following
1374 * MAX_RECLAIM_RETRIES times:
1375 * - No writeback-candidate memcgs found in a memcg tree walk.
1376 * - Shrinking a writeback-candidate memcg failed.
1377 *
1378 * We save iteration cursor memcg into zswap_next_shrink,
1379 * which can be modified by the offline memcg cleaner
1380 * zswap_memcg_offline_cleanup().
1381 *
1382 * Since the offline cleaner is called only once, we cannot leave an
1383 * offline memcg reference in zswap_next_shrink.
1384 * We can rely on the cleaner only if we get online memcg under lock.
1385 *
1386 * If we get an offline memcg, we cannot determine if the cleaner has
1387 * already been called or will be called later. We must put back the
1388 * reference before returning from this function. Otherwise, the
1389 * offline memcg left in zswap_next_shrink will hold the reference
1390 * until the next run of shrink_worker().
1391 */
1392 do {
1393 /*
1394 * Start shrinking from the next memcg after zswap_next_shrink.
1395 * When the offline cleaner has already advanced the cursor,
1396 * advancing the cursor here overlooks one memcg, but this
1397 * should be negligibly rare.
1398 *
1399 * If we get an online memcg, keep the extra reference in case
1400 * the original one obtained by mem_cgroup_iter() is dropped by
1401 * zswap_memcg_offline_cleanup() while we are shrinking the
1402 * memcg.
1403 */
1404 spin_lock(&zswap_shrink_lock);
1405 do {
1406 memcg = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
1407 zswap_next_shrink = memcg;
1408 } while (memcg && !mem_cgroup_tryget_online(memcg));
1409 spin_unlock(&zswap_shrink_lock);
1410
1411 if (!memcg) {
1412 /*
1413 * Continue shrinking without incrementing failures if
1414 * we found candidate memcgs in the last tree walk.
1415 */
1416 if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
1417 break;
1418
1419 attempts = 0;
1420 goto resched;
1421 }
1422
1423 ret = shrink_memcg(memcg);
1424 /* drop the extra reference */
1425 mem_cgroup_put(memcg);
1426
1427 /*
1428 * There are no writeback-candidate pages in the memcg.
1429 * This is not an issue as long as we can find another memcg
1430 * with pages in zswap. Skip this without incrementing attempts
1431 * and failures.
1432 */
1433 if (ret == -ENOENT)
1434 continue;
1435 ++attempts;
1436
1437 if (ret && ++failures == MAX_RECLAIM_RETRIES)
1438 break;
1439resched:
1440 cond_resched();
1441 } while (zswap_total_pages() > thr);
1442}
1443
1444/*********************************
1445* main API
1446**********************************/
1447
1448static bool zswap_store_page(struct page *page,
1449 struct obj_cgroup *objcg,
1450 struct zswap_pool *pool)
1451{
1452 swp_entry_t page_swpentry = page_swap_entry(page);
1453 struct zswap_entry *entry, *old;
1454
1455 /* allocate entry */
1456 entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
1457 if (!entry) {
1458 zswap_reject_kmemcache_fail++;
1459 return false;
1460 }
1461
1462 if (!zswap_compress(page, entry, pool))
1463 goto compress_failed;
1464
1465 old = xa_store(swap_zswap_tree(page_swpentry),
1466 swp_offset(page_swpentry),
1467 entry, GFP_KERNEL);
1468 if (xa_is_err(old)) {
1469 int err = xa_err(old);
1470
1471 WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
1472 zswap_reject_alloc_fail++;
1473 goto store_failed;
1474 }
1475
1476 /*
1477 * We may have had an existing entry that became stale when
1478 * the folio was redirtied and now the new version is being
1479 * swapped out. Get rid of the old.
1480 */
1481 if (old)
1482 zswap_entry_free(old);
1483
1484 /*
1485 * The entry is successfully compressed and stored in the tree, there is
1486 * no further possibility of failure. Grab refs to the pool and objcg,
1487 * charge zswap memory, and increment zswap_stored_pages.
1488 * The opposite actions will be performed by zswap_entry_free()
1489 * when the entry is removed from the tree.
1490 */
1491 zswap_pool_get(pool);
1492 if (objcg) {
1493 obj_cgroup_get(objcg);
1494 obj_cgroup_charge_zswap(objcg, entry->length);
1495 }
1496 atomic_long_inc(&zswap_stored_pages);
1497
1498 /*
1499 * We finish initializing the entry while it's already in xarray.
1500 * This is safe because:
1501 *
1502 * 1. Concurrent stores and invalidations are excluded by folio lock.
1503 *
1504 * 2. Writeback is excluded by the entry not being on the LRU yet.
1505 * The publishing order matters to prevent writeback from seeing
1506 * an incoherent entry.
1507 */
1508 entry->pool = pool;
1509 entry->swpentry = page_swpentry;
1510 entry->objcg = objcg;
1511 entry->referenced = true;
1512 if (entry->length) {
1513 INIT_LIST_HEAD(&entry->lru);
1514 zswap_lru_add(&zswap_list_lru, entry);
1515 }
1516
1517 return true;
1518
1519store_failed:
1520 zpool_free(pool->zpool, entry->handle);
1521compress_failed:
1522 zswap_entry_cache_free(entry);
1523 return false;
1524}
1525
1526bool zswap_store(struct folio *folio)
1527{
1528 long nr_pages = folio_nr_pages(folio);
1529 swp_entry_t swp = folio->swap;
1530 struct obj_cgroup *objcg = NULL;
1531 struct mem_cgroup *memcg = NULL;
1532 struct zswap_pool *pool;
1533 bool ret = false;
1534 long index;
1535
1536 VM_WARN_ON_ONCE(!folio_test_locked(folio));
1537 VM_WARN_ON_ONCE(!folio_test_swapcache(folio));
1538
1539 if (!zswap_enabled)
1540 goto check_old;
1541
1542 objcg = get_obj_cgroup_from_folio(folio);
1543 if (objcg && !obj_cgroup_may_zswap(objcg)) {
1544 memcg = get_mem_cgroup_from_objcg(objcg);
1545 if (shrink_memcg(memcg)) {
1546 mem_cgroup_put(memcg);
1547 goto put_objcg;
1548 }
1549 mem_cgroup_put(memcg);
1550 }
1551
1552 if (zswap_check_limits())
1553 goto put_objcg;
1554
1555 pool = zswap_pool_current_get();
1556 if (!pool)
1557 goto put_objcg;
1558
1559 if (objcg) {
1560 memcg = get_mem_cgroup_from_objcg(objcg);
1561 if (memcg_list_lru_alloc(memcg, &zswap_list_lru, GFP_KERNEL)) {
1562 mem_cgroup_put(memcg);
1563 goto put_pool;
1564 }
1565 mem_cgroup_put(memcg);
1566 }
1567
1568 for (index = 0; index < nr_pages; ++index) {
1569 struct page *page = folio_page(folio, index);
1570
1571 if (!zswap_store_page(page, objcg, pool))
1572 goto put_pool;
1573 }
1574
1575 if (objcg)
1576 count_objcg_events(objcg, ZSWPOUT, nr_pages);
1577
1578 count_vm_events(ZSWPOUT, nr_pages);
1579
1580 ret = true;
1581
1582put_pool:
1583 zswap_pool_put(pool);
1584put_objcg:
1585 obj_cgroup_put(objcg);
1586 if (!ret && zswap_pool_reached_full)
1587 queue_work(shrink_wq, &zswap_shrink_work);
1588check_old:
1589 /*
1590 * If the zswap store fails or zswap is disabled, we must invalidate
1591 * the possibly stale entries which were previously stored at the
1592 * offsets corresponding to each page of the folio. Otherwise,
1593 * writeback could overwrite the new data in the swapfile.
1594 */
1595 if (!ret) {
1596 unsigned type = swp_type(swp);
1597 pgoff_t offset = swp_offset(swp);
1598 struct zswap_entry *entry;
1599 struct xarray *tree;
1600
1601 for (index = 0; index < nr_pages; ++index) {
1602 tree = swap_zswap_tree(swp_entry(type, offset + index));
1603 entry = xa_erase(tree, offset + index);
1604 if (entry)
1605 zswap_entry_free(entry);
1606 }
1607 }
1608
1609 return ret;
1610}
1611
1612bool zswap_load(struct folio *folio)
1613{
1614 swp_entry_t swp = folio->swap;
1615 pgoff_t offset = swp_offset(swp);
1616 bool swapcache = folio_test_swapcache(folio);
1617 struct xarray *tree = swap_zswap_tree(swp);
1618 struct zswap_entry *entry;
1619
1620 VM_WARN_ON_ONCE(!folio_test_locked(folio));
1621
1622 if (zswap_never_enabled())
1623 return false;
1624
1625 /*
1626 * Large folios should not be swapped in while zswap is being used, as
1627 * they are not properly handled. Zswap does not properly load large
1628 * folios, and a large folio may only be partially in zswap.
1629 *
1630 * Return true without marking the folio uptodate so that an IO error is
1631 * emitted (e.g. do_swap_page() will sigbus).
1632 */
1633 if (WARN_ON_ONCE(folio_test_large(folio)))
1634 return true;
1635
1636 /*
1637 * When reading into the swapcache, invalidate our entry. The
1638 * swapcache can be the authoritative owner of the page and
1639 * its mappings, and the pressure that results from having two
1640 * in-memory copies outweighs any benefits of caching the
1641 * compression work.
1642 *
1643 * (Most swapins go through the swapcache. The notable
1644 * exception is the singleton fault on SWP_SYNCHRONOUS_IO
1645 * files, which reads into a private page and may free it if
1646 * the fault fails. We remain the primary owner of the entry.)
1647 */
1648 if (swapcache)
1649 entry = xa_erase(tree, offset);
1650 else
1651 entry = xa_load(tree, offset);
1652
1653 if (!entry)
1654 return false;
1655
1656 zswap_decompress(entry, folio);
1657
1658 count_vm_event(ZSWPIN);
1659 if (entry->objcg)
1660 count_objcg_events(entry->objcg, ZSWPIN, 1);
1661
1662 if (swapcache) {
1663 zswap_entry_free(entry);
1664 folio_mark_dirty(folio);
1665 }
1666
1667 folio_mark_uptodate(folio);
1668 return true;
1669}
1670
1671void zswap_invalidate(swp_entry_t swp)
1672{
1673 pgoff_t offset = swp_offset(swp);
1674 struct xarray *tree = swap_zswap_tree(swp);
1675 struct zswap_entry *entry;
1676
1677 if (xa_empty(tree))
1678 return;
1679
1680 entry = xa_erase(tree, offset);
1681 if (entry)
1682 zswap_entry_free(entry);
1683}
1684
1685int zswap_swapon(int type, unsigned long nr_pages)
1686{
1687 struct xarray *trees, *tree;
1688 unsigned int nr, i;
1689
1690 nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);
1691 trees = kvcalloc(nr, sizeof(*tree), GFP_KERNEL);
1692 if (!trees) {
1693 pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1694 return -ENOMEM;
1695 }
1696
1697 for (i = 0; i < nr; i++)
1698 xa_init(trees + i);
1699
1700 nr_zswap_trees[type] = nr;
1701 zswap_trees[type] = trees;
1702 return 0;
1703}
1704
1705void zswap_swapoff(int type)
1706{
1707 struct xarray *trees = zswap_trees[type];
1708 unsigned int i;
1709
1710 if (!trees)
1711 return;
1712
1713 /* try_to_unuse() invalidated all the entries already */
1714 for (i = 0; i < nr_zswap_trees[type]; i++)
1715 WARN_ON_ONCE(!xa_empty(trees + i));
1716
1717 kvfree(trees);
1718 nr_zswap_trees[type] = 0;
1719 zswap_trees[type] = NULL;
1720}
1721
1722/*********************************
1723* debugfs functions
1724**********************************/
1725#ifdef CONFIG_DEBUG_FS
1726#include <linux/debugfs.h>
1727
1728static struct dentry *zswap_debugfs_root;
1729
1730static int debugfs_get_total_size(void *data, u64 *val)
1731{
1732 *val = zswap_total_pages() * PAGE_SIZE;
1733 return 0;
1734}
1735DEFINE_DEBUGFS_ATTRIBUTE(total_size_fops, debugfs_get_total_size, NULL, "%llu\n");
1736
1737static int debugfs_get_stored_pages(void *data, u64 *val)
1738{
1739 *val = atomic_long_read(&zswap_stored_pages);
1740 return 0;
1741}
1742DEFINE_DEBUGFS_ATTRIBUTE(stored_pages_fops, debugfs_get_stored_pages, NULL, "%llu\n");
1743
1744static int zswap_debugfs_init(void)
1745{
1746 if (!debugfs_initialized())
1747 return -ENODEV;
1748
1749 zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1750
1751 debugfs_create_u64("pool_limit_hit", 0444,
1752 zswap_debugfs_root, &zswap_pool_limit_hit);
1753 debugfs_create_u64("reject_reclaim_fail", 0444,
1754 zswap_debugfs_root, &zswap_reject_reclaim_fail);
1755 debugfs_create_u64("reject_alloc_fail", 0444,
1756 zswap_debugfs_root, &zswap_reject_alloc_fail);
1757 debugfs_create_u64("reject_kmemcache_fail", 0444,
1758 zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1759 debugfs_create_u64("reject_compress_fail", 0444,
1760 zswap_debugfs_root, &zswap_reject_compress_fail);
1761 debugfs_create_u64("reject_compress_poor", 0444,
1762 zswap_debugfs_root, &zswap_reject_compress_poor);
1763 debugfs_create_u64("written_back_pages", 0444,
1764 zswap_debugfs_root, &zswap_written_back_pages);
1765 debugfs_create_file("pool_total_size", 0444,
1766 zswap_debugfs_root, NULL, &total_size_fops);
1767 debugfs_create_file("stored_pages", 0444,
1768 zswap_debugfs_root, NULL, &stored_pages_fops);
1769
1770 return 0;
1771}
1772#else
1773static int zswap_debugfs_init(void)
1774{
1775 return 0;
1776}
1777#endif
1778
1779/*********************************
1780* module init and exit
1781**********************************/
1782static int zswap_setup(void)
1783{
1784 struct zswap_pool *pool;
1785 int ret;
1786
1787 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
1788 if (!zswap_entry_cache) {
1789 pr_err("entry cache creation failed\n");
1790 goto cache_fail;
1791 }
1792
1793 ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
1794 "mm/zswap_pool:prepare",
1795 zswap_cpu_comp_prepare,
1796 zswap_cpu_comp_dead);
1797 if (ret)
1798 goto hp_fail;
1799
1800 shrink_wq = alloc_workqueue("zswap-shrink",
1801 WQ_UNBOUND|WQ_MEM_RECLAIM, 1);
1802 if (!shrink_wq)
1803 goto shrink_wq_fail;
1804
1805 zswap_shrinker = zswap_alloc_shrinker();
1806 if (!zswap_shrinker)
1807 goto shrinker_fail;
1808 if (list_lru_init_memcg(&zswap_list_lru, zswap_shrinker))
1809 goto lru_fail;
1810 shrinker_register(zswap_shrinker);
1811
1812 INIT_WORK(&zswap_shrink_work, shrink_worker);
1813
1814 pool = __zswap_pool_create_fallback();
1815 if (pool) {
1816 pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1817 zpool_get_type(pool->zpool));
1818 list_add(&pool->list, &zswap_pools);
1819 zswap_has_pool = true;
1820 static_branch_enable(&zswap_ever_enabled);
1821 } else {
1822 pr_err("pool creation failed\n");
1823 zswap_enabled = false;
1824 }
1825
1826 if (zswap_debugfs_init())
1827 pr_warn("debugfs initialization failed\n");
1828 zswap_init_state = ZSWAP_INIT_SUCCEED;
1829 return 0;
1830
1831lru_fail:
1832 shrinker_free(zswap_shrinker);
1833shrinker_fail:
1834 destroy_workqueue(shrink_wq);
1835shrink_wq_fail:
1836 cpuhp_remove_multi_state(CPUHP_MM_ZSWP_POOL_PREPARE);
1837hp_fail:
1838 kmem_cache_destroy(zswap_entry_cache);
1839cache_fail:
1840 /* if built-in, we aren't unloaded on failure; don't allow use */
1841 zswap_init_state = ZSWAP_INIT_FAILED;
1842 zswap_enabled = false;
1843 return -ENOMEM;
1844}
1845
1846static int __init zswap_init(void)
1847{
1848 if (!zswap_enabled)
1849 return 0;
1850 return zswap_setup();
1851}
1852/* must be late so crypto has time to come up */
1853late_initcall(zswap_init);
1854
1855MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1856MODULE_DESCRIPTION("Compressed cache for swap pages");