Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * zswap.c - zswap driver file
4 *
5 * zswap is a backend for frontswap that takes pages that are in the process
6 * of being swapped out and attempts to compress and store them in a
7 * RAM-based memory pool. This can result in a significant I/O reduction on
8 * the swap device and, in the case where decompressing from RAM is faster
9 * than reading from the swap device, can also improve workload performance.
10 *
11 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com>
12*/
13
14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16#include <linux/module.h>
17#include <linux/cpu.h>
18#include <linux/highmem.h>
19#include <linux/slab.h>
20#include <linux/spinlock.h>
21#include <linux/types.h>
22#include <linux/atomic.h>
23#include <linux/frontswap.h>
24#include <linux/rbtree.h>
25#include <linux/swap.h>
26#include <linux/crypto.h>
27#include <linux/scatterlist.h>
28#include <linux/mempool.h>
29#include <linux/zpool.h>
30#include <crypto/acompress.h>
31
32#include <linux/mm_types.h>
33#include <linux/page-flags.h>
34#include <linux/swapops.h>
35#include <linux/writeback.h>
36#include <linux/pagemap.h>
37#include <linux/workqueue.h>
38
39#include "swap.h"
40
41/*********************************
42* statistics
43**********************************/
44/* Total bytes used by the compressed storage */
45u64 zswap_pool_total_size;
46/* The number of compressed pages currently stored in zswap */
47atomic_t zswap_stored_pages = ATOMIC_INIT(0);
48/* The number of same-value filled pages currently stored in zswap */
49static atomic_t zswap_same_filled_pages = ATOMIC_INIT(0);
50
51/*
52 * The statistics below are not protected from concurrent access for
53 * performance reasons so they may not be a 100% accurate. However,
54 * they do provide useful information on roughly how many times a
55 * certain event is occurring.
56*/
57
58/* Pool limit was hit (see zswap_max_pool_percent) */
59static u64 zswap_pool_limit_hit;
60/* Pages written back when pool limit was reached */
61static u64 zswap_written_back_pages;
62/* Store failed due to a reclaim failure after pool limit was reached */
63static u64 zswap_reject_reclaim_fail;
64/* Compressed page was too big for the allocator to (optimally) store */
65static u64 zswap_reject_compress_poor;
66/* Store failed because underlying allocator could not get memory */
67static u64 zswap_reject_alloc_fail;
68/* Store failed because the entry metadata could not be allocated (rare) */
69static u64 zswap_reject_kmemcache_fail;
70/* Duplicate store was encountered (rare) */
71static u64 zswap_duplicate_entry;
72
73/* Shrinker work queue */
74static struct workqueue_struct *shrink_wq;
75/* Pool limit was hit, we need to calm down */
76static bool zswap_pool_reached_full;
77
78/*********************************
79* tunables
80**********************************/
81
82#define ZSWAP_PARAM_UNSET ""
83
84/* Enable/disable zswap */
85static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON);
86static int zswap_enabled_param_set(const char *,
87 const struct kernel_param *);
88static const struct kernel_param_ops zswap_enabled_param_ops = {
89 .set = zswap_enabled_param_set,
90 .get = param_get_bool,
91};
92module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
93
94/* Crypto compressor to use */
95static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
96static int zswap_compressor_param_set(const char *,
97 const struct kernel_param *);
98static const struct kernel_param_ops zswap_compressor_param_ops = {
99 .set = zswap_compressor_param_set,
100 .get = param_get_charp,
101 .free = param_free_charp,
102};
103module_param_cb(compressor, &zswap_compressor_param_ops,
104 &zswap_compressor, 0644);
105
106/* Compressed storage zpool to use */
107static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
108static int zswap_zpool_param_set(const char *, const struct kernel_param *);
109static const struct kernel_param_ops zswap_zpool_param_ops = {
110 .set = zswap_zpool_param_set,
111 .get = param_get_charp,
112 .free = param_free_charp,
113};
114module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
115
116/* The maximum percentage of memory that the compressed pool can occupy */
117static unsigned int zswap_max_pool_percent = 20;
118module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
119
120/* The threshold for accepting new pages after the max_pool_percent was hit */
121static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */
122module_param_named(accept_threshold_percent, zswap_accept_thr_percent,
123 uint, 0644);
124
125/*
126 * Enable/disable handling same-value filled pages (enabled by default).
127 * If disabled every page is considered non-same-value filled.
128 */
129static bool zswap_same_filled_pages_enabled = true;
130module_param_named(same_filled_pages_enabled, zswap_same_filled_pages_enabled,
131 bool, 0644);
132
133/* Enable/disable handling non-same-value filled pages (enabled by default) */
134static bool zswap_non_same_filled_pages_enabled = true;
135module_param_named(non_same_filled_pages_enabled, zswap_non_same_filled_pages_enabled,
136 bool, 0644);
137
138/*********************************
139* data structures
140**********************************/
141
142struct crypto_acomp_ctx {
143 struct crypto_acomp *acomp;
144 struct acomp_req *req;
145 struct crypto_wait wait;
146 u8 *dstmem;
147 struct mutex *mutex;
148};
149
150struct zswap_pool {
151 struct zpool *zpool;
152 struct crypto_acomp_ctx __percpu *acomp_ctx;
153 struct kref kref;
154 struct list_head list;
155 struct work_struct release_work;
156 struct work_struct shrink_work;
157 struct hlist_node node;
158 char tfm_name[CRYPTO_MAX_ALG_NAME];
159};
160
161/*
162 * struct zswap_entry
163 *
164 * This structure contains the metadata for tracking a single compressed
165 * page within zswap.
166 *
167 * rbnode - links the entry into red-black tree for the appropriate swap type
168 * offset - the swap offset for the entry. Index into the red-black tree.
169 * refcount - the number of outstanding reference to the entry. This is needed
170 * to protect against premature freeing of the entry by code
171 * concurrent calls to load, invalidate, and writeback. The lock
172 * for the zswap_tree structure that contains the entry must
173 * be held while changing the refcount. Since the lock must
174 * be held, there is no reason to also make refcount atomic.
175 * length - the length in bytes of the compressed page data. Needed during
176 * decompression. For a same value filled page length is 0.
177 * pool - the zswap_pool the entry's data is in
178 * handle - zpool allocation handle that stores the compressed page data
179 * value - value of the same-value filled pages which have same content
180 */
181struct zswap_entry {
182 struct rb_node rbnode;
183 pgoff_t offset;
184 int refcount;
185 unsigned int length;
186 struct zswap_pool *pool;
187 union {
188 unsigned long handle;
189 unsigned long value;
190 };
191 struct obj_cgroup *objcg;
192};
193
194struct zswap_header {
195 swp_entry_t swpentry;
196};
197
198/*
199 * The tree lock in the zswap_tree struct protects a few things:
200 * - the rbtree
201 * - the refcount field of each entry in the tree
202 */
203struct zswap_tree {
204 struct rb_root rbroot;
205 spinlock_t lock;
206};
207
208static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
209
210/* RCU-protected iteration */
211static LIST_HEAD(zswap_pools);
212/* protects zswap_pools list modification */
213static DEFINE_SPINLOCK(zswap_pools_lock);
214/* pool counter to provide unique names to zpool */
215static atomic_t zswap_pools_count = ATOMIC_INIT(0);
216
217/* used by param callback function */
218static bool zswap_init_started;
219
220/* fatal error during init */
221static bool zswap_init_failed;
222
223/* init completed, but couldn't create the initial pool */
224static bool zswap_has_pool;
225
226/*********************************
227* helpers and fwd declarations
228**********************************/
229
230#define zswap_pool_debug(msg, p) \
231 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
232 zpool_get_type((p)->zpool))
233
234static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
235static int zswap_pool_get(struct zswap_pool *pool);
236static void zswap_pool_put(struct zswap_pool *pool);
237
238static const struct zpool_ops zswap_zpool_ops = {
239 .evict = zswap_writeback_entry
240};
241
242static bool zswap_is_full(void)
243{
244 return totalram_pages() * zswap_max_pool_percent / 100 <
245 DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
246}
247
248static bool zswap_can_accept(void)
249{
250 return totalram_pages() * zswap_accept_thr_percent / 100 *
251 zswap_max_pool_percent / 100 >
252 DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
253}
254
255static void zswap_update_total_size(void)
256{
257 struct zswap_pool *pool;
258 u64 total = 0;
259
260 rcu_read_lock();
261
262 list_for_each_entry_rcu(pool, &zswap_pools, list)
263 total += zpool_get_total_size(pool->zpool);
264
265 rcu_read_unlock();
266
267 zswap_pool_total_size = total;
268}
269
270/*********************************
271* zswap entry functions
272**********************************/
273static struct kmem_cache *zswap_entry_cache;
274
275static int __init zswap_entry_cache_create(void)
276{
277 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
278 return zswap_entry_cache == NULL;
279}
280
281static void __init zswap_entry_cache_destroy(void)
282{
283 kmem_cache_destroy(zswap_entry_cache);
284}
285
286static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
287{
288 struct zswap_entry *entry;
289 entry = kmem_cache_alloc(zswap_entry_cache, gfp);
290 if (!entry)
291 return NULL;
292 entry->refcount = 1;
293 RB_CLEAR_NODE(&entry->rbnode);
294 return entry;
295}
296
297static void zswap_entry_cache_free(struct zswap_entry *entry)
298{
299 kmem_cache_free(zswap_entry_cache, entry);
300}
301
302/*********************************
303* rbtree functions
304**********************************/
305static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
306{
307 struct rb_node *node = root->rb_node;
308 struct zswap_entry *entry;
309
310 while (node) {
311 entry = rb_entry(node, struct zswap_entry, rbnode);
312 if (entry->offset > offset)
313 node = node->rb_left;
314 else if (entry->offset < offset)
315 node = node->rb_right;
316 else
317 return entry;
318 }
319 return NULL;
320}
321
322/*
323 * In the case that a entry with the same offset is found, a pointer to
324 * the existing entry is stored in dupentry and the function returns -EEXIST
325 */
326static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
327 struct zswap_entry **dupentry)
328{
329 struct rb_node **link = &root->rb_node, *parent = NULL;
330 struct zswap_entry *myentry;
331
332 while (*link) {
333 parent = *link;
334 myentry = rb_entry(parent, struct zswap_entry, rbnode);
335 if (myentry->offset > entry->offset)
336 link = &(*link)->rb_left;
337 else if (myentry->offset < entry->offset)
338 link = &(*link)->rb_right;
339 else {
340 *dupentry = myentry;
341 return -EEXIST;
342 }
343 }
344 rb_link_node(&entry->rbnode, parent, link);
345 rb_insert_color(&entry->rbnode, root);
346 return 0;
347}
348
349static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
350{
351 if (!RB_EMPTY_NODE(&entry->rbnode)) {
352 rb_erase(&entry->rbnode, root);
353 RB_CLEAR_NODE(&entry->rbnode);
354 }
355}
356
357/*
358 * Carries out the common pattern of freeing and entry's zpool allocation,
359 * freeing the entry itself, and decrementing the number of stored pages.
360 */
361static void zswap_free_entry(struct zswap_entry *entry)
362{
363 if (entry->objcg) {
364 obj_cgroup_uncharge_zswap(entry->objcg, entry->length);
365 obj_cgroup_put(entry->objcg);
366 }
367 if (!entry->length)
368 atomic_dec(&zswap_same_filled_pages);
369 else {
370 zpool_free(entry->pool->zpool, entry->handle);
371 zswap_pool_put(entry->pool);
372 }
373 zswap_entry_cache_free(entry);
374 atomic_dec(&zswap_stored_pages);
375 zswap_update_total_size();
376}
377
378/* caller must hold the tree lock */
379static void zswap_entry_get(struct zswap_entry *entry)
380{
381 entry->refcount++;
382}
383
384/* caller must hold the tree lock
385* remove from the tree and free it, if nobody reference the entry
386*/
387static void zswap_entry_put(struct zswap_tree *tree,
388 struct zswap_entry *entry)
389{
390 int refcount = --entry->refcount;
391
392 BUG_ON(refcount < 0);
393 if (refcount == 0) {
394 zswap_rb_erase(&tree->rbroot, entry);
395 zswap_free_entry(entry);
396 }
397}
398
399/* caller must hold the tree lock */
400static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
401 pgoff_t offset)
402{
403 struct zswap_entry *entry;
404
405 entry = zswap_rb_search(root, offset);
406 if (entry)
407 zswap_entry_get(entry);
408
409 return entry;
410}
411
412/*********************************
413* per-cpu code
414**********************************/
415static DEFINE_PER_CPU(u8 *, zswap_dstmem);
416/*
417 * If users dynamically change the zpool type and compressor at runtime, i.e.
418 * zswap is running, zswap can have more than one zpool on one cpu, but they
419 * are sharing dtsmem. So we need this mutex to be per-cpu.
420 */
421static DEFINE_PER_CPU(struct mutex *, zswap_mutex);
422
423static int zswap_dstmem_prepare(unsigned int cpu)
424{
425 struct mutex *mutex;
426 u8 *dst;
427
428 dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
429 if (!dst)
430 return -ENOMEM;
431
432 mutex = kmalloc_node(sizeof(*mutex), GFP_KERNEL, cpu_to_node(cpu));
433 if (!mutex) {
434 kfree(dst);
435 return -ENOMEM;
436 }
437
438 mutex_init(mutex);
439 per_cpu(zswap_dstmem, cpu) = dst;
440 per_cpu(zswap_mutex, cpu) = mutex;
441 return 0;
442}
443
444static int zswap_dstmem_dead(unsigned int cpu)
445{
446 struct mutex *mutex;
447 u8 *dst;
448
449 mutex = per_cpu(zswap_mutex, cpu);
450 kfree(mutex);
451 per_cpu(zswap_mutex, cpu) = NULL;
452
453 dst = per_cpu(zswap_dstmem, cpu);
454 kfree(dst);
455 per_cpu(zswap_dstmem, cpu) = NULL;
456
457 return 0;
458}
459
460static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
461{
462 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
463 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
464 struct crypto_acomp *acomp;
465 struct acomp_req *req;
466
467 acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));
468 if (IS_ERR(acomp)) {
469 pr_err("could not alloc crypto acomp %s : %ld\n",
470 pool->tfm_name, PTR_ERR(acomp));
471 return PTR_ERR(acomp);
472 }
473 acomp_ctx->acomp = acomp;
474
475 req = acomp_request_alloc(acomp_ctx->acomp);
476 if (!req) {
477 pr_err("could not alloc crypto acomp_request %s\n",
478 pool->tfm_name);
479 crypto_free_acomp(acomp_ctx->acomp);
480 return -ENOMEM;
481 }
482 acomp_ctx->req = req;
483
484 crypto_init_wait(&acomp_ctx->wait);
485 /*
486 * if the backend of acomp is async zip, crypto_req_done() will wakeup
487 * crypto_wait_req(); if the backend of acomp is scomp, the callback
488 * won't be called, crypto_wait_req() will return without blocking.
489 */
490 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
491 crypto_req_done, &acomp_ctx->wait);
492
493 acomp_ctx->mutex = per_cpu(zswap_mutex, cpu);
494 acomp_ctx->dstmem = per_cpu(zswap_dstmem, cpu);
495
496 return 0;
497}
498
499static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
500{
501 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
502 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
503
504 if (!IS_ERR_OR_NULL(acomp_ctx)) {
505 if (!IS_ERR_OR_NULL(acomp_ctx->req))
506 acomp_request_free(acomp_ctx->req);
507 if (!IS_ERR_OR_NULL(acomp_ctx->acomp))
508 crypto_free_acomp(acomp_ctx->acomp);
509 }
510
511 return 0;
512}
513
514/*********************************
515* pool functions
516**********************************/
517
518static struct zswap_pool *__zswap_pool_current(void)
519{
520 struct zswap_pool *pool;
521
522 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
523 WARN_ONCE(!pool && zswap_has_pool,
524 "%s: no page storage pool!\n", __func__);
525
526 return pool;
527}
528
529static struct zswap_pool *zswap_pool_current(void)
530{
531 assert_spin_locked(&zswap_pools_lock);
532
533 return __zswap_pool_current();
534}
535
536static struct zswap_pool *zswap_pool_current_get(void)
537{
538 struct zswap_pool *pool;
539
540 rcu_read_lock();
541
542 pool = __zswap_pool_current();
543 if (!zswap_pool_get(pool))
544 pool = NULL;
545
546 rcu_read_unlock();
547
548 return pool;
549}
550
551static struct zswap_pool *zswap_pool_last_get(void)
552{
553 struct zswap_pool *pool, *last = NULL;
554
555 rcu_read_lock();
556
557 list_for_each_entry_rcu(pool, &zswap_pools, list)
558 last = pool;
559 WARN_ONCE(!last && zswap_has_pool,
560 "%s: no page storage pool!\n", __func__);
561 if (!zswap_pool_get(last))
562 last = NULL;
563
564 rcu_read_unlock();
565
566 return last;
567}
568
569/* type and compressor must be null-terminated */
570static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
571{
572 struct zswap_pool *pool;
573
574 assert_spin_locked(&zswap_pools_lock);
575
576 list_for_each_entry_rcu(pool, &zswap_pools, list) {
577 if (strcmp(pool->tfm_name, compressor))
578 continue;
579 if (strcmp(zpool_get_type(pool->zpool), type))
580 continue;
581 /* if we can't get it, it's about to be destroyed */
582 if (!zswap_pool_get(pool))
583 continue;
584 return pool;
585 }
586
587 return NULL;
588}
589
590static void shrink_worker(struct work_struct *w)
591{
592 struct zswap_pool *pool = container_of(w, typeof(*pool),
593 shrink_work);
594
595 if (zpool_shrink(pool->zpool, 1, NULL))
596 zswap_reject_reclaim_fail++;
597 zswap_pool_put(pool);
598}
599
600static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
601{
602 struct zswap_pool *pool;
603 char name[38]; /* 'zswap' + 32 char (max) num + \0 */
604 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
605 int ret;
606
607 if (!zswap_has_pool) {
608 /* if either are unset, pool initialization failed, and we
609 * need both params to be set correctly before trying to
610 * create a pool.
611 */
612 if (!strcmp(type, ZSWAP_PARAM_UNSET))
613 return NULL;
614 if (!strcmp(compressor, ZSWAP_PARAM_UNSET))
615 return NULL;
616 }
617
618 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
619 if (!pool)
620 return NULL;
621
622 /* unique name for each pool specifically required by zsmalloc */
623 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
624
625 pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);
626 if (!pool->zpool) {
627 pr_err("%s zpool not available\n", type);
628 goto error;
629 }
630 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
631
632 strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
633
634 pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx);
635 if (!pool->acomp_ctx) {
636 pr_err("percpu alloc failed\n");
637 goto error;
638 }
639
640 ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
641 &pool->node);
642 if (ret)
643 goto error;
644 pr_debug("using %s compressor\n", pool->tfm_name);
645
646 /* being the current pool takes 1 ref; this func expects the
647 * caller to always add the new pool as the current pool
648 */
649 kref_init(&pool->kref);
650 INIT_LIST_HEAD(&pool->list);
651 INIT_WORK(&pool->shrink_work, shrink_worker);
652
653 zswap_pool_debug("created", pool);
654
655 return pool;
656
657error:
658 if (pool->acomp_ctx)
659 free_percpu(pool->acomp_ctx);
660 if (pool->zpool)
661 zpool_destroy_pool(pool->zpool);
662 kfree(pool);
663 return NULL;
664}
665
666static __init struct zswap_pool *__zswap_pool_create_fallback(void)
667{
668 bool has_comp, has_zpool;
669
670 has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
671 if (!has_comp && strcmp(zswap_compressor,
672 CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) {
673 pr_err("compressor %s not available, using default %s\n",
674 zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT);
675 param_free_charp(&zswap_compressor);
676 zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
677 has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
678 }
679 if (!has_comp) {
680 pr_err("default compressor %s not available\n",
681 zswap_compressor);
682 param_free_charp(&zswap_compressor);
683 zswap_compressor = ZSWAP_PARAM_UNSET;
684 }
685
686 has_zpool = zpool_has_pool(zswap_zpool_type);
687 if (!has_zpool && strcmp(zswap_zpool_type,
688 CONFIG_ZSWAP_ZPOOL_DEFAULT)) {
689 pr_err("zpool %s not available, using default %s\n",
690 zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT);
691 param_free_charp(&zswap_zpool_type);
692 zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
693 has_zpool = zpool_has_pool(zswap_zpool_type);
694 }
695 if (!has_zpool) {
696 pr_err("default zpool %s not available\n",
697 zswap_zpool_type);
698 param_free_charp(&zswap_zpool_type);
699 zswap_zpool_type = ZSWAP_PARAM_UNSET;
700 }
701
702 if (!has_comp || !has_zpool)
703 return NULL;
704
705 return zswap_pool_create(zswap_zpool_type, zswap_compressor);
706}
707
708static void zswap_pool_destroy(struct zswap_pool *pool)
709{
710 zswap_pool_debug("destroying", pool);
711
712 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
713 free_percpu(pool->acomp_ctx);
714 zpool_destroy_pool(pool->zpool);
715 kfree(pool);
716}
717
718static int __must_check zswap_pool_get(struct zswap_pool *pool)
719{
720 if (!pool)
721 return 0;
722
723 return kref_get_unless_zero(&pool->kref);
724}
725
726static void __zswap_pool_release(struct work_struct *work)
727{
728 struct zswap_pool *pool = container_of(work, typeof(*pool),
729 release_work);
730
731 synchronize_rcu();
732
733 /* nobody should have been able to get a kref... */
734 WARN_ON(kref_get_unless_zero(&pool->kref));
735
736 /* pool is now off zswap_pools list and has no references. */
737 zswap_pool_destroy(pool);
738}
739
740static void __zswap_pool_empty(struct kref *kref)
741{
742 struct zswap_pool *pool;
743
744 pool = container_of(kref, typeof(*pool), kref);
745
746 spin_lock(&zswap_pools_lock);
747
748 WARN_ON(pool == zswap_pool_current());
749
750 list_del_rcu(&pool->list);
751
752 INIT_WORK(&pool->release_work, __zswap_pool_release);
753 schedule_work(&pool->release_work);
754
755 spin_unlock(&zswap_pools_lock);
756}
757
758static void zswap_pool_put(struct zswap_pool *pool)
759{
760 kref_put(&pool->kref, __zswap_pool_empty);
761}
762
763/*********************************
764* param callbacks
765**********************************/
766
767/* val must be a null-terminated string */
768static int __zswap_param_set(const char *val, const struct kernel_param *kp,
769 char *type, char *compressor)
770{
771 struct zswap_pool *pool, *put_pool = NULL;
772 char *s = strstrip((char *)val);
773 int ret;
774
775 if (zswap_init_failed) {
776 pr_err("can't set param, initialization failed\n");
777 return -ENODEV;
778 }
779
780 /* no change required */
781 if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)
782 return 0;
783
784 /* if this is load-time (pre-init) param setting,
785 * don't create a pool; that's done during init.
786 */
787 if (!zswap_init_started)
788 return param_set_charp(s, kp);
789
790 if (!type) {
791 if (!zpool_has_pool(s)) {
792 pr_err("zpool %s not available\n", s);
793 return -ENOENT;
794 }
795 type = s;
796 } else if (!compressor) {
797 if (!crypto_has_acomp(s, 0, 0)) {
798 pr_err("compressor %s not available\n", s);
799 return -ENOENT;
800 }
801 compressor = s;
802 } else {
803 WARN_ON(1);
804 return -EINVAL;
805 }
806
807 spin_lock(&zswap_pools_lock);
808
809 pool = zswap_pool_find_get(type, compressor);
810 if (pool) {
811 zswap_pool_debug("using existing", pool);
812 WARN_ON(pool == zswap_pool_current());
813 list_del_rcu(&pool->list);
814 }
815
816 spin_unlock(&zswap_pools_lock);
817
818 if (!pool)
819 pool = zswap_pool_create(type, compressor);
820
821 if (pool)
822 ret = param_set_charp(s, kp);
823 else
824 ret = -EINVAL;
825
826 spin_lock(&zswap_pools_lock);
827
828 if (!ret) {
829 put_pool = zswap_pool_current();
830 list_add_rcu(&pool->list, &zswap_pools);
831 zswap_has_pool = true;
832 } else if (pool) {
833 /* add the possibly pre-existing pool to the end of the pools
834 * list; if it's new (and empty) then it'll be removed and
835 * destroyed by the put after we drop the lock
836 */
837 list_add_tail_rcu(&pool->list, &zswap_pools);
838 put_pool = pool;
839 }
840
841 spin_unlock(&zswap_pools_lock);
842
843 if (!zswap_has_pool && !pool) {
844 /* if initial pool creation failed, and this pool creation also
845 * failed, maybe both compressor and zpool params were bad.
846 * Allow changing this param, so pool creation will succeed
847 * when the other param is changed. We already verified this
848 * param is ok in the zpool_has_pool() or crypto_has_acomp()
849 * checks above.
850 */
851 ret = param_set_charp(s, kp);
852 }
853
854 /* drop the ref from either the old current pool,
855 * or the new pool we failed to add
856 */
857 if (put_pool)
858 zswap_pool_put(put_pool);
859
860 return ret;
861}
862
863static int zswap_compressor_param_set(const char *val,
864 const struct kernel_param *kp)
865{
866 return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
867}
868
869static int zswap_zpool_param_set(const char *val,
870 const struct kernel_param *kp)
871{
872 return __zswap_param_set(val, kp, NULL, zswap_compressor);
873}
874
875static int zswap_enabled_param_set(const char *val,
876 const struct kernel_param *kp)
877{
878 if (zswap_init_failed) {
879 pr_err("can't enable, initialization failed\n");
880 return -ENODEV;
881 }
882 if (!zswap_has_pool && zswap_init_started) {
883 pr_err("can't enable, no pool configured\n");
884 return -ENODEV;
885 }
886
887 return param_set_bool(val, kp);
888}
889
890/*********************************
891* writeback code
892**********************************/
893/* return enum for zswap_get_swap_cache_page */
894enum zswap_get_swap_ret {
895 ZSWAP_SWAPCACHE_NEW,
896 ZSWAP_SWAPCACHE_EXIST,
897 ZSWAP_SWAPCACHE_FAIL,
898};
899
900/*
901 * zswap_get_swap_cache_page
902 *
903 * This is an adaption of read_swap_cache_async()
904 *
905 * This function tries to find a page with the given swap entry
906 * in the swapper_space address space (the swap cache). If the page
907 * is found, it is returned in retpage. Otherwise, a page is allocated,
908 * added to the swap cache, and returned in retpage.
909 *
910 * If success, the swap cache page is returned in retpage
911 * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
912 * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
913 * the new page is added to swapcache and locked
914 * Returns ZSWAP_SWAPCACHE_FAIL on error
915 */
916static int zswap_get_swap_cache_page(swp_entry_t entry,
917 struct page **retpage)
918{
919 bool page_was_allocated;
920
921 *retpage = __read_swap_cache_async(entry, GFP_KERNEL,
922 NULL, 0, &page_was_allocated);
923 if (page_was_allocated)
924 return ZSWAP_SWAPCACHE_NEW;
925 if (!*retpage)
926 return ZSWAP_SWAPCACHE_FAIL;
927 return ZSWAP_SWAPCACHE_EXIST;
928}
929
930/*
931 * Attempts to free an entry by adding a page to the swap cache,
932 * decompressing the entry data into the page, and issuing a
933 * bio write to write the page back to the swap device.
934 *
935 * This can be thought of as a "resumed writeback" of the page
936 * to the swap device. We are basically resuming the same swap
937 * writeback path that was intercepted with the frontswap_store()
938 * in the first place. After the page has been decompressed into
939 * the swap cache, the compressed version stored by zswap can be
940 * freed.
941 */
942static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
943{
944 struct zswap_header *zhdr;
945 swp_entry_t swpentry;
946 struct zswap_tree *tree;
947 pgoff_t offset;
948 struct zswap_entry *entry;
949 struct page *page;
950 struct scatterlist input, output;
951 struct crypto_acomp_ctx *acomp_ctx;
952
953 u8 *src, *tmp = NULL;
954 unsigned int dlen;
955 int ret;
956 struct writeback_control wbc = {
957 .sync_mode = WB_SYNC_NONE,
958 };
959
960 if (!zpool_can_sleep_mapped(pool)) {
961 tmp = kmalloc(PAGE_SIZE, GFP_KERNEL);
962 if (!tmp)
963 return -ENOMEM;
964 }
965
966 /* extract swpentry from data */
967 zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
968 swpentry = zhdr->swpentry; /* here */
969 tree = zswap_trees[swp_type(swpentry)];
970 offset = swp_offset(swpentry);
971 zpool_unmap_handle(pool, handle);
972
973 /* find and ref zswap entry */
974 spin_lock(&tree->lock);
975 entry = zswap_entry_find_get(&tree->rbroot, offset);
976 if (!entry) {
977 /* entry was invalidated */
978 spin_unlock(&tree->lock);
979 kfree(tmp);
980 return 0;
981 }
982 spin_unlock(&tree->lock);
983 BUG_ON(offset != entry->offset);
984
985 /* try to allocate swap cache page */
986 switch (zswap_get_swap_cache_page(swpentry, &page)) {
987 case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
988 ret = -ENOMEM;
989 goto fail;
990
991 case ZSWAP_SWAPCACHE_EXIST:
992 /* page is already in the swap cache, ignore for now */
993 put_page(page);
994 ret = -EEXIST;
995 goto fail;
996
997 case ZSWAP_SWAPCACHE_NEW: /* page is locked */
998 /* decompress */
999 acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
1000 dlen = PAGE_SIZE;
1001
1002 zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
1003 src = (u8 *)zhdr + sizeof(struct zswap_header);
1004 if (!zpool_can_sleep_mapped(pool)) {
1005 memcpy(tmp, src, entry->length);
1006 src = tmp;
1007 zpool_unmap_handle(pool, handle);
1008 }
1009
1010 mutex_lock(acomp_ctx->mutex);
1011 sg_init_one(&input, src, entry->length);
1012 sg_init_table(&output, 1);
1013 sg_set_page(&output, page, PAGE_SIZE, 0);
1014 acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, dlen);
1015 ret = crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait);
1016 dlen = acomp_ctx->req->dlen;
1017 mutex_unlock(acomp_ctx->mutex);
1018
1019 if (!zpool_can_sleep_mapped(pool))
1020 kfree(tmp);
1021 else
1022 zpool_unmap_handle(pool, handle);
1023
1024 BUG_ON(ret);
1025 BUG_ON(dlen != PAGE_SIZE);
1026
1027 /* page is up to date */
1028 SetPageUptodate(page);
1029 }
1030
1031 /* move it to the tail of the inactive list after end_writeback */
1032 SetPageReclaim(page);
1033
1034 /* start writeback */
1035 __swap_writepage(page, &wbc);
1036 put_page(page);
1037 zswap_written_back_pages++;
1038
1039 spin_lock(&tree->lock);
1040 /* drop local reference */
1041 zswap_entry_put(tree, entry);
1042
1043 /*
1044 * There are two possible situations for entry here:
1045 * (1) refcount is 1(normal case), entry is valid and on the tree
1046 * (2) refcount is 0, entry is freed and not on the tree
1047 * because invalidate happened during writeback
1048 * search the tree and free the entry if find entry
1049 */
1050 if (entry == zswap_rb_search(&tree->rbroot, offset))
1051 zswap_entry_put(tree, entry);
1052 spin_unlock(&tree->lock);
1053
1054 return ret;
1055
1056fail:
1057 if (!zpool_can_sleep_mapped(pool))
1058 kfree(tmp);
1059
1060 /*
1061 * if we get here due to ZSWAP_SWAPCACHE_EXIST
1062 * a load may be happening concurrently.
1063 * it is safe and okay to not free the entry.
1064 * if we free the entry in the following put
1065 * it is also okay to return !0
1066 */
1067 spin_lock(&tree->lock);
1068 zswap_entry_put(tree, entry);
1069 spin_unlock(&tree->lock);
1070
1071 return ret;
1072}
1073
1074static int zswap_is_page_same_filled(void *ptr, unsigned long *value)
1075{
1076 unsigned int pos;
1077 unsigned long *page;
1078
1079 page = (unsigned long *)ptr;
1080 for (pos = 1; pos < PAGE_SIZE / sizeof(*page); pos++) {
1081 if (page[pos] != page[0])
1082 return 0;
1083 }
1084 *value = page[0];
1085 return 1;
1086}
1087
1088static void zswap_fill_page(void *ptr, unsigned long value)
1089{
1090 unsigned long *page;
1091
1092 page = (unsigned long *)ptr;
1093 memset_l(page, value, PAGE_SIZE / sizeof(unsigned long));
1094}
1095
1096/*********************************
1097* frontswap hooks
1098**********************************/
1099/* attempts to compress and store an single page */
1100static int zswap_frontswap_store(unsigned type, pgoff_t offset,
1101 struct page *page)
1102{
1103 struct zswap_tree *tree = zswap_trees[type];
1104 struct zswap_entry *entry, *dupentry;
1105 struct scatterlist input, output;
1106 struct crypto_acomp_ctx *acomp_ctx;
1107 struct obj_cgroup *objcg = NULL;
1108 struct zswap_pool *pool;
1109 int ret;
1110 unsigned int hlen, dlen = PAGE_SIZE;
1111 unsigned long handle, value;
1112 char *buf;
1113 u8 *src, *dst;
1114 struct zswap_header zhdr = { .swpentry = swp_entry(type, offset) };
1115 gfp_t gfp;
1116
1117 /* THP isn't supported */
1118 if (PageTransHuge(page)) {
1119 ret = -EINVAL;
1120 goto reject;
1121 }
1122
1123 if (!zswap_enabled || !tree) {
1124 ret = -ENODEV;
1125 goto reject;
1126 }
1127
1128 objcg = get_obj_cgroup_from_page(page);
1129 if (objcg && !obj_cgroup_may_zswap(objcg))
1130 goto shrink;
1131
1132 /* reclaim space if needed */
1133 if (zswap_is_full()) {
1134 zswap_pool_limit_hit++;
1135 zswap_pool_reached_full = true;
1136 goto shrink;
1137 }
1138
1139 if (zswap_pool_reached_full) {
1140 if (!zswap_can_accept()) {
1141 ret = -ENOMEM;
1142 goto reject;
1143 } else
1144 zswap_pool_reached_full = false;
1145 }
1146
1147 /* allocate entry */
1148 entry = zswap_entry_cache_alloc(GFP_KERNEL);
1149 if (!entry) {
1150 zswap_reject_kmemcache_fail++;
1151 ret = -ENOMEM;
1152 goto reject;
1153 }
1154
1155 if (zswap_same_filled_pages_enabled) {
1156 src = kmap_atomic(page);
1157 if (zswap_is_page_same_filled(src, &value)) {
1158 kunmap_atomic(src);
1159 entry->offset = offset;
1160 entry->length = 0;
1161 entry->value = value;
1162 atomic_inc(&zswap_same_filled_pages);
1163 goto insert_entry;
1164 }
1165 kunmap_atomic(src);
1166 }
1167
1168 if (!zswap_non_same_filled_pages_enabled) {
1169 ret = -EINVAL;
1170 goto freepage;
1171 }
1172
1173 /* if entry is successfully added, it keeps the reference */
1174 entry->pool = zswap_pool_current_get();
1175 if (!entry->pool) {
1176 ret = -EINVAL;
1177 goto freepage;
1178 }
1179
1180 /* compress */
1181 acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
1182
1183 mutex_lock(acomp_ctx->mutex);
1184
1185 dst = acomp_ctx->dstmem;
1186 sg_init_table(&input, 1);
1187 sg_set_page(&input, page, PAGE_SIZE, 0);
1188
1189 /* zswap_dstmem is of size (PAGE_SIZE * 2). Reflect same in sg_list */
1190 sg_init_one(&output, dst, PAGE_SIZE * 2);
1191 acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
1192 /*
1193 * it maybe looks a little bit silly that we send an asynchronous request,
1194 * then wait for its completion synchronously. This makes the process look
1195 * synchronous in fact.
1196 * Theoretically, acomp supports users send multiple acomp requests in one
1197 * acomp instance, then get those requests done simultaneously. but in this
1198 * case, frontswap actually does store and load page by page, there is no
1199 * existing method to send the second page before the first page is done
1200 * in one thread doing frontswap.
1201 * but in different threads running on different cpu, we have different
1202 * acomp instance, so multiple threads can do (de)compression in parallel.
1203 */
1204 ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);
1205 dlen = acomp_ctx->req->dlen;
1206
1207 if (ret) {
1208 ret = -EINVAL;
1209 goto put_dstmem;
1210 }
1211
1212 /* store */
1213 hlen = zpool_evictable(entry->pool->zpool) ? sizeof(zhdr) : 0;
1214 gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
1215 if (zpool_malloc_support_movable(entry->pool->zpool))
1216 gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;
1217 ret = zpool_malloc(entry->pool->zpool, hlen + dlen, gfp, &handle);
1218 if (ret == -ENOSPC) {
1219 zswap_reject_compress_poor++;
1220 goto put_dstmem;
1221 }
1222 if (ret) {
1223 zswap_reject_alloc_fail++;
1224 goto put_dstmem;
1225 }
1226 buf = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_WO);
1227 memcpy(buf, &zhdr, hlen);
1228 memcpy(buf + hlen, dst, dlen);
1229 zpool_unmap_handle(entry->pool->zpool, handle);
1230 mutex_unlock(acomp_ctx->mutex);
1231
1232 /* populate entry */
1233 entry->offset = offset;
1234 entry->handle = handle;
1235 entry->length = dlen;
1236
1237insert_entry:
1238 entry->objcg = objcg;
1239 if (objcg) {
1240 obj_cgroup_charge_zswap(objcg, entry->length);
1241 /* Account before objcg ref is moved to tree */
1242 count_objcg_event(objcg, ZSWPOUT);
1243 }
1244
1245 /* map */
1246 spin_lock(&tree->lock);
1247 do {
1248 ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
1249 if (ret == -EEXIST) {
1250 zswap_duplicate_entry++;
1251 /* remove from rbtree */
1252 zswap_rb_erase(&tree->rbroot, dupentry);
1253 zswap_entry_put(tree, dupentry);
1254 }
1255 } while (ret == -EEXIST);
1256 spin_unlock(&tree->lock);
1257
1258 /* update stats */
1259 atomic_inc(&zswap_stored_pages);
1260 zswap_update_total_size();
1261 count_vm_event(ZSWPOUT);
1262
1263 return 0;
1264
1265put_dstmem:
1266 mutex_unlock(acomp_ctx->mutex);
1267 zswap_pool_put(entry->pool);
1268freepage:
1269 zswap_entry_cache_free(entry);
1270reject:
1271 if (objcg)
1272 obj_cgroup_put(objcg);
1273 return ret;
1274
1275shrink:
1276 pool = zswap_pool_last_get();
1277 if (pool)
1278 queue_work(shrink_wq, &pool->shrink_work);
1279 ret = -ENOMEM;
1280 goto reject;
1281}
1282
1283/*
1284 * returns 0 if the page was successfully decompressed
1285 * return -1 on entry not found or error
1286*/
1287static int zswap_frontswap_load(unsigned type, pgoff_t offset,
1288 struct page *page)
1289{
1290 struct zswap_tree *tree = zswap_trees[type];
1291 struct zswap_entry *entry;
1292 struct scatterlist input, output;
1293 struct crypto_acomp_ctx *acomp_ctx;
1294 u8 *src, *dst, *tmp;
1295 unsigned int dlen;
1296 int ret;
1297
1298 /* find */
1299 spin_lock(&tree->lock);
1300 entry = zswap_entry_find_get(&tree->rbroot, offset);
1301 if (!entry) {
1302 /* entry was written back */
1303 spin_unlock(&tree->lock);
1304 return -1;
1305 }
1306 spin_unlock(&tree->lock);
1307
1308 if (!entry->length) {
1309 dst = kmap_atomic(page);
1310 zswap_fill_page(dst, entry->value);
1311 kunmap_atomic(dst);
1312 ret = 0;
1313 goto stats;
1314 }
1315
1316 if (!zpool_can_sleep_mapped(entry->pool->zpool)) {
1317 tmp = kmalloc(entry->length, GFP_KERNEL);
1318 if (!tmp) {
1319 ret = -ENOMEM;
1320 goto freeentry;
1321 }
1322 }
1323
1324 /* decompress */
1325 dlen = PAGE_SIZE;
1326 src = zpool_map_handle(entry->pool->zpool, entry->handle, ZPOOL_MM_RO);
1327 if (zpool_evictable(entry->pool->zpool))
1328 src += sizeof(struct zswap_header);
1329
1330 if (!zpool_can_sleep_mapped(entry->pool->zpool)) {
1331 memcpy(tmp, src, entry->length);
1332 src = tmp;
1333 zpool_unmap_handle(entry->pool->zpool, entry->handle);
1334 }
1335
1336 acomp_ctx = raw_cpu_ptr(entry->pool->acomp_ctx);
1337 mutex_lock(acomp_ctx->mutex);
1338 sg_init_one(&input, src, entry->length);
1339 sg_init_table(&output, 1);
1340 sg_set_page(&output, page, PAGE_SIZE, 0);
1341 acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, dlen);
1342 ret = crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait);
1343 mutex_unlock(acomp_ctx->mutex);
1344
1345 if (zpool_can_sleep_mapped(entry->pool->zpool))
1346 zpool_unmap_handle(entry->pool->zpool, entry->handle);
1347 else
1348 kfree(tmp);
1349
1350 BUG_ON(ret);
1351stats:
1352 count_vm_event(ZSWPIN);
1353 if (entry->objcg)
1354 count_objcg_event(entry->objcg, ZSWPIN);
1355freeentry:
1356 spin_lock(&tree->lock);
1357 zswap_entry_put(tree, entry);
1358 spin_unlock(&tree->lock);
1359
1360 return ret;
1361}
1362
1363/* frees an entry in zswap */
1364static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
1365{
1366 struct zswap_tree *tree = zswap_trees[type];
1367 struct zswap_entry *entry;
1368
1369 /* find */
1370 spin_lock(&tree->lock);
1371 entry = zswap_rb_search(&tree->rbroot, offset);
1372 if (!entry) {
1373 /* entry was written back */
1374 spin_unlock(&tree->lock);
1375 return;
1376 }
1377
1378 /* remove from rbtree */
1379 zswap_rb_erase(&tree->rbroot, entry);
1380
1381 /* drop the initial reference from entry creation */
1382 zswap_entry_put(tree, entry);
1383
1384 spin_unlock(&tree->lock);
1385}
1386
1387/* frees all zswap entries for the given swap type */
1388static void zswap_frontswap_invalidate_area(unsigned type)
1389{
1390 struct zswap_tree *tree = zswap_trees[type];
1391 struct zswap_entry *entry, *n;
1392
1393 if (!tree)
1394 return;
1395
1396 /* walk the tree and free everything */
1397 spin_lock(&tree->lock);
1398 rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
1399 zswap_free_entry(entry);
1400 tree->rbroot = RB_ROOT;
1401 spin_unlock(&tree->lock);
1402 kfree(tree);
1403 zswap_trees[type] = NULL;
1404}
1405
1406static void zswap_frontswap_init(unsigned type)
1407{
1408 struct zswap_tree *tree;
1409
1410 tree = kzalloc(sizeof(*tree), GFP_KERNEL);
1411 if (!tree) {
1412 pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1413 return;
1414 }
1415
1416 tree->rbroot = RB_ROOT;
1417 spin_lock_init(&tree->lock);
1418 zswap_trees[type] = tree;
1419}
1420
1421static const struct frontswap_ops zswap_frontswap_ops = {
1422 .store = zswap_frontswap_store,
1423 .load = zswap_frontswap_load,
1424 .invalidate_page = zswap_frontswap_invalidate_page,
1425 .invalidate_area = zswap_frontswap_invalidate_area,
1426 .init = zswap_frontswap_init
1427};
1428
1429/*********************************
1430* debugfs functions
1431**********************************/
1432#ifdef CONFIG_DEBUG_FS
1433#include <linux/debugfs.h>
1434
1435static struct dentry *zswap_debugfs_root;
1436
1437static int __init zswap_debugfs_init(void)
1438{
1439 if (!debugfs_initialized())
1440 return -ENODEV;
1441
1442 zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1443
1444 debugfs_create_u64("pool_limit_hit", 0444,
1445 zswap_debugfs_root, &zswap_pool_limit_hit);
1446 debugfs_create_u64("reject_reclaim_fail", 0444,
1447 zswap_debugfs_root, &zswap_reject_reclaim_fail);
1448 debugfs_create_u64("reject_alloc_fail", 0444,
1449 zswap_debugfs_root, &zswap_reject_alloc_fail);
1450 debugfs_create_u64("reject_kmemcache_fail", 0444,
1451 zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1452 debugfs_create_u64("reject_compress_poor", 0444,
1453 zswap_debugfs_root, &zswap_reject_compress_poor);
1454 debugfs_create_u64("written_back_pages", 0444,
1455 zswap_debugfs_root, &zswap_written_back_pages);
1456 debugfs_create_u64("duplicate_entry", 0444,
1457 zswap_debugfs_root, &zswap_duplicate_entry);
1458 debugfs_create_u64("pool_total_size", 0444,
1459 zswap_debugfs_root, &zswap_pool_total_size);
1460 debugfs_create_atomic_t("stored_pages", 0444,
1461 zswap_debugfs_root, &zswap_stored_pages);
1462 debugfs_create_atomic_t("same_filled_pages", 0444,
1463 zswap_debugfs_root, &zswap_same_filled_pages);
1464
1465 return 0;
1466}
1467#else
1468static int __init zswap_debugfs_init(void)
1469{
1470 return 0;
1471}
1472#endif
1473
1474/*********************************
1475* module init and exit
1476**********************************/
1477static int __init init_zswap(void)
1478{
1479 struct zswap_pool *pool;
1480 int ret;
1481
1482 zswap_init_started = true;
1483
1484 if (zswap_entry_cache_create()) {
1485 pr_err("entry cache creation failed\n");
1486 goto cache_fail;
1487 }
1488
1489 ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare",
1490 zswap_dstmem_prepare, zswap_dstmem_dead);
1491 if (ret) {
1492 pr_err("dstmem alloc failed\n");
1493 goto dstmem_fail;
1494 }
1495
1496 ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
1497 "mm/zswap_pool:prepare",
1498 zswap_cpu_comp_prepare,
1499 zswap_cpu_comp_dead);
1500 if (ret)
1501 goto hp_fail;
1502
1503 pool = __zswap_pool_create_fallback();
1504 if (pool) {
1505 pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1506 zpool_get_type(pool->zpool));
1507 list_add(&pool->list, &zswap_pools);
1508 zswap_has_pool = true;
1509 } else {
1510 pr_err("pool creation failed\n");
1511 zswap_enabled = false;
1512 }
1513
1514 shrink_wq = create_workqueue("zswap-shrink");
1515 if (!shrink_wq)
1516 goto fallback_fail;
1517
1518 ret = frontswap_register_ops(&zswap_frontswap_ops);
1519 if (ret)
1520 goto destroy_wq;
1521 if (zswap_debugfs_init())
1522 pr_warn("debugfs initialization failed\n");
1523 return 0;
1524
1525destroy_wq:
1526 destroy_workqueue(shrink_wq);
1527fallback_fail:
1528 if (pool)
1529 zswap_pool_destroy(pool);
1530hp_fail:
1531 cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE);
1532dstmem_fail:
1533 zswap_entry_cache_destroy();
1534cache_fail:
1535 /* if built-in, we aren't unloaded on failure; don't allow use */
1536 zswap_init_failed = true;
1537 zswap_enabled = false;
1538 return -ENOMEM;
1539}
1540/* must be late so crypto has time to come up */
1541late_initcall(init_zswap);
1542
1543MODULE_LICENSE("GPL");
1544MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1545MODULE_DESCRIPTION("Compressed cache for swap pages");
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * zswap.c - zswap driver file
4 *
5 * zswap is a cache that takes pages that are in the process
6 * of being swapped out and attempts to compress and store them in a
7 * RAM-based memory pool. This can result in a significant I/O reduction on
8 * the swap device and, in the case where decompressing from RAM is faster
9 * than reading from the swap device, can also improve workload performance.
10 *
11 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com>
12*/
13
14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16#include <linux/module.h>
17#include <linux/cpu.h>
18#include <linux/highmem.h>
19#include <linux/slab.h>
20#include <linux/spinlock.h>
21#include <linux/types.h>
22#include <linux/atomic.h>
23#include <linux/swap.h>
24#include <linux/crypto.h>
25#include <linux/scatterlist.h>
26#include <linux/mempolicy.h>
27#include <linux/mempool.h>
28#include <linux/zpool.h>
29#include <crypto/acompress.h>
30#include <linux/zswap.h>
31#include <linux/mm_types.h>
32#include <linux/page-flags.h>
33#include <linux/swapops.h>
34#include <linux/writeback.h>
35#include <linux/pagemap.h>
36#include <linux/workqueue.h>
37#include <linux/list_lru.h>
38
39#include "swap.h"
40#include "internal.h"
41
42/*********************************
43* statistics
44**********************************/
45/* The number of compressed pages currently stored in zswap */
46atomic_long_t zswap_stored_pages = ATOMIC_INIT(0);
47
48/*
49 * The statistics below are not protected from concurrent access for
50 * performance reasons so they may not be a 100% accurate. However,
51 * they do provide useful information on roughly how many times a
52 * certain event is occurring.
53*/
54
55/* Pool limit was hit (see zswap_max_pool_percent) */
56static u64 zswap_pool_limit_hit;
57/* Pages written back when pool limit was reached */
58static u64 zswap_written_back_pages;
59/* Store failed due to a reclaim failure after pool limit was reached */
60static u64 zswap_reject_reclaim_fail;
61/* Store failed due to compression algorithm failure */
62static u64 zswap_reject_compress_fail;
63/* Compressed page was too big for the allocator to (optimally) store */
64static u64 zswap_reject_compress_poor;
65/* Store failed because underlying allocator could not get memory */
66static u64 zswap_reject_alloc_fail;
67/* Store failed because the entry metadata could not be allocated (rare) */
68static u64 zswap_reject_kmemcache_fail;
69
70/* Shrinker work queue */
71static struct workqueue_struct *shrink_wq;
72/* Pool limit was hit, we need to calm down */
73static bool zswap_pool_reached_full;
74
75/*********************************
76* tunables
77**********************************/
78
79#define ZSWAP_PARAM_UNSET ""
80
81static int zswap_setup(void);
82
83/* Enable/disable zswap */
84static DEFINE_STATIC_KEY_MAYBE(CONFIG_ZSWAP_DEFAULT_ON, zswap_ever_enabled);
85static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON);
86static int zswap_enabled_param_set(const char *,
87 const struct kernel_param *);
88static const struct kernel_param_ops zswap_enabled_param_ops = {
89 .set = zswap_enabled_param_set,
90 .get = param_get_bool,
91};
92module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
93
94/* Crypto compressor to use */
95static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
96static int zswap_compressor_param_set(const char *,
97 const struct kernel_param *);
98static const struct kernel_param_ops zswap_compressor_param_ops = {
99 .set = zswap_compressor_param_set,
100 .get = param_get_charp,
101 .free = param_free_charp,
102};
103module_param_cb(compressor, &zswap_compressor_param_ops,
104 &zswap_compressor, 0644);
105
106/* Compressed storage zpool to use */
107static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
108static int zswap_zpool_param_set(const char *, const struct kernel_param *);
109static const struct kernel_param_ops zswap_zpool_param_ops = {
110 .set = zswap_zpool_param_set,
111 .get = param_get_charp,
112 .free = param_free_charp,
113};
114module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
115
116/* The maximum percentage of memory that the compressed pool can occupy */
117static unsigned int zswap_max_pool_percent = 20;
118module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
119
120/* The threshold for accepting new pages after the max_pool_percent was hit */
121static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */
122module_param_named(accept_threshold_percent, zswap_accept_thr_percent,
123 uint, 0644);
124
125/* Enable/disable memory pressure-based shrinker. */
126static bool zswap_shrinker_enabled = IS_ENABLED(
127 CONFIG_ZSWAP_SHRINKER_DEFAULT_ON);
128module_param_named(shrinker_enabled, zswap_shrinker_enabled, bool, 0644);
129
130bool zswap_is_enabled(void)
131{
132 return zswap_enabled;
133}
134
135bool zswap_never_enabled(void)
136{
137 return !static_branch_maybe(CONFIG_ZSWAP_DEFAULT_ON, &zswap_ever_enabled);
138}
139
140/*********************************
141* data structures
142**********************************/
143
144struct crypto_acomp_ctx {
145 struct crypto_acomp *acomp;
146 struct acomp_req *req;
147 struct crypto_wait wait;
148 u8 *buffer;
149 struct mutex mutex;
150 bool is_sleepable;
151};
152
153/*
154 * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock.
155 * The only case where lru_lock is not acquired while holding tree.lock is
156 * when a zswap_entry is taken off the lru for writeback, in that case it
157 * needs to be verified that it's still valid in the tree.
158 */
159struct zswap_pool {
160 struct zpool *zpool;
161 struct crypto_acomp_ctx __percpu *acomp_ctx;
162 struct percpu_ref ref;
163 struct list_head list;
164 struct work_struct release_work;
165 struct hlist_node node;
166 char tfm_name[CRYPTO_MAX_ALG_NAME];
167};
168
169/* Global LRU lists shared by all zswap pools. */
170static struct list_lru zswap_list_lru;
171
172/* The lock protects zswap_next_shrink updates. */
173static DEFINE_SPINLOCK(zswap_shrink_lock);
174static struct mem_cgroup *zswap_next_shrink;
175static struct work_struct zswap_shrink_work;
176static struct shrinker *zswap_shrinker;
177
178/*
179 * struct zswap_entry
180 *
181 * This structure contains the metadata for tracking a single compressed
182 * page within zswap.
183 *
184 * swpentry - associated swap entry, the offset indexes into the red-black tree
185 * length - the length in bytes of the compressed page data. Needed during
186 * decompression.
187 * referenced - true if the entry recently entered the zswap pool. Unset by the
188 * writeback logic. The entry is only reclaimed by the writeback
189 * logic if referenced is unset. See comments in the shrinker
190 * section for context.
191 * pool - the zswap_pool the entry's data is in
192 * handle - zpool allocation handle that stores the compressed page data
193 * objcg - the obj_cgroup that the compressed memory is charged to
194 * lru - handle to the pool's lru used to evict pages.
195 */
196struct zswap_entry {
197 swp_entry_t swpentry;
198 unsigned int length;
199 bool referenced;
200 struct zswap_pool *pool;
201 unsigned long handle;
202 struct obj_cgroup *objcg;
203 struct list_head lru;
204};
205
206static struct xarray *zswap_trees[MAX_SWAPFILES];
207static unsigned int nr_zswap_trees[MAX_SWAPFILES];
208
209/* RCU-protected iteration */
210static LIST_HEAD(zswap_pools);
211/* protects zswap_pools list modification */
212static DEFINE_SPINLOCK(zswap_pools_lock);
213/* pool counter to provide unique names to zpool */
214static atomic_t zswap_pools_count = ATOMIC_INIT(0);
215
216enum zswap_init_type {
217 ZSWAP_UNINIT,
218 ZSWAP_INIT_SUCCEED,
219 ZSWAP_INIT_FAILED
220};
221
222static enum zswap_init_type zswap_init_state;
223
224/* used to ensure the integrity of initialization */
225static DEFINE_MUTEX(zswap_init_lock);
226
227/* init completed, but couldn't create the initial pool */
228static bool zswap_has_pool;
229
230/*********************************
231* helpers and fwd declarations
232**********************************/
233
234static inline struct xarray *swap_zswap_tree(swp_entry_t swp)
235{
236 return &zswap_trees[swp_type(swp)][swp_offset(swp)
237 >> SWAP_ADDRESS_SPACE_SHIFT];
238}
239
240#define zswap_pool_debug(msg, p) \
241 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
242 zpool_get_type((p)->zpool))
243
244/*********************************
245* pool functions
246**********************************/
247static void __zswap_pool_empty(struct percpu_ref *ref);
248
249static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
250{
251 struct zswap_pool *pool;
252 char name[38]; /* 'zswap' + 32 char (max) num + \0 */
253 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
254 int ret, cpu;
255
256 if (!zswap_has_pool) {
257 /* if either are unset, pool initialization failed, and we
258 * need both params to be set correctly before trying to
259 * create a pool.
260 */
261 if (!strcmp(type, ZSWAP_PARAM_UNSET))
262 return NULL;
263 if (!strcmp(compressor, ZSWAP_PARAM_UNSET))
264 return NULL;
265 }
266
267 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
268 if (!pool)
269 return NULL;
270
271 /* unique name for each pool specifically required by zsmalloc */
272 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
273 pool->zpool = zpool_create_pool(type, name, gfp);
274 if (!pool->zpool) {
275 pr_err("%s zpool not available\n", type);
276 goto error;
277 }
278 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
279
280 strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
281
282 pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx);
283 if (!pool->acomp_ctx) {
284 pr_err("percpu alloc failed\n");
285 goto error;
286 }
287
288 for_each_possible_cpu(cpu)
289 mutex_init(&per_cpu_ptr(pool->acomp_ctx, cpu)->mutex);
290
291 ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
292 &pool->node);
293 if (ret)
294 goto error;
295
296 /* being the current pool takes 1 ref; this func expects the
297 * caller to always add the new pool as the current pool
298 */
299 ret = percpu_ref_init(&pool->ref, __zswap_pool_empty,
300 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
301 if (ret)
302 goto ref_fail;
303 INIT_LIST_HEAD(&pool->list);
304
305 zswap_pool_debug("created", pool);
306
307 return pool;
308
309ref_fail:
310 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
311error:
312 if (pool->acomp_ctx)
313 free_percpu(pool->acomp_ctx);
314 if (pool->zpool)
315 zpool_destroy_pool(pool->zpool);
316 kfree(pool);
317 return NULL;
318}
319
320static struct zswap_pool *__zswap_pool_create_fallback(void)
321{
322 bool has_comp, has_zpool;
323
324 has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
325 if (!has_comp && strcmp(zswap_compressor,
326 CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) {
327 pr_err("compressor %s not available, using default %s\n",
328 zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT);
329 param_free_charp(&zswap_compressor);
330 zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
331 has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
332 }
333 if (!has_comp) {
334 pr_err("default compressor %s not available\n",
335 zswap_compressor);
336 param_free_charp(&zswap_compressor);
337 zswap_compressor = ZSWAP_PARAM_UNSET;
338 }
339
340 has_zpool = zpool_has_pool(zswap_zpool_type);
341 if (!has_zpool && strcmp(zswap_zpool_type,
342 CONFIG_ZSWAP_ZPOOL_DEFAULT)) {
343 pr_err("zpool %s not available, using default %s\n",
344 zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT);
345 param_free_charp(&zswap_zpool_type);
346 zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
347 has_zpool = zpool_has_pool(zswap_zpool_type);
348 }
349 if (!has_zpool) {
350 pr_err("default zpool %s not available\n",
351 zswap_zpool_type);
352 param_free_charp(&zswap_zpool_type);
353 zswap_zpool_type = ZSWAP_PARAM_UNSET;
354 }
355
356 if (!has_comp || !has_zpool)
357 return NULL;
358
359 return zswap_pool_create(zswap_zpool_type, zswap_compressor);
360}
361
362static void zswap_pool_destroy(struct zswap_pool *pool)
363{
364 zswap_pool_debug("destroying", pool);
365
366 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
367 free_percpu(pool->acomp_ctx);
368
369 zpool_destroy_pool(pool->zpool);
370 kfree(pool);
371}
372
373static void __zswap_pool_release(struct work_struct *work)
374{
375 struct zswap_pool *pool = container_of(work, typeof(*pool),
376 release_work);
377
378 synchronize_rcu();
379
380 /* nobody should have been able to get a ref... */
381 WARN_ON(!percpu_ref_is_zero(&pool->ref));
382 percpu_ref_exit(&pool->ref);
383
384 /* pool is now off zswap_pools list and has no references. */
385 zswap_pool_destroy(pool);
386}
387
388static struct zswap_pool *zswap_pool_current(void);
389
390static void __zswap_pool_empty(struct percpu_ref *ref)
391{
392 struct zswap_pool *pool;
393
394 pool = container_of(ref, typeof(*pool), ref);
395
396 spin_lock_bh(&zswap_pools_lock);
397
398 WARN_ON(pool == zswap_pool_current());
399
400 list_del_rcu(&pool->list);
401
402 INIT_WORK(&pool->release_work, __zswap_pool_release);
403 schedule_work(&pool->release_work);
404
405 spin_unlock_bh(&zswap_pools_lock);
406}
407
408static int __must_check zswap_pool_tryget(struct zswap_pool *pool)
409{
410 if (!pool)
411 return 0;
412
413 return percpu_ref_tryget(&pool->ref);
414}
415
416/* The caller must already have a reference. */
417static void zswap_pool_get(struct zswap_pool *pool)
418{
419 percpu_ref_get(&pool->ref);
420}
421
422static void zswap_pool_put(struct zswap_pool *pool)
423{
424 percpu_ref_put(&pool->ref);
425}
426
427static struct zswap_pool *__zswap_pool_current(void)
428{
429 struct zswap_pool *pool;
430
431 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
432 WARN_ONCE(!pool && zswap_has_pool,
433 "%s: no page storage pool!\n", __func__);
434
435 return pool;
436}
437
438static struct zswap_pool *zswap_pool_current(void)
439{
440 assert_spin_locked(&zswap_pools_lock);
441
442 return __zswap_pool_current();
443}
444
445static struct zswap_pool *zswap_pool_current_get(void)
446{
447 struct zswap_pool *pool;
448
449 rcu_read_lock();
450
451 pool = __zswap_pool_current();
452 if (!zswap_pool_tryget(pool))
453 pool = NULL;
454
455 rcu_read_unlock();
456
457 return pool;
458}
459
460/* type and compressor must be null-terminated */
461static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
462{
463 struct zswap_pool *pool;
464
465 assert_spin_locked(&zswap_pools_lock);
466
467 list_for_each_entry_rcu(pool, &zswap_pools, list) {
468 if (strcmp(pool->tfm_name, compressor))
469 continue;
470 if (strcmp(zpool_get_type(pool->zpool), type))
471 continue;
472 /* if we can't get it, it's about to be destroyed */
473 if (!zswap_pool_tryget(pool))
474 continue;
475 return pool;
476 }
477
478 return NULL;
479}
480
481static unsigned long zswap_max_pages(void)
482{
483 return totalram_pages() * zswap_max_pool_percent / 100;
484}
485
486static unsigned long zswap_accept_thr_pages(void)
487{
488 return zswap_max_pages() * zswap_accept_thr_percent / 100;
489}
490
491unsigned long zswap_total_pages(void)
492{
493 struct zswap_pool *pool;
494 unsigned long total = 0;
495
496 rcu_read_lock();
497 list_for_each_entry_rcu(pool, &zswap_pools, list)
498 total += zpool_get_total_pages(pool->zpool);
499 rcu_read_unlock();
500
501 return total;
502}
503
504static bool zswap_check_limits(void)
505{
506 unsigned long cur_pages = zswap_total_pages();
507 unsigned long max_pages = zswap_max_pages();
508
509 if (cur_pages >= max_pages) {
510 zswap_pool_limit_hit++;
511 zswap_pool_reached_full = true;
512 } else if (zswap_pool_reached_full &&
513 cur_pages <= zswap_accept_thr_pages()) {
514 zswap_pool_reached_full = false;
515 }
516 return zswap_pool_reached_full;
517}
518
519/*********************************
520* param callbacks
521**********************************/
522
523static bool zswap_pool_changed(const char *s, const struct kernel_param *kp)
524{
525 /* no change required */
526 if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)
527 return false;
528 return true;
529}
530
531/* val must be a null-terminated string */
532static int __zswap_param_set(const char *val, const struct kernel_param *kp,
533 char *type, char *compressor)
534{
535 struct zswap_pool *pool, *put_pool = NULL;
536 char *s = strstrip((char *)val);
537 int ret = 0;
538 bool new_pool = false;
539
540 mutex_lock(&zswap_init_lock);
541 switch (zswap_init_state) {
542 case ZSWAP_UNINIT:
543 /* if this is load-time (pre-init) param setting,
544 * don't create a pool; that's done during init.
545 */
546 ret = param_set_charp(s, kp);
547 break;
548 case ZSWAP_INIT_SUCCEED:
549 new_pool = zswap_pool_changed(s, kp);
550 break;
551 case ZSWAP_INIT_FAILED:
552 pr_err("can't set param, initialization failed\n");
553 ret = -ENODEV;
554 }
555 mutex_unlock(&zswap_init_lock);
556
557 /* no need to create a new pool, return directly */
558 if (!new_pool)
559 return ret;
560
561 if (!type) {
562 if (!zpool_has_pool(s)) {
563 pr_err("zpool %s not available\n", s);
564 return -ENOENT;
565 }
566 type = s;
567 } else if (!compressor) {
568 if (!crypto_has_acomp(s, 0, 0)) {
569 pr_err("compressor %s not available\n", s);
570 return -ENOENT;
571 }
572 compressor = s;
573 } else {
574 WARN_ON(1);
575 return -EINVAL;
576 }
577
578 spin_lock_bh(&zswap_pools_lock);
579
580 pool = zswap_pool_find_get(type, compressor);
581 if (pool) {
582 zswap_pool_debug("using existing", pool);
583 WARN_ON(pool == zswap_pool_current());
584 list_del_rcu(&pool->list);
585 }
586
587 spin_unlock_bh(&zswap_pools_lock);
588
589 if (!pool)
590 pool = zswap_pool_create(type, compressor);
591 else {
592 /*
593 * Restore the initial ref dropped by percpu_ref_kill()
594 * when the pool was decommissioned and switch it again
595 * to percpu mode.
596 */
597 percpu_ref_resurrect(&pool->ref);
598
599 /* Drop the ref from zswap_pool_find_get(). */
600 zswap_pool_put(pool);
601 }
602
603 if (pool)
604 ret = param_set_charp(s, kp);
605 else
606 ret = -EINVAL;
607
608 spin_lock_bh(&zswap_pools_lock);
609
610 if (!ret) {
611 put_pool = zswap_pool_current();
612 list_add_rcu(&pool->list, &zswap_pools);
613 zswap_has_pool = true;
614 } else if (pool) {
615 /* add the possibly pre-existing pool to the end of the pools
616 * list; if it's new (and empty) then it'll be removed and
617 * destroyed by the put after we drop the lock
618 */
619 list_add_tail_rcu(&pool->list, &zswap_pools);
620 put_pool = pool;
621 }
622
623 spin_unlock_bh(&zswap_pools_lock);
624
625 if (!zswap_has_pool && !pool) {
626 /* if initial pool creation failed, and this pool creation also
627 * failed, maybe both compressor and zpool params were bad.
628 * Allow changing this param, so pool creation will succeed
629 * when the other param is changed. We already verified this
630 * param is ok in the zpool_has_pool() or crypto_has_acomp()
631 * checks above.
632 */
633 ret = param_set_charp(s, kp);
634 }
635
636 /* drop the ref from either the old current pool,
637 * or the new pool we failed to add
638 */
639 if (put_pool)
640 percpu_ref_kill(&put_pool->ref);
641
642 return ret;
643}
644
645static int zswap_compressor_param_set(const char *val,
646 const struct kernel_param *kp)
647{
648 return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
649}
650
651static int zswap_zpool_param_set(const char *val,
652 const struct kernel_param *kp)
653{
654 return __zswap_param_set(val, kp, NULL, zswap_compressor);
655}
656
657static int zswap_enabled_param_set(const char *val,
658 const struct kernel_param *kp)
659{
660 int ret = -ENODEV;
661
662 /* if this is load-time (pre-init) param setting, only set param. */
663 if (system_state != SYSTEM_RUNNING)
664 return param_set_bool(val, kp);
665
666 mutex_lock(&zswap_init_lock);
667 switch (zswap_init_state) {
668 case ZSWAP_UNINIT:
669 if (zswap_setup())
670 break;
671 fallthrough;
672 case ZSWAP_INIT_SUCCEED:
673 if (!zswap_has_pool)
674 pr_err("can't enable, no pool configured\n");
675 else
676 ret = param_set_bool(val, kp);
677 break;
678 case ZSWAP_INIT_FAILED:
679 pr_err("can't enable, initialization failed\n");
680 }
681 mutex_unlock(&zswap_init_lock);
682
683 return ret;
684}
685
686/*********************************
687* lru functions
688**********************************/
689
690/* should be called under RCU */
691#ifdef CONFIG_MEMCG
692static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
693{
694 return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL;
695}
696#else
697static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
698{
699 return NULL;
700}
701#endif
702
703static inline int entry_to_nid(struct zswap_entry *entry)
704{
705 return page_to_nid(virt_to_page(entry));
706}
707
708static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry)
709{
710 int nid = entry_to_nid(entry);
711 struct mem_cgroup *memcg;
712
713 /*
714 * Note that it is safe to use rcu_read_lock() here, even in the face of
715 * concurrent memcg offlining:
716 *
717 * 1. list_lru_add() is called before list_lru_one is dead. The
718 * new entry will be reparented to memcg's parent's list_lru.
719 * 2. list_lru_add() is called after list_lru_one is dead. The
720 * new entry will be added directly to memcg's parent's list_lru.
721 *
722 * Similar reasoning holds for list_lru_del().
723 */
724 rcu_read_lock();
725 memcg = mem_cgroup_from_entry(entry);
726 /* will always succeed */
727 list_lru_add(list_lru, &entry->lru, nid, memcg);
728 rcu_read_unlock();
729}
730
731static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry)
732{
733 int nid = entry_to_nid(entry);
734 struct mem_cgroup *memcg;
735
736 rcu_read_lock();
737 memcg = mem_cgroup_from_entry(entry);
738 /* will always succeed */
739 list_lru_del(list_lru, &entry->lru, nid, memcg);
740 rcu_read_unlock();
741}
742
743void zswap_lruvec_state_init(struct lruvec *lruvec)
744{
745 atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0);
746}
747
748void zswap_folio_swapin(struct folio *folio)
749{
750 struct lruvec *lruvec;
751
752 if (folio) {
753 lruvec = folio_lruvec(folio);
754 atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins);
755 }
756}
757
758/*
759 * This function should be called when a memcg is being offlined.
760 *
761 * Since the global shrinker shrink_worker() may hold a reference
762 * of the memcg, we must check and release the reference in
763 * zswap_next_shrink.
764 *
765 * shrink_worker() must handle the case where this function releases
766 * the reference of memcg being shrunk.
767 */
768void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg)
769{
770 /* lock out zswap shrinker walking memcg tree */
771 spin_lock(&zswap_shrink_lock);
772 if (zswap_next_shrink == memcg) {
773 do {
774 zswap_next_shrink = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
775 } while (zswap_next_shrink && !mem_cgroup_online(zswap_next_shrink));
776 }
777 spin_unlock(&zswap_shrink_lock);
778}
779
780/*********************************
781* zswap entry functions
782**********************************/
783static struct kmem_cache *zswap_entry_cache;
784
785static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid)
786{
787 struct zswap_entry *entry;
788 entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid);
789 if (!entry)
790 return NULL;
791 return entry;
792}
793
794static void zswap_entry_cache_free(struct zswap_entry *entry)
795{
796 kmem_cache_free(zswap_entry_cache, entry);
797}
798
799/*
800 * Carries out the common pattern of freeing and entry's zpool allocation,
801 * freeing the entry itself, and decrementing the number of stored pages.
802 */
803static void zswap_entry_free(struct zswap_entry *entry)
804{
805 zswap_lru_del(&zswap_list_lru, entry);
806 zpool_free(entry->pool->zpool, entry->handle);
807 zswap_pool_put(entry->pool);
808 if (entry->objcg) {
809 obj_cgroup_uncharge_zswap(entry->objcg, entry->length);
810 obj_cgroup_put(entry->objcg);
811 }
812 zswap_entry_cache_free(entry);
813 atomic_long_dec(&zswap_stored_pages);
814}
815
816/*********************************
817* compressed storage functions
818**********************************/
819static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
820{
821 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
822 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
823 struct crypto_acomp *acomp = NULL;
824 struct acomp_req *req = NULL;
825 u8 *buffer = NULL;
826 int ret;
827
828 buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
829 if (!buffer) {
830 ret = -ENOMEM;
831 goto fail;
832 }
833
834 acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));
835 if (IS_ERR(acomp)) {
836 pr_err("could not alloc crypto acomp %s : %ld\n",
837 pool->tfm_name, PTR_ERR(acomp));
838 ret = PTR_ERR(acomp);
839 goto fail;
840 }
841
842 req = acomp_request_alloc(acomp);
843 if (!req) {
844 pr_err("could not alloc crypto acomp_request %s\n",
845 pool->tfm_name);
846 ret = -ENOMEM;
847 goto fail;
848 }
849
850 /*
851 * Only hold the mutex after completing allocations, otherwise we may
852 * recurse into zswap through reclaim and attempt to hold the mutex
853 * again resulting in a deadlock.
854 */
855 mutex_lock(&acomp_ctx->mutex);
856 crypto_init_wait(&acomp_ctx->wait);
857
858 /*
859 * if the backend of acomp is async zip, crypto_req_done() will wakeup
860 * crypto_wait_req(); if the backend of acomp is scomp, the callback
861 * won't be called, crypto_wait_req() will return without blocking.
862 */
863 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
864 crypto_req_done, &acomp_ctx->wait);
865
866 acomp_ctx->buffer = buffer;
867 acomp_ctx->acomp = acomp;
868 acomp_ctx->is_sleepable = acomp_is_async(acomp);
869 acomp_ctx->req = req;
870 mutex_unlock(&acomp_ctx->mutex);
871 return 0;
872
873fail:
874 if (acomp)
875 crypto_free_acomp(acomp);
876 kfree(buffer);
877 return ret;
878}
879
880static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
881{
882 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
883 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
884
885 mutex_lock(&acomp_ctx->mutex);
886 if (!IS_ERR_OR_NULL(acomp_ctx)) {
887 if (!IS_ERR_OR_NULL(acomp_ctx->req))
888 acomp_request_free(acomp_ctx->req);
889 acomp_ctx->req = NULL;
890 if (!IS_ERR_OR_NULL(acomp_ctx->acomp))
891 crypto_free_acomp(acomp_ctx->acomp);
892 kfree(acomp_ctx->buffer);
893 }
894 mutex_unlock(&acomp_ctx->mutex);
895
896 return 0;
897}
898
899static struct crypto_acomp_ctx *acomp_ctx_get_cpu_lock(struct zswap_pool *pool)
900{
901 struct crypto_acomp_ctx *acomp_ctx;
902
903 for (;;) {
904 acomp_ctx = raw_cpu_ptr(pool->acomp_ctx);
905 mutex_lock(&acomp_ctx->mutex);
906 if (likely(acomp_ctx->req))
907 return acomp_ctx;
908 /*
909 * It is possible that we were migrated to a different CPU after
910 * getting the per-CPU ctx but before the mutex was acquired. If
911 * the old CPU got offlined, zswap_cpu_comp_dead() could have
912 * already freed ctx->req (among other things) and set it to
913 * NULL. Just try again on the new CPU that we ended up on.
914 */
915 mutex_unlock(&acomp_ctx->mutex);
916 }
917}
918
919static void acomp_ctx_put_unlock(struct crypto_acomp_ctx *acomp_ctx)
920{
921 mutex_unlock(&acomp_ctx->mutex);
922}
923
924static bool zswap_compress(struct page *page, struct zswap_entry *entry,
925 struct zswap_pool *pool)
926{
927 struct crypto_acomp_ctx *acomp_ctx;
928 struct scatterlist input, output;
929 int comp_ret = 0, alloc_ret = 0;
930 unsigned int dlen = PAGE_SIZE;
931 unsigned long handle;
932 struct zpool *zpool;
933 char *buf;
934 gfp_t gfp;
935 u8 *dst;
936
937 acomp_ctx = acomp_ctx_get_cpu_lock(pool);
938 dst = acomp_ctx->buffer;
939 sg_init_table(&input, 1);
940 sg_set_page(&input, page, PAGE_SIZE, 0);
941
942 /*
943 * We need PAGE_SIZE * 2 here since there maybe over-compression case,
944 * and hardware-accelerators may won't check the dst buffer size, so
945 * giving the dst buffer with enough length to avoid buffer overflow.
946 */
947 sg_init_one(&output, dst, PAGE_SIZE * 2);
948 acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
949
950 /*
951 * it maybe looks a little bit silly that we send an asynchronous request,
952 * then wait for its completion synchronously. This makes the process look
953 * synchronous in fact.
954 * Theoretically, acomp supports users send multiple acomp requests in one
955 * acomp instance, then get those requests done simultaneously. but in this
956 * case, zswap actually does store and load page by page, there is no
957 * existing method to send the second page before the first page is done
958 * in one thread doing zwap.
959 * but in different threads running on different cpu, we have different
960 * acomp instance, so multiple threads can do (de)compression in parallel.
961 */
962 comp_ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);
963 dlen = acomp_ctx->req->dlen;
964 if (comp_ret)
965 goto unlock;
966
967 zpool = pool->zpool;
968 gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
969 if (zpool_malloc_support_movable(zpool))
970 gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;
971 alloc_ret = zpool_malloc(zpool, dlen, gfp, &handle);
972 if (alloc_ret)
973 goto unlock;
974
975 buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);
976 memcpy(buf, dst, dlen);
977 zpool_unmap_handle(zpool, handle);
978
979 entry->handle = handle;
980 entry->length = dlen;
981
982unlock:
983 if (comp_ret == -ENOSPC || alloc_ret == -ENOSPC)
984 zswap_reject_compress_poor++;
985 else if (comp_ret)
986 zswap_reject_compress_fail++;
987 else if (alloc_ret)
988 zswap_reject_alloc_fail++;
989
990 acomp_ctx_put_unlock(acomp_ctx);
991 return comp_ret == 0 && alloc_ret == 0;
992}
993
994static void zswap_decompress(struct zswap_entry *entry, struct folio *folio)
995{
996 struct zpool *zpool = entry->pool->zpool;
997 struct scatterlist input, output;
998 struct crypto_acomp_ctx *acomp_ctx;
999 u8 *src;
1000
1001 acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool);
1002 src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
1003 /*
1004 * If zpool_map_handle is atomic, we cannot reliably utilize its mapped buffer
1005 * to do crypto_acomp_decompress() which might sleep. In such cases, we must
1006 * resort to copying the buffer to a temporary one.
1007 * Meanwhile, zpool_map_handle() might return a non-linearly mapped buffer,
1008 * such as a kmap address of high memory or even ever a vmap address.
1009 * However, sg_init_one is only equipped to handle linearly mapped low memory.
1010 * In such cases, we also must copy the buffer to a temporary and lowmem one.
1011 */
1012 if ((acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) ||
1013 !virt_addr_valid(src)) {
1014 memcpy(acomp_ctx->buffer, src, entry->length);
1015 src = acomp_ctx->buffer;
1016 zpool_unmap_handle(zpool, entry->handle);
1017 }
1018
1019 sg_init_one(&input, src, entry->length);
1020 sg_init_table(&output, 1);
1021 sg_set_folio(&output, folio, PAGE_SIZE, 0);
1022 acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);
1023 BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));
1024 BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
1025
1026 if (src != acomp_ctx->buffer)
1027 zpool_unmap_handle(zpool, entry->handle);
1028 acomp_ctx_put_unlock(acomp_ctx);
1029}
1030
1031/*********************************
1032* writeback code
1033**********************************/
1034/*
1035 * Attempts to free an entry by adding a folio to the swap cache,
1036 * decompressing the entry data into the folio, and issuing a
1037 * bio write to write the folio back to the swap device.
1038 *
1039 * This can be thought of as a "resumed writeback" of the folio
1040 * to the swap device. We are basically resuming the same swap
1041 * writeback path that was intercepted with the zswap_store()
1042 * in the first place. After the folio has been decompressed into
1043 * the swap cache, the compressed version stored by zswap can be
1044 * freed.
1045 */
1046static int zswap_writeback_entry(struct zswap_entry *entry,
1047 swp_entry_t swpentry)
1048{
1049 struct xarray *tree;
1050 pgoff_t offset = swp_offset(swpentry);
1051 struct folio *folio;
1052 struct mempolicy *mpol;
1053 bool folio_was_allocated;
1054 struct writeback_control wbc = {
1055 .sync_mode = WB_SYNC_NONE,
1056 };
1057
1058 /* try to allocate swap cache folio */
1059 mpol = get_task_policy(current);
1060 folio = __read_swap_cache_async(swpentry, GFP_KERNEL, mpol,
1061 NO_INTERLEAVE_INDEX, &folio_was_allocated, true);
1062 if (!folio)
1063 return -ENOMEM;
1064
1065 /*
1066 * Found an existing folio, we raced with swapin or concurrent
1067 * shrinker. We generally writeback cold folios from zswap, and
1068 * swapin means the folio just became hot, so skip this folio.
1069 * For unlikely concurrent shrinker case, it will be unlinked
1070 * and freed when invalidated by the concurrent shrinker anyway.
1071 */
1072 if (!folio_was_allocated) {
1073 folio_put(folio);
1074 return -EEXIST;
1075 }
1076
1077 /*
1078 * folio is locked, and the swapcache is now secured against
1079 * concurrent swapping to and from the slot, and concurrent
1080 * swapoff so we can safely dereference the zswap tree here.
1081 * Verify that the swap entry hasn't been invalidated and recycled
1082 * behind our backs, to avoid overwriting a new swap folio with
1083 * old compressed data. Only when this is successful can the entry
1084 * be dereferenced.
1085 */
1086 tree = swap_zswap_tree(swpentry);
1087 if (entry != xa_cmpxchg(tree, offset, entry, NULL, GFP_KERNEL)) {
1088 delete_from_swap_cache(folio);
1089 folio_unlock(folio);
1090 folio_put(folio);
1091 return -ENOMEM;
1092 }
1093
1094 zswap_decompress(entry, folio);
1095
1096 count_vm_event(ZSWPWB);
1097 if (entry->objcg)
1098 count_objcg_events(entry->objcg, ZSWPWB, 1);
1099
1100 zswap_entry_free(entry);
1101
1102 /* folio is up to date */
1103 folio_mark_uptodate(folio);
1104
1105 /* move it to the tail of the inactive list after end_writeback */
1106 folio_set_reclaim(folio);
1107
1108 /* start writeback */
1109 __swap_writepage(folio, &wbc);
1110 folio_put(folio);
1111
1112 return 0;
1113}
1114
1115/*********************************
1116* shrinker functions
1117**********************************/
1118/*
1119 * The dynamic shrinker is modulated by the following factors:
1120 *
1121 * 1. Each zswap entry has a referenced bit, which the shrinker unsets (giving
1122 * the entry a second chance) before rotating it in the LRU list. If the
1123 * entry is considered again by the shrinker, with its referenced bit unset,
1124 * it is written back. The writeback rate as a result is dynamically
1125 * adjusted by the pool activities - if the pool is dominated by new entries
1126 * (i.e lots of recent zswapouts), these entries will be protected and
1127 * the writeback rate will slow down. On the other hand, if the pool has a
1128 * lot of stagnant entries, these entries will be reclaimed immediately,
1129 * effectively increasing the writeback rate.
1130 *
1131 * 2. Swapins counter: If we observe swapins, it is a sign that we are
1132 * overshrinking and should slow down. We maintain a swapins counter, which
1133 * is consumed and subtract from the number of eligible objects on the LRU
1134 * in zswap_shrinker_count().
1135 *
1136 * 3. Compression ratio. The better the workload compresses, the less gains we
1137 * can expect from writeback. We scale down the number of objects available
1138 * for reclaim by this ratio.
1139 */
1140static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_one *l,
1141 void *arg)
1142{
1143 struct zswap_entry *entry = container_of(item, struct zswap_entry, lru);
1144 bool *encountered_page_in_swapcache = (bool *)arg;
1145 swp_entry_t swpentry;
1146 enum lru_status ret = LRU_REMOVED_RETRY;
1147 int writeback_result;
1148
1149 /*
1150 * Second chance algorithm: if the entry has its referenced bit set, give it
1151 * a second chance. Only clear the referenced bit and rotate it in the
1152 * zswap's LRU list.
1153 */
1154 if (entry->referenced) {
1155 entry->referenced = false;
1156 return LRU_ROTATE;
1157 }
1158
1159 /*
1160 * As soon as we drop the LRU lock, the entry can be freed by
1161 * a concurrent invalidation. This means the following:
1162 *
1163 * 1. We extract the swp_entry_t to the stack, allowing
1164 * zswap_writeback_entry() to pin the swap entry and
1165 * then validate the zwap entry against that swap entry's
1166 * tree using pointer value comparison. Only when that
1167 * is successful can the entry be dereferenced.
1168 *
1169 * 2. Usually, objects are taken off the LRU for reclaim. In
1170 * this case this isn't possible, because if reclaim fails
1171 * for whatever reason, we have no means of knowing if the
1172 * entry is alive to put it back on the LRU.
1173 *
1174 * So rotate it before dropping the lock. If the entry is
1175 * written back or invalidated, the free path will unlink
1176 * it. For failures, rotation is the right thing as well.
1177 *
1178 * Temporary failures, where the same entry should be tried
1179 * again immediately, almost never happen for this shrinker.
1180 * We don't do any trylocking; -ENOMEM comes closest,
1181 * but that's extremely rare and doesn't happen spuriously
1182 * either. Don't bother distinguishing this case.
1183 */
1184 list_move_tail(item, &l->list);
1185
1186 /*
1187 * Once the lru lock is dropped, the entry might get freed. The
1188 * swpentry is copied to the stack, and entry isn't deref'd again
1189 * until the entry is verified to still be alive in the tree.
1190 */
1191 swpentry = entry->swpentry;
1192
1193 /*
1194 * It's safe to drop the lock here because we return either
1195 * LRU_REMOVED_RETRY or LRU_RETRY.
1196 */
1197 spin_unlock(&l->lock);
1198
1199 writeback_result = zswap_writeback_entry(entry, swpentry);
1200
1201 if (writeback_result) {
1202 zswap_reject_reclaim_fail++;
1203 ret = LRU_RETRY;
1204
1205 /*
1206 * Encountering a page already in swap cache is a sign that we are shrinking
1207 * into the warmer region. We should terminate shrinking (if we're in the dynamic
1208 * shrinker context).
1209 */
1210 if (writeback_result == -EEXIST && encountered_page_in_swapcache) {
1211 ret = LRU_STOP;
1212 *encountered_page_in_swapcache = true;
1213 }
1214 } else {
1215 zswap_written_back_pages++;
1216 }
1217
1218 return ret;
1219}
1220
1221static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
1222 struct shrink_control *sc)
1223{
1224 unsigned long shrink_ret;
1225 bool encountered_page_in_swapcache = false;
1226
1227 if (!zswap_shrinker_enabled ||
1228 !mem_cgroup_zswap_writeback_enabled(sc->memcg)) {
1229 sc->nr_scanned = 0;
1230 return SHRINK_STOP;
1231 }
1232
1233 shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb,
1234 &encountered_page_in_swapcache);
1235
1236 if (encountered_page_in_swapcache)
1237 return SHRINK_STOP;
1238
1239 return shrink_ret ? shrink_ret : SHRINK_STOP;
1240}
1241
1242static unsigned long zswap_shrinker_count(struct shrinker *shrinker,
1243 struct shrink_control *sc)
1244{
1245 struct mem_cgroup *memcg = sc->memcg;
1246 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid));
1247 atomic_long_t *nr_disk_swapins =
1248 &lruvec->zswap_lruvec_state.nr_disk_swapins;
1249 unsigned long nr_backing, nr_stored, nr_freeable, nr_disk_swapins_cur,
1250 nr_remain;
1251
1252 if (!zswap_shrinker_enabled || !mem_cgroup_zswap_writeback_enabled(memcg))
1253 return 0;
1254
1255 /*
1256 * The shrinker resumes swap writeback, which will enter block
1257 * and may enter fs. XXX: Harmonize with vmscan.c __GFP_FS
1258 * rules (may_enter_fs()), which apply on a per-folio basis.
1259 */
1260 if (!gfp_has_io_fs(sc->gfp_mask))
1261 return 0;
1262
1263 /*
1264 * For memcg, use the cgroup-wide ZSWAP stats since we don't
1265 * have them per-node and thus per-lruvec. Careful if memcg is
1266 * runtime-disabled: we can get sc->memcg == NULL, which is ok
1267 * for the lruvec, but not for memcg_page_state().
1268 *
1269 * Without memcg, use the zswap pool-wide metrics.
1270 */
1271 if (!mem_cgroup_disabled()) {
1272 mem_cgroup_flush_stats(memcg);
1273 nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT;
1274 nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED);
1275 } else {
1276 nr_backing = zswap_total_pages();
1277 nr_stored = atomic_long_read(&zswap_stored_pages);
1278 }
1279
1280 if (!nr_stored)
1281 return 0;
1282
1283 nr_freeable = list_lru_shrink_count(&zswap_list_lru, sc);
1284 if (!nr_freeable)
1285 return 0;
1286
1287 /*
1288 * Subtract from the lru size the number of pages that are recently swapped
1289 * in from disk. The idea is that had we protect the zswap's LRU by this
1290 * amount of pages, these disk swapins would not have happened.
1291 */
1292 nr_disk_swapins_cur = atomic_long_read(nr_disk_swapins);
1293 do {
1294 if (nr_freeable >= nr_disk_swapins_cur)
1295 nr_remain = 0;
1296 else
1297 nr_remain = nr_disk_swapins_cur - nr_freeable;
1298 } while (!atomic_long_try_cmpxchg(
1299 nr_disk_swapins, &nr_disk_swapins_cur, nr_remain));
1300
1301 nr_freeable -= nr_disk_swapins_cur - nr_remain;
1302 if (!nr_freeable)
1303 return 0;
1304
1305 /*
1306 * Scale the number of freeable pages by the memory saving factor.
1307 * This ensures that the better zswap compresses memory, the fewer
1308 * pages we will evict to swap (as it will otherwise incur IO for
1309 * relatively small memory saving).
1310 */
1311 return mult_frac(nr_freeable, nr_backing, nr_stored);
1312}
1313
1314static struct shrinker *zswap_alloc_shrinker(void)
1315{
1316 struct shrinker *shrinker;
1317
1318 shrinker =
1319 shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, "mm-zswap");
1320 if (!shrinker)
1321 return NULL;
1322
1323 shrinker->scan_objects = zswap_shrinker_scan;
1324 shrinker->count_objects = zswap_shrinker_count;
1325 shrinker->batch = 0;
1326 shrinker->seeks = DEFAULT_SEEKS;
1327 return shrinker;
1328}
1329
1330static int shrink_memcg(struct mem_cgroup *memcg)
1331{
1332 int nid, shrunk = 0, scanned = 0;
1333
1334 if (!mem_cgroup_zswap_writeback_enabled(memcg))
1335 return -ENOENT;
1336
1337 /*
1338 * Skip zombies because their LRUs are reparented and we would be
1339 * reclaiming from the parent instead of the dead memcg.
1340 */
1341 if (memcg && !mem_cgroup_online(memcg))
1342 return -ENOENT;
1343
1344 for_each_node_state(nid, N_NORMAL_MEMORY) {
1345 unsigned long nr_to_walk = 1;
1346
1347 shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg,
1348 &shrink_memcg_cb, NULL, &nr_to_walk);
1349 scanned += 1 - nr_to_walk;
1350 }
1351
1352 if (!scanned)
1353 return -ENOENT;
1354
1355 return shrunk ? 0 : -EAGAIN;
1356}
1357
1358static void shrink_worker(struct work_struct *w)
1359{
1360 struct mem_cgroup *memcg;
1361 int ret, failures = 0, attempts = 0;
1362 unsigned long thr;
1363
1364 /* Reclaim down to the accept threshold */
1365 thr = zswap_accept_thr_pages();
1366
1367 /*
1368 * Global reclaim will select cgroup in a round-robin fashion from all
1369 * online memcgs, but memcgs that have no pages in zswap and
1370 * writeback-disabled memcgs (memory.zswap.writeback=0) are not
1371 * candidates for shrinking.
1372 *
1373 * Shrinking will be aborted if we encounter the following
1374 * MAX_RECLAIM_RETRIES times:
1375 * - No writeback-candidate memcgs found in a memcg tree walk.
1376 * - Shrinking a writeback-candidate memcg failed.
1377 *
1378 * We save iteration cursor memcg into zswap_next_shrink,
1379 * which can be modified by the offline memcg cleaner
1380 * zswap_memcg_offline_cleanup().
1381 *
1382 * Since the offline cleaner is called only once, we cannot leave an
1383 * offline memcg reference in zswap_next_shrink.
1384 * We can rely on the cleaner only if we get online memcg under lock.
1385 *
1386 * If we get an offline memcg, we cannot determine if the cleaner has
1387 * already been called or will be called later. We must put back the
1388 * reference before returning from this function. Otherwise, the
1389 * offline memcg left in zswap_next_shrink will hold the reference
1390 * until the next run of shrink_worker().
1391 */
1392 do {
1393 /*
1394 * Start shrinking from the next memcg after zswap_next_shrink.
1395 * When the offline cleaner has already advanced the cursor,
1396 * advancing the cursor here overlooks one memcg, but this
1397 * should be negligibly rare.
1398 *
1399 * If we get an online memcg, keep the extra reference in case
1400 * the original one obtained by mem_cgroup_iter() is dropped by
1401 * zswap_memcg_offline_cleanup() while we are shrinking the
1402 * memcg.
1403 */
1404 spin_lock(&zswap_shrink_lock);
1405 do {
1406 memcg = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
1407 zswap_next_shrink = memcg;
1408 } while (memcg && !mem_cgroup_tryget_online(memcg));
1409 spin_unlock(&zswap_shrink_lock);
1410
1411 if (!memcg) {
1412 /*
1413 * Continue shrinking without incrementing failures if
1414 * we found candidate memcgs in the last tree walk.
1415 */
1416 if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
1417 break;
1418
1419 attempts = 0;
1420 goto resched;
1421 }
1422
1423 ret = shrink_memcg(memcg);
1424 /* drop the extra reference */
1425 mem_cgroup_put(memcg);
1426
1427 /*
1428 * There are no writeback-candidate pages in the memcg.
1429 * This is not an issue as long as we can find another memcg
1430 * with pages in zswap. Skip this without incrementing attempts
1431 * and failures.
1432 */
1433 if (ret == -ENOENT)
1434 continue;
1435 ++attempts;
1436
1437 if (ret && ++failures == MAX_RECLAIM_RETRIES)
1438 break;
1439resched:
1440 cond_resched();
1441 } while (zswap_total_pages() > thr);
1442}
1443
1444/*********************************
1445* main API
1446**********************************/
1447
1448static bool zswap_store_page(struct page *page,
1449 struct obj_cgroup *objcg,
1450 struct zswap_pool *pool)
1451{
1452 swp_entry_t page_swpentry = page_swap_entry(page);
1453 struct zswap_entry *entry, *old;
1454
1455 /* allocate entry */
1456 entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
1457 if (!entry) {
1458 zswap_reject_kmemcache_fail++;
1459 return false;
1460 }
1461
1462 if (!zswap_compress(page, entry, pool))
1463 goto compress_failed;
1464
1465 old = xa_store(swap_zswap_tree(page_swpentry),
1466 swp_offset(page_swpentry),
1467 entry, GFP_KERNEL);
1468 if (xa_is_err(old)) {
1469 int err = xa_err(old);
1470
1471 WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
1472 zswap_reject_alloc_fail++;
1473 goto store_failed;
1474 }
1475
1476 /*
1477 * We may have had an existing entry that became stale when
1478 * the folio was redirtied and now the new version is being
1479 * swapped out. Get rid of the old.
1480 */
1481 if (old)
1482 zswap_entry_free(old);
1483
1484 /*
1485 * The entry is successfully compressed and stored in the tree, there is
1486 * no further possibility of failure. Grab refs to the pool and objcg,
1487 * charge zswap memory, and increment zswap_stored_pages.
1488 * The opposite actions will be performed by zswap_entry_free()
1489 * when the entry is removed from the tree.
1490 */
1491 zswap_pool_get(pool);
1492 if (objcg) {
1493 obj_cgroup_get(objcg);
1494 obj_cgroup_charge_zswap(objcg, entry->length);
1495 }
1496 atomic_long_inc(&zswap_stored_pages);
1497
1498 /*
1499 * We finish initializing the entry while it's already in xarray.
1500 * This is safe because:
1501 *
1502 * 1. Concurrent stores and invalidations are excluded by folio lock.
1503 *
1504 * 2. Writeback is excluded by the entry not being on the LRU yet.
1505 * The publishing order matters to prevent writeback from seeing
1506 * an incoherent entry.
1507 */
1508 entry->pool = pool;
1509 entry->swpentry = page_swpentry;
1510 entry->objcg = objcg;
1511 entry->referenced = true;
1512 if (entry->length) {
1513 INIT_LIST_HEAD(&entry->lru);
1514 zswap_lru_add(&zswap_list_lru, entry);
1515 }
1516
1517 return true;
1518
1519store_failed:
1520 zpool_free(pool->zpool, entry->handle);
1521compress_failed:
1522 zswap_entry_cache_free(entry);
1523 return false;
1524}
1525
1526bool zswap_store(struct folio *folio)
1527{
1528 long nr_pages = folio_nr_pages(folio);
1529 swp_entry_t swp = folio->swap;
1530 struct obj_cgroup *objcg = NULL;
1531 struct mem_cgroup *memcg = NULL;
1532 struct zswap_pool *pool;
1533 bool ret = false;
1534 long index;
1535
1536 VM_WARN_ON_ONCE(!folio_test_locked(folio));
1537 VM_WARN_ON_ONCE(!folio_test_swapcache(folio));
1538
1539 if (!zswap_enabled)
1540 goto check_old;
1541
1542 objcg = get_obj_cgroup_from_folio(folio);
1543 if (objcg && !obj_cgroup_may_zswap(objcg)) {
1544 memcg = get_mem_cgroup_from_objcg(objcg);
1545 if (shrink_memcg(memcg)) {
1546 mem_cgroup_put(memcg);
1547 goto put_objcg;
1548 }
1549 mem_cgroup_put(memcg);
1550 }
1551
1552 if (zswap_check_limits())
1553 goto put_objcg;
1554
1555 pool = zswap_pool_current_get();
1556 if (!pool)
1557 goto put_objcg;
1558
1559 if (objcg) {
1560 memcg = get_mem_cgroup_from_objcg(objcg);
1561 if (memcg_list_lru_alloc(memcg, &zswap_list_lru, GFP_KERNEL)) {
1562 mem_cgroup_put(memcg);
1563 goto put_pool;
1564 }
1565 mem_cgroup_put(memcg);
1566 }
1567
1568 for (index = 0; index < nr_pages; ++index) {
1569 struct page *page = folio_page(folio, index);
1570
1571 if (!zswap_store_page(page, objcg, pool))
1572 goto put_pool;
1573 }
1574
1575 if (objcg)
1576 count_objcg_events(objcg, ZSWPOUT, nr_pages);
1577
1578 count_vm_events(ZSWPOUT, nr_pages);
1579
1580 ret = true;
1581
1582put_pool:
1583 zswap_pool_put(pool);
1584put_objcg:
1585 obj_cgroup_put(objcg);
1586 if (!ret && zswap_pool_reached_full)
1587 queue_work(shrink_wq, &zswap_shrink_work);
1588check_old:
1589 /*
1590 * If the zswap store fails or zswap is disabled, we must invalidate
1591 * the possibly stale entries which were previously stored at the
1592 * offsets corresponding to each page of the folio. Otherwise,
1593 * writeback could overwrite the new data in the swapfile.
1594 */
1595 if (!ret) {
1596 unsigned type = swp_type(swp);
1597 pgoff_t offset = swp_offset(swp);
1598 struct zswap_entry *entry;
1599 struct xarray *tree;
1600
1601 for (index = 0; index < nr_pages; ++index) {
1602 tree = swap_zswap_tree(swp_entry(type, offset + index));
1603 entry = xa_erase(tree, offset + index);
1604 if (entry)
1605 zswap_entry_free(entry);
1606 }
1607 }
1608
1609 return ret;
1610}
1611
1612bool zswap_load(struct folio *folio)
1613{
1614 swp_entry_t swp = folio->swap;
1615 pgoff_t offset = swp_offset(swp);
1616 bool swapcache = folio_test_swapcache(folio);
1617 struct xarray *tree = swap_zswap_tree(swp);
1618 struct zswap_entry *entry;
1619
1620 VM_WARN_ON_ONCE(!folio_test_locked(folio));
1621
1622 if (zswap_never_enabled())
1623 return false;
1624
1625 /*
1626 * Large folios should not be swapped in while zswap is being used, as
1627 * they are not properly handled. Zswap does not properly load large
1628 * folios, and a large folio may only be partially in zswap.
1629 *
1630 * Return true without marking the folio uptodate so that an IO error is
1631 * emitted (e.g. do_swap_page() will sigbus).
1632 */
1633 if (WARN_ON_ONCE(folio_test_large(folio)))
1634 return true;
1635
1636 /*
1637 * When reading into the swapcache, invalidate our entry. The
1638 * swapcache can be the authoritative owner of the page and
1639 * its mappings, and the pressure that results from having two
1640 * in-memory copies outweighs any benefits of caching the
1641 * compression work.
1642 *
1643 * (Most swapins go through the swapcache. The notable
1644 * exception is the singleton fault on SWP_SYNCHRONOUS_IO
1645 * files, which reads into a private page and may free it if
1646 * the fault fails. We remain the primary owner of the entry.)
1647 */
1648 if (swapcache)
1649 entry = xa_erase(tree, offset);
1650 else
1651 entry = xa_load(tree, offset);
1652
1653 if (!entry)
1654 return false;
1655
1656 zswap_decompress(entry, folio);
1657
1658 count_vm_event(ZSWPIN);
1659 if (entry->objcg)
1660 count_objcg_events(entry->objcg, ZSWPIN, 1);
1661
1662 if (swapcache) {
1663 zswap_entry_free(entry);
1664 folio_mark_dirty(folio);
1665 }
1666
1667 folio_mark_uptodate(folio);
1668 return true;
1669}
1670
1671void zswap_invalidate(swp_entry_t swp)
1672{
1673 pgoff_t offset = swp_offset(swp);
1674 struct xarray *tree = swap_zswap_tree(swp);
1675 struct zswap_entry *entry;
1676
1677 if (xa_empty(tree))
1678 return;
1679
1680 entry = xa_erase(tree, offset);
1681 if (entry)
1682 zswap_entry_free(entry);
1683}
1684
1685int zswap_swapon(int type, unsigned long nr_pages)
1686{
1687 struct xarray *trees, *tree;
1688 unsigned int nr, i;
1689
1690 nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);
1691 trees = kvcalloc(nr, sizeof(*tree), GFP_KERNEL);
1692 if (!trees) {
1693 pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1694 return -ENOMEM;
1695 }
1696
1697 for (i = 0; i < nr; i++)
1698 xa_init(trees + i);
1699
1700 nr_zswap_trees[type] = nr;
1701 zswap_trees[type] = trees;
1702 return 0;
1703}
1704
1705void zswap_swapoff(int type)
1706{
1707 struct xarray *trees = zswap_trees[type];
1708 unsigned int i;
1709
1710 if (!trees)
1711 return;
1712
1713 /* try_to_unuse() invalidated all the entries already */
1714 for (i = 0; i < nr_zswap_trees[type]; i++)
1715 WARN_ON_ONCE(!xa_empty(trees + i));
1716
1717 kvfree(trees);
1718 nr_zswap_trees[type] = 0;
1719 zswap_trees[type] = NULL;
1720}
1721
1722/*********************************
1723* debugfs functions
1724**********************************/
1725#ifdef CONFIG_DEBUG_FS
1726#include <linux/debugfs.h>
1727
1728static struct dentry *zswap_debugfs_root;
1729
1730static int debugfs_get_total_size(void *data, u64 *val)
1731{
1732 *val = zswap_total_pages() * PAGE_SIZE;
1733 return 0;
1734}
1735DEFINE_DEBUGFS_ATTRIBUTE(total_size_fops, debugfs_get_total_size, NULL, "%llu\n");
1736
1737static int debugfs_get_stored_pages(void *data, u64 *val)
1738{
1739 *val = atomic_long_read(&zswap_stored_pages);
1740 return 0;
1741}
1742DEFINE_DEBUGFS_ATTRIBUTE(stored_pages_fops, debugfs_get_stored_pages, NULL, "%llu\n");
1743
1744static int zswap_debugfs_init(void)
1745{
1746 if (!debugfs_initialized())
1747 return -ENODEV;
1748
1749 zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1750
1751 debugfs_create_u64("pool_limit_hit", 0444,
1752 zswap_debugfs_root, &zswap_pool_limit_hit);
1753 debugfs_create_u64("reject_reclaim_fail", 0444,
1754 zswap_debugfs_root, &zswap_reject_reclaim_fail);
1755 debugfs_create_u64("reject_alloc_fail", 0444,
1756 zswap_debugfs_root, &zswap_reject_alloc_fail);
1757 debugfs_create_u64("reject_kmemcache_fail", 0444,
1758 zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1759 debugfs_create_u64("reject_compress_fail", 0444,
1760 zswap_debugfs_root, &zswap_reject_compress_fail);
1761 debugfs_create_u64("reject_compress_poor", 0444,
1762 zswap_debugfs_root, &zswap_reject_compress_poor);
1763 debugfs_create_u64("written_back_pages", 0444,
1764 zswap_debugfs_root, &zswap_written_back_pages);
1765 debugfs_create_file("pool_total_size", 0444,
1766 zswap_debugfs_root, NULL, &total_size_fops);
1767 debugfs_create_file("stored_pages", 0444,
1768 zswap_debugfs_root, NULL, &stored_pages_fops);
1769
1770 return 0;
1771}
1772#else
1773static int zswap_debugfs_init(void)
1774{
1775 return 0;
1776}
1777#endif
1778
1779/*********************************
1780* module init and exit
1781**********************************/
1782static int zswap_setup(void)
1783{
1784 struct zswap_pool *pool;
1785 int ret;
1786
1787 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
1788 if (!zswap_entry_cache) {
1789 pr_err("entry cache creation failed\n");
1790 goto cache_fail;
1791 }
1792
1793 ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
1794 "mm/zswap_pool:prepare",
1795 zswap_cpu_comp_prepare,
1796 zswap_cpu_comp_dead);
1797 if (ret)
1798 goto hp_fail;
1799
1800 shrink_wq = alloc_workqueue("zswap-shrink",
1801 WQ_UNBOUND|WQ_MEM_RECLAIM, 1);
1802 if (!shrink_wq)
1803 goto shrink_wq_fail;
1804
1805 zswap_shrinker = zswap_alloc_shrinker();
1806 if (!zswap_shrinker)
1807 goto shrinker_fail;
1808 if (list_lru_init_memcg(&zswap_list_lru, zswap_shrinker))
1809 goto lru_fail;
1810 shrinker_register(zswap_shrinker);
1811
1812 INIT_WORK(&zswap_shrink_work, shrink_worker);
1813
1814 pool = __zswap_pool_create_fallback();
1815 if (pool) {
1816 pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1817 zpool_get_type(pool->zpool));
1818 list_add(&pool->list, &zswap_pools);
1819 zswap_has_pool = true;
1820 static_branch_enable(&zswap_ever_enabled);
1821 } else {
1822 pr_err("pool creation failed\n");
1823 zswap_enabled = false;
1824 }
1825
1826 if (zswap_debugfs_init())
1827 pr_warn("debugfs initialization failed\n");
1828 zswap_init_state = ZSWAP_INIT_SUCCEED;
1829 return 0;
1830
1831lru_fail:
1832 shrinker_free(zswap_shrinker);
1833shrinker_fail:
1834 destroy_workqueue(shrink_wq);
1835shrink_wq_fail:
1836 cpuhp_remove_multi_state(CPUHP_MM_ZSWP_POOL_PREPARE);
1837hp_fail:
1838 kmem_cache_destroy(zswap_entry_cache);
1839cache_fail:
1840 /* if built-in, we aren't unloaded on failure; don't allow use */
1841 zswap_init_state = ZSWAP_INIT_FAILED;
1842 zswap_enabled = false;
1843 return -ENOMEM;
1844}
1845
1846static int __init zswap_init(void)
1847{
1848 if (!zswap_enabled)
1849 return 0;
1850 return zswap_setup();
1851}
1852/* must be late so crypto has time to come up */
1853late_initcall(zswap_init);
1854
1855MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1856MODULE_DESCRIPTION("Compressed cache for swap pages");