Loading...
1/*
2 * zswap.c - zswap driver file
3 *
4 * zswap is a backend for frontswap that takes pages that are in the process
5 * of being swapped out and attempts to compress and store them in a
6 * RAM-based memory pool. This can result in a significant I/O reduction on
7 * the swap device and, in the case where decompressing from RAM is faster
8 * than reading from the swap device, can also improve workload performance.
9 *
10 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com>
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU General Public License for more details.
21*/
22
23#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24
25#include <linux/module.h>
26#include <linux/cpu.h>
27#include <linux/highmem.h>
28#include <linux/slab.h>
29#include <linux/spinlock.h>
30#include <linux/types.h>
31#include <linux/atomic.h>
32#include <linux/frontswap.h>
33#include <linux/rbtree.h>
34#include <linux/swap.h>
35#include <linux/crypto.h>
36#include <linux/mempool.h>
37#include <linux/zpool.h>
38
39#include <linux/mm_types.h>
40#include <linux/page-flags.h>
41#include <linux/swapops.h>
42#include <linux/writeback.h>
43#include <linux/pagemap.h>
44
45/*********************************
46* statistics
47**********************************/
48/* Total bytes used by the compressed storage */
49static u64 zswap_pool_total_size;
50/* The number of compressed pages currently stored in zswap */
51static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
52
53/*
54 * The statistics below are not protected from concurrent access for
55 * performance reasons so they may not be a 100% accurate. However,
56 * they do provide useful information on roughly how many times a
57 * certain event is occurring.
58*/
59
60/* Pool limit was hit (see zswap_max_pool_percent) */
61static u64 zswap_pool_limit_hit;
62/* Pages written back when pool limit was reached */
63static u64 zswap_written_back_pages;
64/* Store failed due to a reclaim failure after pool limit was reached */
65static u64 zswap_reject_reclaim_fail;
66/* Compressed page was too big for the allocator to (optimally) store */
67static u64 zswap_reject_compress_poor;
68/* Store failed because underlying allocator could not get memory */
69static u64 zswap_reject_alloc_fail;
70/* Store failed because the entry metadata could not be allocated (rare) */
71static u64 zswap_reject_kmemcache_fail;
72/* Duplicate store was encountered (rare) */
73static u64 zswap_duplicate_entry;
74
75/*********************************
76* tunables
77**********************************/
78
79/* Enable/disable zswap (disabled by default) */
80static bool zswap_enabled;
81module_param_named(enabled, zswap_enabled, bool, 0644);
82
83/* Crypto compressor to use */
84#define ZSWAP_COMPRESSOR_DEFAULT "lzo"
85static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
86static int zswap_compressor_param_set(const char *,
87 const struct kernel_param *);
88static struct kernel_param_ops zswap_compressor_param_ops = {
89 .set = zswap_compressor_param_set,
90 .get = param_get_charp,
91 .free = param_free_charp,
92};
93module_param_cb(compressor, &zswap_compressor_param_ops,
94 &zswap_compressor, 0644);
95
96/* Compressed storage zpool to use */
97#define ZSWAP_ZPOOL_DEFAULT "zbud"
98static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
99static int zswap_zpool_param_set(const char *, const struct kernel_param *);
100static struct kernel_param_ops zswap_zpool_param_ops = {
101 .set = zswap_zpool_param_set,
102 .get = param_get_charp,
103 .free = param_free_charp,
104};
105module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
106
107/* The maximum percentage of memory that the compressed pool can occupy */
108static unsigned int zswap_max_pool_percent = 20;
109module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
110
111/*********************************
112* data structures
113**********************************/
114
115struct zswap_pool {
116 struct zpool *zpool;
117 struct crypto_comp * __percpu *tfm;
118 struct kref kref;
119 struct list_head list;
120 struct rcu_head rcu_head;
121 struct notifier_block notifier;
122 char tfm_name[CRYPTO_MAX_ALG_NAME];
123};
124
125/*
126 * struct zswap_entry
127 *
128 * This structure contains the metadata for tracking a single compressed
129 * page within zswap.
130 *
131 * rbnode - links the entry into red-black tree for the appropriate swap type
132 * offset - the swap offset for the entry. Index into the red-black tree.
133 * refcount - the number of outstanding reference to the entry. This is needed
134 * to protect against premature freeing of the entry by code
135 * concurrent calls to load, invalidate, and writeback. The lock
136 * for the zswap_tree structure that contains the entry must
137 * be held while changing the refcount. Since the lock must
138 * be held, there is no reason to also make refcount atomic.
139 * length - the length in bytes of the compressed page data. Needed during
140 * decompression
141 * pool - the zswap_pool the entry's data is in
142 * handle - zpool allocation handle that stores the compressed page data
143 */
144struct zswap_entry {
145 struct rb_node rbnode;
146 pgoff_t offset;
147 int refcount;
148 unsigned int length;
149 struct zswap_pool *pool;
150 unsigned long handle;
151};
152
153struct zswap_header {
154 swp_entry_t swpentry;
155};
156
157/*
158 * The tree lock in the zswap_tree struct protects a few things:
159 * - the rbtree
160 * - the refcount field of each entry in the tree
161 */
162struct zswap_tree {
163 struct rb_root rbroot;
164 spinlock_t lock;
165};
166
167static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
168
169/* RCU-protected iteration */
170static LIST_HEAD(zswap_pools);
171/* protects zswap_pools list modification */
172static DEFINE_SPINLOCK(zswap_pools_lock);
173/* pool counter to provide unique names to zpool */
174static atomic_t zswap_pools_count = ATOMIC_INIT(0);
175
176/* used by param callback function */
177static bool zswap_init_started;
178
179/*********************************
180* helpers and fwd declarations
181**********************************/
182
183#define zswap_pool_debug(msg, p) \
184 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
185 zpool_get_type((p)->zpool))
186
187static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
188static int zswap_pool_get(struct zswap_pool *pool);
189static void zswap_pool_put(struct zswap_pool *pool);
190
191static const struct zpool_ops zswap_zpool_ops = {
192 .evict = zswap_writeback_entry
193};
194
195static bool zswap_is_full(void)
196{
197 return totalram_pages * zswap_max_pool_percent / 100 <
198 DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
199}
200
201static void zswap_update_total_size(void)
202{
203 struct zswap_pool *pool;
204 u64 total = 0;
205
206 rcu_read_lock();
207
208 list_for_each_entry_rcu(pool, &zswap_pools, list)
209 total += zpool_get_total_size(pool->zpool);
210
211 rcu_read_unlock();
212
213 zswap_pool_total_size = total;
214}
215
216/*********************************
217* zswap entry functions
218**********************************/
219static struct kmem_cache *zswap_entry_cache;
220
221static int __init zswap_entry_cache_create(void)
222{
223 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
224 return zswap_entry_cache == NULL;
225}
226
227static void __init zswap_entry_cache_destroy(void)
228{
229 kmem_cache_destroy(zswap_entry_cache);
230}
231
232static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
233{
234 struct zswap_entry *entry;
235 entry = kmem_cache_alloc(zswap_entry_cache, gfp);
236 if (!entry)
237 return NULL;
238 entry->refcount = 1;
239 RB_CLEAR_NODE(&entry->rbnode);
240 return entry;
241}
242
243static void zswap_entry_cache_free(struct zswap_entry *entry)
244{
245 kmem_cache_free(zswap_entry_cache, entry);
246}
247
248/*********************************
249* rbtree functions
250**********************************/
251static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
252{
253 struct rb_node *node = root->rb_node;
254 struct zswap_entry *entry;
255
256 while (node) {
257 entry = rb_entry(node, struct zswap_entry, rbnode);
258 if (entry->offset > offset)
259 node = node->rb_left;
260 else if (entry->offset < offset)
261 node = node->rb_right;
262 else
263 return entry;
264 }
265 return NULL;
266}
267
268/*
269 * In the case that a entry with the same offset is found, a pointer to
270 * the existing entry is stored in dupentry and the function returns -EEXIST
271 */
272static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
273 struct zswap_entry **dupentry)
274{
275 struct rb_node **link = &root->rb_node, *parent = NULL;
276 struct zswap_entry *myentry;
277
278 while (*link) {
279 parent = *link;
280 myentry = rb_entry(parent, struct zswap_entry, rbnode);
281 if (myentry->offset > entry->offset)
282 link = &(*link)->rb_left;
283 else if (myentry->offset < entry->offset)
284 link = &(*link)->rb_right;
285 else {
286 *dupentry = myentry;
287 return -EEXIST;
288 }
289 }
290 rb_link_node(&entry->rbnode, parent, link);
291 rb_insert_color(&entry->rbnode, root);
292 return 0;
293}
294
295static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
296{
297 if (!RB_EMPTY_NODE(&entry->rbnode)) {
298 rb_erase(&entry->rbnode, root);
299 RB_CLEAR_NODE(&entry->rbnode);
300 }
301}
302
303/*
304 * Carries out the common pattern of freeing and entry's zpool allocation,
305 * freeing the entry itself, and decrementing the number of stored pages.
306 */
307static void zswap_free_entry(struct zswap_entry *entry)
308{
309 zpool_free(entry->pool->zpool, entry->handle);
310 zswap_pool_put(entry->pool);
311 zswap_entry_cache_free(entry);
312 atomic_dec(&zswap_stored_pages);
313 zswap_update_total_size();
314}
315
316/* caller must hold the tree lock */
317static void zswap_entry_get(struct zswap_entry *entry)
318{
319 entry->refcount++;
320}
321
322/* caller must hold the tree lock
323* remove from the tree and free it, if nobody reference the entry
324*/
325static void zswap_entry_put(struct zswap_tree *tree,
326 struct zswap_entry *entry)
327{
328 int refcount = --entry->refcount;
329
330 BUG_ON(refcount < 0);
331 if (refcount == 0) {
332 zswap_rb_erase(&tree->rbroot, entry);
333 zswap_free_entry(entry);
334 }
335}
336
337/* caller must hold the tree lock */
338static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
339 pgoff_t offset)
340{
341 struct zswap_entry *entry;
342
343 entry = zswap_rb_search(root, offset);
344 if (entry)
345 zswap_entry_get(entry);
346
347 return entry;
348}
349
350/*********************************
351* per-cpu code
352**********************************/
353static DEFINE_PER_CPU(u8 *, zswap_dstmem);
354
355static int __zswap_cpu_dstmem_notifier(unsigned long action, unsigned long cpu)
356{
357 u8 *dst;
358
359 switch (action) {
360 case CPU_UP_PREPARE:
361 dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
362 if (!dst) {
363 pr_err("can't allocate compressor buffer\n");
364 return NOTIFY_BAD;
365 }
366 per_cpu(zswap_dstmem, cpu) = dst;
367 break;
368 case CPU_DEAD:
369 case CPU_UP_CANCELED:
370 dst = per_cpu(zswap_dstmem, cpu);
371 kfree(dst);
372 per_cpu(zswap_dstmem, cpu) = NULL;
373 break;
374 default:
375 break;
376 }
377 return NOTIFY_OK;
378}
379
380static int zswap_cpu_dstmem_notifier(struct notifier_block *nb,
381 unsigned long action, void *pcpu)
382{
383 return __zswap_cpu_dstmem_notifier(action, (unsigned long)pcpu);
384}
385
386static struct notifier_block zswap_dstmem_notifier = {
387 .notifier_call = zswap_cpu_dstmem_notifier,
388};
389
390static int __init zswap_cpu_dstmem_init(void)
391{
392 unsigned long cpu;
393
394 cpu_notifier_register_begin();
395 for_each_online_cpu(cpu)
396 if (__zswap_cpu_dstmem_notifier(CPU_UP_PREPARE, cpu) ==
397 NOTIFY_BAD)
398 goto cleanup;
399 __register_cpu_notifier(&zswap_dstmem_notifier);
400 cpu_notifier_register_done();
401 return 0;
402
403cleanup:
404 for_each_online_cpu(cpu)
405 __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
406 cpu_notifier_register_done();
407 return -ENOMEM;
408}
409
410static void zswap_cpu_dstmem_destroy(void)
411{
412 unsigned long cpu;
413
414 cpu_notifier_register_begin();
415 for_each_online_cpu(cpu)
416 __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
417 __unregister_cpu_notifier(&zswap_dstmem_notifier);
418 cpu_notifier_register_done();
419}
420
421static int __zswap_cpu_comp_notifier(struct zswap_pool *pool,
422 unsigned long action, unsigned long cpu)
423{
424 struct crypto_comp *tfm;
425
426 switch (action) {
427 case CPU_UP_PREPARE:
428 if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
429 break;
430 tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
431 if (IS_ERR_OR_NULL(tfm)) {
432 pr_err("could not alloc crypto comp %s : %ld\n",
433 pool->tfm_name, PTR_ERR(tfm));
434 return NOTIFY_BAD;
435 }
436 *per_cpu_ptr(pool->tfm, cpu) = tfm;
437 break;
438 case CPU_DEAD:
439 case CPU_UP_CANCELED:
440 tfm = *per_cpu_ptr(pool->tfm, cpu);
441 if (!IS_ERR_OR_NULL(tfm))
442 crypto_free_comp(tfm);
443 *per_cpu_ptr(pool->tfm, cpu) = NULL;
444 break;
445 default:
446 break;
447 }
448 return NOTIFY_OK;
449}
450
451static int zswap_cpu_comp_notifier(struct notifier_block *nb,
452 unsigned long action, void *pcpu)
453{
454 unsigned long cpu = (unsigned long)pcpu;
455 struct zswap_pool *pool = container_of(nb, typeof(*pool), notifier);
456
457 return __zswap_cpu_comp_notifier(pool, action, cpu);
458}
459
460static int zswap_cpu_comp_init(struct zswap_pool *pool)
461{
462 unsigned long cpu;
463
464 memset(&pool->notifier, 0, sizeof(pool->notifier));
465 pool->notifier.notifier_call = zswap_cpu_comp_notifier;
466
467 cpu_notifier_register_begin();
468 for_each_online_cpu(cpu)
469 if (__zswap_cpu_comp_notifier(pool, CPU_UP_PREPARE, cpu) ==
470 NOTIFY_BAD)
471 goto cleanup;
472 __register_cpu_notifier(&pool->notifier);
473 cpu_notifier_register_done();
474 return 0;
475
476cleanup:
477 for_each_online_cpu(cpu)
478 __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
479 cpu_notifier_register_done();
480 return -ENOMEM;
481}
482
483static void zswap_cpu_comp_destroy(struct zswap_pool *pool)
484{
485 unsigned long cpu;
486
487 cpu_notifier_register_begin();
488 for_each_online_cpu(cpu)
489 __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
490 __unregister_cpu_notifier(&pool->notifier);
491 cpu_notifier_register_done();
492}
493
494/*********************************
495* pool functions
496**********************************/
497
498static struct zswap_pool *__zswap_pool_current(void)
499{
500 struct zswap_pool *pool;
501
502 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
503 WARN_ON(!pool);
504
505 return pool;
506}
507
508static struct zswap_pool *zswap_pool_current(void)
509{
510 assert_spin_locked(&zswap_pools_lock);
511
512 return __zswap_pool_current();
513}
514
515static struct zswap_pool *zswap_pool_current_get(void)
516{
517 struct zswap_pool *pool;
518
519 rcu_read_lock();
520
521 pool = __zswap_pool_current();
522 if (!pool || !zswap_pool_get(pool))
523 pool = NULL;
524
525 rcu_read_unlock();
526
527 return pool;
528}
529
530static struct zswap_pool *zswap_pool_last_get(void)
531{
532 struct zswap_pool *pool, *last = NULL;
533
534 rcu_read_lock();
535
536 list_for_each_entry_rcu(pool, &zswap_pools, list)
537 last = pool;
538 if (!WARN_ON(!last) && !zswap_pool_get(last))
539 last = NULL;
540
541 rcu_read_unlock();
542
543 return last;
544}
545
546/* type and compressor must be null-terminated */
547static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
548{
549 struct zswap_pool *pool;
550
551 assert_spin_locked(&zswap_pools_lock);
552
553 list_for_each_entry_rcu(pool, &zswap_pools, list) {
554 if (strcmp(pool->tfm_name, compressor))
555 continue;
556 if (strcmp(zpool_get_type(pool->zpool), type))
557 continue;
558 /* if we can't get it, it's about to be destroyed */
559 if (!zswap_pool_get(pool))
560 continue;
561 return pool;
562 }
563
564 return NULL;
565}
566
567static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
568{
569 struct zswap_pool *pool;
570 char name[38]; /* 'zswap' + 32 char (max) num + \0 */
571 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
572
573 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
574 if (!pool) {
575 pr_err("pool alloc failed\n");
576 return NULL;
577 }
578
579 /* unique name for each pool specifically required by zsmalloc */
580 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
581
582 pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);
583 if (!pool->zpool) {
584 pr_err("%s zpool not available\n", type);
585 goto error;
586 }
587 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
588
589 strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
590 pool->tfm = alloc_percpu(struct crypto_comp *);
591 if (!pool->tfm) {
592 pr_err("percpu alloc failed\n");
593 goto error;
594 }
595
596 if (zswap_cpu_comp_init(pool))
597 goto error;
598 pr_debug("using %s compressor\n", pool->tfm_name);
599
600 /* being the current pool takes 1 ref; this func expects the
601 * caller to always add the new pool as the current pool
602 */
603 kref_init(&pool->kref);
604 INIT_LIST_HEAD(&pool->list);
605
606 zswap_pool_debug("created", pool);
607
608 return pool;
609
610error:
611 free_percpu(pool->tfm);
612 if (pool->zpool)
613 zpool_destroy_pool(pool->zpool);
614 kfree(pool);
615 return NULL;
616}
617
618static __init struct zswap_pool *__zswap_pool_create_fallback(void)
619{
620 if (!crypto_has_comp(zswap_compressor, 0, 0)) {
621 if (!strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) {
622 pr_err("default compressor %s not available\n",
623 zswap_compressor);
624 return NULL;
625 }
626 pr_err("compressor %s not available, using default %s\n",
627 zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT);
628 param_free_charp(&zswap_compressor);
629 zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
630 }
631 if (!zpool_has_pool(zswap_zpool_type)) {
632 if (!strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) {
633 pr_err("default zpool %s not available\n",
634 zswap_zpool_type);
635 return NULL;
636 }
637 pr_err("zpool %s not available, using default %s\n",
638 zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT);
639 param_free_charp(&zswap_zpool_type);
640 zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
641 }
642
643 return zswap_pool_create(zswap_zpool_type, zswap_compressor);
644}
645
646static void zswap_pool_destroy(struct zswap_pool *pool)
647{
648 zswap_pool_debug("destroying", pool);
649
650 zswap_cpu_comp_destroy(pool);
651 free_percpu(pool->tfm);
652 zpool_destroy_pool(pool->zpool);
653 kfree(pool);
654}
655
656static int __must_check zswap_pool_get(struct zswap_pool *pool)
657{
658 return kref_get_unless_zero(&pool->kref);
659}
660
661static void __zswap_pool_release(struct rcu_head *head)
662{
663 struct zswap_pool *pool = container_of(head, typeof(*pool), rcu_head);
664
665 /* nobody should have been able to get a kref... */
666 WARN_ON(kref_get_unless_zero(&pool->kref));
667
668 /* pool is now off zswap_pools list and has no references. */
669 zswap_pool_destroy(pool);
670}
671
672static void __zswap_pool_empty(struct kref *kref)
673{
674 struct zswap_pool *pool;
675
676 pool = container_of(kref, typeof(*pool), kref);
677
678 spin_lock(&zswap_pools_lock);
679
680 WARN_ON(pool == zswap_pool_current());
681
682 list_del_rcu(&pool->list);
683 call_rcu(&pool->rcu_head, __zswap_pool_release);
684
685 spin_unlock(&zswap_pools_lock);
686}
687
688static void zswap_pool_put(struct zswap_pool *pool)
689{
690 kref_put(&pool->kref, __zswap_pool_empty);
691}
692
693/*********************************
694* param callbacks
695**********************************/
696
697/* val must be a null-terminated string */
698static int __zswap_param_set(const char *val, const struct kernel_param *kp,
699 char *type, char *compressor)
700{
701 struct zswap_pool *pool, *put_pool = NULL;
702 char *s = strstrip((char *)val);
703 int ret;
704
705 /* no change required */
706 if (!strcmp(s, *(char **)kp->arg))
707 return 0;
708
709 /* if this is load-time (pre-init) param setting,
710 * don't create a pool; that's done during init.
711 */
712 if (!zswap_init_started)
713 return param_set_charp(s, kp);
714
715 if (!type) {
716 if (!zpool_has_pool(s)) {
717 pr_err("zpool %s not available\n", s);
718 return -ENOENT;
719 }
720 type = s;
721 } else if (!compressor) {
722 if (!crypto_has_comp(s, 0, 0)) {
723 pr_err("compressor %s not available\n", s);
724 return -ENOENT;
725 }
726 compressor = s;
727 } else {
728 WARN_ON(1);
729 return -EINVAL;
730 }
731
732 spin_lock(&zswap_pools_lock);
733
734 pool = zswap_pool_find_get(type, compressor);
735 if (pool) {
736 zswap_pool_debug("using existing", pool);
737 list_del_rcu(&pool->list);
738 } else {
739 spin_unlock(&zswap_pools_lock);
740 pool = zswap_pool_create(type, compressor);
741 spin_lock(&zswap_pools_lock);
742 }
743
744 if (pool)
745 ret = param_set_charp(s, kp);
746 else
747 ret = -EINVAL;
748
749 if (!ret) {
750 put_pool = zswap_pool_current();
751 list_add_rcu(&pool->list, &zswap_pools);
752 } else if (pool) {
753 /* add the possibly pre-existing pool to the end of the pools
754 * list; if it's new (and empty) then it'll be removed and
755 * destroyed by the put after we drop the lock
756 */
757 list_add_tail_rcu(&pool->list, &zswap_pools);
758 put_pool = pool;
759 }
760
761 spin_unlock(&zswap_pools_lock);
762
763 /* drop the ref from either the old current pool,
764 * or the new pool we failed to add
765 */
766 if (put_pool)
767 zswap_pool_put(put_pool);
768
769 return ret;
770}
771
772static int zswap_compressor_param_set(const char *val,
773 const struct kernel_param *kp)
774{
775 return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
776}
777
778static int zswap_zpool_param_set(const char *val,
779 const struct kernel_param *kp)
780{
781 return __zswap_param_set(val, kp, NULL, zswap_compressor);
782}
783
784/*********************************
785* writeback code
786**********************************/
787/* return enum for zswap_get_swap_cache_page */
788enum zswap_get_swap_ret {
789 ZSWAP_SWAPCACHE_NEW,
790 ZSWAP_SWAPCACHE_EXIST,
791 ZSWAP_SWAPCACHE_FAIL,
792};
793
794/*
795 * zswap_get_swap_cache_page
796 *
797 * This is an adaption of read_swap_cache_async()
798 *
799 * This function tries to find a page with the given swap entry
800 * in the swapper_space address space (the swap cache). If the page
801 * is found, it is returned in retpage. Otherwise, a page is allocated,
802 * added to the swap cache, and returned in retpage.
803 *
804 * If success, the swap cache page is returned in retpage
805 * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
806 * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
807 * the new page is added to swapcache and locked
808 * Returns ZSWAP_SWAPCACHE_FAIL on error
809 */
810static int zswap_get_swap_cache_page(swp_entry_t entry,
811 struct page **retpage)
812{
813 bool page_was_allocated;
814
815 *retpage = __read_swap_cache_async(entry, GFP_KERNEL,
816 NULL, 0, &page_was_allocated);
817 if (page_was_allocated)
818 return ZSWAP_SWAPCACHE_NEW;
819 if (!*retpage)
820 return ZSWAP_SWAPCACHE_FAIL;
821 return ZSWAP_SWAPCACHE_EXIST;
822}
823
824/*
825 * Attempts to free an entry by adding a page to the swap cache,
826 * decompressing the entry data into the page, and issuing a
827 * bio write to write the page back to the swap device.
828 *
829 * This can be thought of as a "resumed writeback" of the page
830 * to the swap device. We are basically resuming the same swap
831 * writeback path that was intercepted with the frontswap_store()
832 * in the first place. After the page has been decompressed into
833 * the swap cache, the compressed version stored by zswap can be
834 * freed.
835 */
836static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
837{
838 struct zswap_header *zhdr;
839 swp_entry_t swpentry;
840 struct zswap_tree *tree;
841 pgoff_t offset;
842 struct zswap_entry *entry;
843 struct page *page;
844 struct crypto_comp *tfm;
845 u8 *src, *dst;
846 unsigned int dlen;
847 int ret;
848 struct writeback_control wbc = {
849 .sync_mode = WB_SYNC_NONE,
850 };
851
852 /* extract swpentry from data */
853 zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
854 swpentry = zhdr->swpentry; /* here */
855 zpool_unmap_handle(pool, handle);
856 tree = zswap_trees[swp_type(swpentry)];
857 offset = swp_offset(swpentry);
858
859 /* find and ref zswap entry */
860 spin_lock(&tree->lock);
861 entry = zswap_entry_find_get(&tree->rbroot, offset);
862 if (!entry) {
863 /* entry was invalidated */
864 spin_unlock(&tree->lock);
865 return 0;
866 }
867 spin_unlock(&tree->lock);
868 BUG_ON(offset != entry->offset);
869
870 /* try to allocate swap cache page */
871 switch (zswap_get_swap_cache_page(swpentry, &page)) {
872 case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
873 ret = -ENOMEM;
874 goto fail;
875
876 case ZSWAP_SWAPCACHE_EXIST:
877 /* page is already in the swap cache, ignore for now */
878 put_page(page);
879 ret = -EEXIST;
880 goto fail;
881
882 case ZSWAP_SWAPCACHE_NEW: /* page is locked */
883 /* decompress */
884 dlen = PAGE_SIZE;
885 src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
886 ZPOOL_MM_RO) + sizeof(struct zswap_header);
887 dst = kmap_atomic(page);
888 tfm = *get_cpu_ptr(entry->pool->tfm);
889 ret = crypto_comp_decompress(tfm, src, entry->length,
890 dst, &dlen);
891 put_cpu_ptr(entry->pool->tfm);
892 kunmap_atomic(dst);
893 zpool_unmap_handle(entry->pool->zpool, entry->handle);
894 BUG_ON(ret);
895 BUG_ON(dlen != PAGE_SIZE);
896
897 /* page is up to date */
898 SetPageUptodate(page);
899 }
900
901 /* move it to the tail of the inactive list after end_writeback */
902 SetPageReclaim(page);
903
904 /* start writeback */
905 __swap_writepage(page, &wbc, end_swap_bio_write);
906 put_page(page);
907 zswap_written_back_pages++;
908
909 spin_lock(&tree->lock);
910 /* drop local reference */
911 zswap_entry_put(tree, entry);
912
913 /*
914 * There are two possible situations for entry here:
915 * (1) refcount is 1(normal case), entry is valid and on the tree
916 * (2) refcount is 0, entry is freed and not on the tree
917 * because invalidate happened during writeback
918 * search the tree and free the entry if find entry
919 */
920 if (entry == zswap_rb_search(&tree->rbroot, offset))
921 zswap_entry_put(tree, entry);
922 spin_unlock(&tree->lock);
923
924 goto end;
925
926 /*
927 * if we get here due to ZSWAP_SWAPCACHE_EXIST
928 * a load may happening concurrently
929 * it is safe and okay to not free the entry
930 * if we free the entry in the following put
931 * it it either okay to return !0
932 */
933fail:
934 spin_lock(&tree->lock);
935 zswap_entry_put(tree, entry);
936 spin_unlock(&tree->lock);
937
938end:
939 return ret;
940}
941
942static int zswap_shrink(void)
943{
944 struct zswap_pool *pool;
945 int ret;
946
947 pool = zswap_pool_last_get();
948 if (!pool)
949 return -ENOENT;
950
951 ret = zpool_shrink(pool->zpool, 1, NULL);
952
953 zswap_pool_put(pool);
954
955 return ret;
956}
957
958/*********************************
959* frontswap hooks
960**********************************/
961/* attempts to compress and store an single page */
962static int zswap_frontswap_store(unsigned type, pgoff_t offset,
963 struct page *page)
964{
965 struct zswap_tree *tree = zswap_trees[type];
966 struct zswap_entry *entry, *dupentry;
967 struct crypto_comp *tfm;
968 int ret;
969 unsigned int dlen = PAGE_SIZE, len;
970 unsigned long handle;
971 char *buf;
972 u8 *src, *dst;
973 struct zswap_header *zhdr;
974
975 if (!zswap_enabled || !tree) {
976 ret = -ENODEV;
977 goto reject;
978 }
979
980 /* reclaim space if needed */
981 if (zswap_is_full()) {
982 zswap_pool_limit_hit++;
983 if (zswap_shrink()) {
984 zswap_reject_reclaim_fail++;
985 ret = -ENOMEM;
986 goto reject;
987 }
988 }
989
990 /* allocate entry */
991 entry = zswap_entry_cache_alloc(GFP_KERNEL);
992 if (!entry) {
993 zswap_reject_kmemcache_fail++;
994 ret = -ENOMEM;
995 goto reject;
996 }
997
998 /* if entry is successfully added, it keeps the reference */
999 entry->pool = zswap_pool_current_get();
1000 if (!entry->pool) {
1001 ret = -EINVAL;
1002 goto freepage;
1003 }
1004
1005 /* compress */
1006 dst = get_cpu_var(zswap_dstmem);
1007 tfm = *get_cpu_ptr(entry->pool->tfm);
1008 src = kmap_atomic(page);
1009 ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
1010 kunmap_atomic(src);
1011 put_cpu_ptr(entry->pool->tfm);
1012 if (ret) {
1013 ret = -EINVAL;
1014 goto put_dstmem;
1015 }
1016
1017 /* store */
1018 len = dlen + sizeof(struct zswap_header);
1019 ret = zpool_malloc(entry->pool->zpool, len,
1020 __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM,
1021 &handle);
1022 if (ret == -ENOSPC) {
1023 zswap_reject_compress_poor++;
1024 goto put_dstmem;
1025 }
1026 if (ret) {
1027 zswap_reject_alloc_fail++;
1028 goto put_dstmem;
1029 }
1030 zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
1031 zhdr->swpentry = swp_entry(type, offset);
1032 buf = (u8 *)(zhdr + 1);
1033 memcpy(buf, dst, dlen);
1034 zpool_unmap_handle(entry->pool->zpool, handle);
1035 put_cpu_var(zswap_dstmem);
1036
1037 /* populate entry */
1038 entry->offset = offset;
1039 entry->handle = handle;
1040 entry->length = dlen;
1041
1042 /* map */
1043 spin_lock(&tree->lock);
1044 do {
1045 ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
1046 if (ret == -EEXIST) {
1047 zswap_duplicate_entry++;
1048 /* remove from rbtree */
1049 zswap_rb_erase(&tree->rbroot, dupentry);
1050 zswap_entry_put(tree, dupentry);
1051 }
1052 } while (ret == -EEXIST);
1053 spin_unlock(&tree->lock);
1054
1055 /* update stats */
1056 atomic_inc(&zswap_stored_pages);
1057 zswap_update_total_size();
1058
1059 return 0;
1060
1061put_dstmem:
1062 put_cpu_var(zswap_dstmem);
1063 zswap_pool_put(entry->pool);
1064freepage:
1065 zswap_entry_cache_free(entry);
1066reject:
1067 return ret;
1068}
1069
1070/*
1071 * returns 0 if the page was successfully decompressed
1072 * return -1 on entry not found or error
1073*/
1074static int zswap_frontswap_load(unsigned type, pgoff_t offset,
1075 struct page *page)
1076{
1077 struct zswap_tree *tree = zswap_trees[type];
1078 struct zswap_entry *entry;
1079 struct crypto_comp *tfm;
1080 u8 *src, *dst;
1081 unsigned int dlen;
1082 int ret;
1083
1084 /* find */
1085 spin_lock(&tree->lock);
1086 entry = zswap_entry_find_get(&tree->rbroot, offset);
1087 if (!entry) {
1088 /* entry was written back */
1089 spin_unlock(&tree->lock);
1090 return -1;
1091 }
1092 spin_unlock(&tree->lock);
1093
1094 /* decompress */
1095 dlen = PAGE_SIZE;
1096 src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
1097 ZPOOL_MM_RO) + sizeof(struct zswap_header);
1098 dst = kmap_atomic(page);
1099 tfm = *get_cpu_ptr(entry->pool->tfm);
1100 ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
1101 put_cpu_ptr(entry->pool->tfm);
1102 kunmap_atomic(dst);
1103 zpool_unmap_handle(entry->pool->zpool, entry->handle);
1104 BUG_ON(ret);
1105
1106 spin_lock(&tree->lock);
1107 zswap_entry_put(tree, entry);
1108 spin_unlock(&tree->lock);
1109
1110 return 0;
1111}
1112
1113/* frees an entry in zswap */
1114static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
1115{
1116 struct zswap_tree *tree = zswap_trees[type];
1117 struct zswap_entry *entry;
1118
1119 /* find */
1120 spin_lock(&tree->lock);
1121 entry = zswap_rb_search(&tree->rbroot, offset);
1122 if (!entry) {
1123 /* entry was written back */
1124 spin_unlock(&tree->lock);
1125 return;
1126 }
1127
1128 /* remove from rbtree */
1129 zswap_rb_erase(&tree->rbroot, entry);
1130
1131 /* drop the initial reference from entry creation */
1132 zswap_entry_put(tree, entry);
1133
1134 spin_unlock(&tree->lock);
1135}
1136
1137/* frees all zswap entries for the given swap type */
1138static void zswap_frontswap_invalidate_area(unsigned type)
1139{
1140 struct zswap_tree *tree = zswap_trees[type];
1141 struct zswap_entry *entry, *n;
1142
1143 if (!tree)
1144 return;
1145
1146 /* walk the tree and free everything */
1147 spin_lock(&tree->lock);
1148 rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
1149 zswap_free_entry(entry);
1150 tree->rbroot = RB_ROOT;
1151 spin_unlock(&tree->lock);
1152 kfree(tree);
1153 zswap_trees[type] = NULL;
1154}
1155
1156static void zswap_frontswap_init(unsigned type)
1157{
1158 struct zswap_tree *tree;
1159
1160 tree = kzalloc(sizeof(struct zswap_tree), GFP_KERNEL);
1161 if (!tree) {
1162 pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1163 return;
1164 }
1165
1166 tree->rbroot = RB_ROOT;
1167 spin_lock_init(&tree->lock);
1168 zswap_trees[type] = tree;
1169}
1170
1171static struct frontswap_ops zswap_frontswap_ops = {
1172 .store = zswap_frontswap_store,
1173 .load = zswap_frontswap_load,
1174 .invalidate_page = zswap_frontswap_invalidate_page,
1175 .invalidate_area = zswap_frontswap_invalidate_area,
1176 .init = zswap_frontswap_init
1177};
1178
1179/*********************************
1180* debugfs functions
1181**********************************/
1182#ifdef CONFIG_DEBUG_FS
1183#include <linux/debugfs.h>
1184
1185static struct dentry *zswap_debugfs_root;
1186
1187static int __init zswap_debugfs_init(void)
1188{
1189 if (!debugfs_initialized())
1190 return -ENODEV;
1191
1192 zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1193 if (!zswap_debugfs_root)
1194 return -ENOMEM;
1195
1196 debugfs_create_u64("pool_limit_hit", S_IRUGO,
1197 zswap_debugfs_root, &zswap_pool_limit_hit);
1198 debugfs_create_u64("reject_reclaim_fail", S_IRUGO,
1199 zswap_debugfs_root, &zswap_reject_reclaim_fail);
1200 debugfs_create_u64("reject_alloc_fail", S_IRUGO,
1201 zswap_debugfs_root, &zswap_reject_alloc_fail);
1202 debugfs_create_u64("reject_kmemcache_fail", S_IRUGO,
1203 zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1204 debugfs_create_u64("reject_compress_poor", S_IRUGO,
1205 zswap_debugfs_root, &zswap_reject_compress_poor);
1206 debugfs_create_u64("written_back_pages", S_IRUGO,
1207 zswap_debugfs_root, &zswap_written_back_pages);
1208 debugfs_create_u64("duplicate_entry", S_IRUGO,
1209 zswap_debugfs_root, &zswap_duplicate_entry);
1210 debugfs_create_u64("pool_total_size", S_IRUGO,
1211 zswap_debugfs_root, &zswap_pool_total_size);
1212 debugfs_create_atomic_t("stored_pages", S_IRUGO,
1213 zswap_debugfs_root, &zswap_stored_pages);
1214
1215 return 0;
1216}
1217
1218static void __exit zswap_debugfs_exit(void)
1219{
1220 debugfs_remove_recursive(zswap_debugfs_root);
1221}
1222#else
1223static int __init zswap_debugfs_init(void)
1224{
1225 return 0;
1226}
1227
1228static void __exit zswap_debugfs_exit(void) { }
1229#endif
1230
1231/*********************************
1232* module init and exit
1233**********************************/
1234static int __init init_zswap(void)
1235{
1236 struct zswap_pool *pool;
1237
1238 zswap_init_started = true;
1239
1240 if (zswap_entry_cache_create()) {
1241 pr_err("entry cache creation failed\n");
1242 goto cache_fail;
1243 }
1244
1245 if (zswap_cpu_dstmem_init()) {
1246 pr_err("dstmem alloc failed\n");
1247 goto dstmem_fail;
1248 }
1249
1250 pool = __zswap_pool_create_fallback();
1251 if (!pool) {
1252 pr_err("pool creation failed\n");
1253 goto pool_fail;
1254 }
1255 pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1256 zpool_get_type(pool->zpool));
1257
1258 list_add(&pool->list, &zswap_pools);
1259
1260 frontswap_register_ops(&zswap_frontswap_ops);
1261 if (zswap_debugfs_init())
1262 pr_warn("debugfs initialization failed\n");
1263 return 0;
1264
1265pool_fail:
1266 zswap_cpu_dstmem_destroy();
1267dstmem_fail:
1268 zswap_entry_cache_destroy();
1269cache_fail:
1270 return -ENOMEM;
1271}
1272/* must be late so crypto has time to come up */
1273late_initcall(init_zswap);
1274
1275MODULE_LICENSE("GPL");
1276MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1277MODULE_DESCRIPTION("Compressed cache for swap pages");
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * zswap.c - zswap driver file
4 *
5 * zswap is a cache that takes pages that are in the process
6 * of being swapped out and attempts to compress and store them in a
7 * RAM-based memory pool. This can result in a significant I/O reduction on
8 * the swap device and, in the case where decompressing from RAM is faster
9 * than reading from the swap device, can also improve workload performance.
10 *
11 * Copyright (C) 2012 Seth Jennings <sjenning@linux.vnet.ibm.com>
12*/
13
14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16#include <linux/module.h>
17#include <linux/cpu.h>
18#include <linux/highmem.h>
19#include <linux/slab.h>
20#include <linux/spinlock.h>
21#include <linux/types.h>
22#include <linux/atomic.h>
23#include <linux/swap.h>
24#include <linux/crypto.h>
25#include <linux/scatterlist.h>
26#include <linux/mempolicy.h>
27#include <linux/mempool.h>
28#include <linux/zpool.h>
29#include <crypto/acompress.h>
30#include <linux/zswap.h>
31#include <linux/mm_types.h>
32#include <linux/page-flags.h>
33#include <linux/swapops.h>
34#include <linux/writeback.h>
35#include <linux/pagemap.h>
36#include <linux/workqueue.h>
37#include <linux/list_lru.h>
38
39#include "swap.h"
40#include "internal.h"
41
42/*********************************
43* statistics
44**********************************/
45/* The number of compressed pages currently stored in zswap */
46atomic_long_t zswap_stored_pages = ATOMIC_INIT(0);
47
48/*
49 * The statistics below are not protected from concurrent access for
50 * performance reasons so they may not be a 100% accurate. However,
51 * they do provide useful information on roughly how many times a
52 * certain event is occurring.
53*/
54
55/* Pool limit was hit (see zswap_max_pool_percent) */
56static u64 zswap_pool_limit_hit;
57/* Pages written back when pool limit was reached */
58static u64 zswap_written_back_pages;
59/* Store failed due to a reclaim failure after pool limit was reached */
60static u64 zswap_reject_reclaim_fail;
61/* Store failed due to compression algorithm failure */
62static u64 zswap_reject_compress_fail;
63/* Compressed page was too big for the allocator to (optimally) store */
64static u64 zswap_reject_compress_poor;
65/* Store failed because underlying allocator could not get memory */
66static u64 zswap_reject_alloc_fail;
67/* Store failed because the entry metadata could not be allocated (rare) */
68static u64 zswap_reject_kmemcache_fail;
69
70/* Shrinker work queue */
71static struct workqueue_struct *shrink_wq;
72/* Pool limit was hit, we need to calm down */
73static bool zswap_pool_reached_full;
74
75/*********************************
76* tunables
77**********************************/
78
79#define ZSWAP_PARAM_UNSET ""
80
81static int zswap_setup(void);
82
83/* Enable/disable zswap */
84static DEFINE_STATIC_KEY_MAYBE(CONFIG_ZSWAP_DEFAULT_ON, zswap_ever_enabled);
85static bool zswap_enabled = IS_ENABLED(CONFIG_ZSWAP_DEFAULT_ON);
86static int zswap_enabled_param_set(const char *,
87 const struct kernel_param *);
88static const struct kernel_param_ops zswap_enabled_param_ops = {
89 .set = zswap_enabled_param_set,
90 .get = param_get_bool,
91};
92module_param_cb(enabled, &zswap_enabled_param_ops, &zswap_enabled, 0644);
93
94/* Crypto compressor to use */
95static char *zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
96static int zswap_compressor_param_set(const char *,
97 const struct kernel_param *);
98static const struct kernel_param_ops zswap_compressor_param_ops = {
99 .set = zswap_compressor_param_set,
100 .get = param_get_charp,
101 .free = param_free_charp,
102};
103module_param_cb(compressor, &zswap_compressor_param_ops,
104 &zswap_compressor, 0644);
105
106/* Compressed storage zpool to use */
107static char *zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
108static int zswap_zpool_param_set(const char *, const struct kernel_param *);
109static const struct kernel_param_ops zswap_zpool_param_ops = {
110 .set = zswap_zpool_param_set,
111 .get = param_get_charp,
112 .free = param_free_charp,
113};
114module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
115
116/* The maximum percentage of memory that the compressed pool can occupy */
117static unsigned int zswap_max_pool_percent = 20;
118module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
119
120/* The threshold for accepting new pages after the max_pool_percent was hit */
121static unsigned int zswap_accept_thr_percent = 90; /* of max pool size */
122module_param_named(accept_threshold_percent, zswap_accept_thr_percent,
123 uint, 0644);
124
125/* Enable/disable memory pressure-based shrinker. */
126static bool zswap_shrinker_enabled = IS_ENABLED(
127 CONFIG_ZSWAP_SHRINKER_DEFAULT_ON);
128module_param_named(shrinker_enabled, zswap_shrinker_enabled, bool, 0644);
129
130bool zswap_is_enabled(void)
131{
132 return zswap_enabled;
133}
134
135bool zswap_never_enabled(void)
136{
137 return !static_branch_maybe(CONFIG_ZSWAP_DEFAULT_ON, &zswap_ever_enabled);
138}
139
140/*********************************
141* data structures
142**********************************/
143
144struct crypto_acomp_ctx {
145 struct crypto_acomp *acomp;
146 struct acomp_req *req;
147 struct crypto_wait wait;
148 u8 *buffer;
149 struct mutex mutex;
150 bool is_sleepable;
151};
152
153/*
154 * The lock ordering is zswap_tree.lock -> zswap_pool.lru_lock.
155 * The only case where lru_lock is not acquired while holding tree.lock is
156 * when a zswap_entry is taken off the lru for writeback, in that case it
157 * needs to be verified that it's still valid in the tree.
158 */
159struct zswap_pool {
160 struct zpool *zpool;
161 struct crypto_acomp_ctx __percpu *acomp_ctx;
162 struct percpu_ref ref;
163 struct list_head list;
164 struct work_struct release_work;
165 struct hlist_node node;
166 char tfm_name[CRYPTO_MAX_ALG_NAME];
167};
168
169/* Global LRU lists shared by all zswap pools. */
170static struct list_lru zswap_list_lru;
171
172/* The lock protects zswap_next_shrink updates. */
173static DEFINE_SPINLOCK(zswap_shrink_lock);
174static struct mem_cgroup *zswap_next_shrink;
175static struct work_struct zswap_shrink_work;
176static struct shrinker *zswap_shrinker;
177
178/*
179 * struct zswap_entry
180 *
181 * This structure contains the metadata for tracking a single compressed
182 * page within zswap.
183 *
184 * swpentry - associated swap entry, the offset indexes into the red-black tree
185 * length - the length in bytes of the compressed page data. Needed during
186 * decompression.
187 * referenced - true if the entry recently entered the zswap pool. Unset by the
188 * writeback logic. The entry is only reclaimed by the writeback
189 * logic if referenced is unset. See comments in the shrinker
190 * section for context.
191 * pool - the zswap_pool the entry's data is in
192 * handle - zpool allocation handle that stores the compressed page data
193 * objcg - the obj_cgroup that the compressed memory is charged to
194 * lru - handle to the pool's lru used to evict pages.
195 */
196struct zswap_entry {
197 swp_entry_t swpentry;
198 unsigned int length;
199 bool referenced;
200 struct zswap_pool *pool;
201 unsigned long handle;
202 struct obj_cgroup *objcg;
203 struct list_head lru;
204};
205
206static struct xarray *zswap_trees[MAX_SWAPFILES];
207static unsigned int nr_zswap_trees[MAX_SWAPFILES];
208
209/* RCU-protected iteration */
210static LIST_HEAD(zswap_pools);
211/* protects zswap_pools list modification */
212static DEFINE_SPINLOCK(zswap_pools_lock);
213/* pool counter to provide unique names to zpool */
214static atomic_t zswap_pools_count = ATOMIC_INIT(0);
215
216enum zswap_init_type {
217 ZSWAP_UNINIT,
218 ZSWAP_INIT_SUCCEED,
219 ZSWAP_INIT_FAILED
220};
221
222static enum zswap_init_type zswap_init_state;
223
224/* used to ensure the integrity of initialization */
225static DEFINE_MUTEX(zswap_init_lock);
226
227/* init completed, but couldn't create the initial pool */
228static bool zswap_has_pool;
229
230/*********************************
231* helpers and fwd declarations
232**********************************/
233
234static inline struct xarray *swap_zswap_tree(swp_entry_t swp)
235{
236 return &zswap_trees[swp_type(swp)][swp_offset(swp)
237 >> SWAP_ADDRESS_SPACE_SHIFT];
238}
239
240#define zswap_pool_debug(msg, p) \
241 pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name, \
242 zpool_get_type((p)->zpool))
243
244/*********************************
245* pool functions
246**********************************/
247static void __zswap_pool_empty(struct percpu_ref *ref);
248
249static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
250{
251 struct zswap_pool *pool;
252 char name[38]; /* 'zswap' + 32 char (max) num + \0 */
253 gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
254 int ret, cpu;
255
256 if (!zswap_has_pool) {
257 /* if either are unset, pool initialization failed, and we
258 * need both params to be set correctly before trying to
259 * create a pool.
260 */
261 if (!strcmp(type, ZSWAP_PARAM_UNSET))
262 return NULL;
263 if (!strcmp(compressor, ZSWAP_PARAM_UNSET))
264 return NULL;
265 }
266
267 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
268 if (!pool)
269 return NULL;
270
271 /* unique name for each pool specifically required by zsmalloc */
272 snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
273 pool->zpool = zpool_create_pool(type, name, gfp);
274 if (!pool->zpool) {
275 pr_err("%s zpool not available\n", type);
276 goto error;
277 }
278 pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
279
280 strscpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
281
282 pool->acomp_ctx = alloc_percpu(*pool->acomp_ctx);
283 if (!pool->acomp_ctx) {
284 pr_err("percpu alloc failed\n");
285 goto error;
286 }
287
288 for_each_possible_cpu(cpu)
289 mutex_init(&per_cpu_ptr(pool->acomp_ctx, cpu)->mutex);
290
291 ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE,
292 &pool->node);
293 if (ret)
294 goto error;
295
296 /* being the current pool takes 1 ref; this func expects the
297 * caller to always add the new pool as the current pool
298 */
299 ret = percpu_ref_init(&pool->ref, __zswap_pool_empty,
300 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
301 if (ret)
302 goto ref_fail;
303 INIT_LIST_HEAD(&pool->list);
304
305 zswap_pool_debug("created", pool);
306
307 return pool;
308
309ref_fail:
310 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
311error:
312 if (pool->acomp_ctx)
313 free_percpu(pool->acomp_ctx);
314 if (pool->zpool)
315 zpool_destroy_pool(pool->zpool);
316 kfree(pool);
317 return NULL;
318}
319
320static struct zswap_pool *__zswap_pool_create_fallback(void)
321{
322 bool has_comp, has_zpool;
323
324 has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
325 if (!has_comp && strcmp(zswap_compressor,
326 CONFIG_ZSWAP_COMPRESSOR_DEFAULT)) {
327 pr_err("compressor %s not available, using default %s\n",
328 zswap_compressor, CONFIG_ZSWAP_COMPRESSOR_DEFAULT);
329 param_free_charp(&zswap_compressor);
330 zswap_compressor = CONFIG_ZSWAP_COMPRESSOR_DEFAULT;
331 has_comp = crypto_has_acomp(zswap_compressor, 0, 0);
332 }
333 if (!has_comp) {
334 pr_err("default compressor %s not available\n",
335 zswap_compressor);
336 param_free_charp(&zswap_compressor);
337 zswap_compressor = ZSWAP_PARAM_UNSET;
338 }
339
340 has_zpool = zpool_has_pool(zswap_zpool_type);
341 if (!has_zpool && strcmp(zswap_zpool_type,
342 CONFIG_ZSWAP_ZPOOL_DEFAULT)) {
343 pr_err("zpool %s not available, using default %s\n",
344 zswap_zpool_type, CONFIG_ZSWAP_ZPOOL_DEFAULT);
345 param_free_charp(&zswap_zpool_type);
346 zswap_zpool_type = CONFIG_ZSWAP_ZPOOL_DEFAULT;
347 has_zpool = zpool_has_pool(zswap_zpool_type);
348 }
349 if (!has_zpool) {
350 pr_err("default zpool %s not available\n",
351 zswap_zpool_type);
352 param_free_charp(&zswap_zpool_type);
353 zswap_zpool_type = ZSWAP_PARAM_UNSET;
354 }
355
356 if (!has_comp || !has_zpool)
357 return NULL;
358
359 return zswap_pool_create(zswap_zpool_type, zswap_compressor);
360}
361
362static void zswap_pool_destroy(struct zswap_pool *pool)
363{
364 zswap_pool_debug("destroying", pool);
365
366 cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node);
367 free_percpu(pool->acomp_ctx);
368
369 zpool_destroy_pool(pool->zpool);
370 kfree(pool);
371}
372
373static void __zswap_pool_release(struct work_struct *work)
374{
375 struct zswap_pool *pool = container_of(work, typeof(*pool),
376 release_work);
377
378 synchronize_rcu();
379
380 /* nobody should have been able to get a ref... */
381 WARN_ON(!percpu_ref_is_zero(&pool->ref));
382 percpu_ref_exit(&pool->ref);
383
384 /* pool is now off zswap_pools list and has no references. */
385 zswap_pool_destroy(pool);
386}
387
388static struct zswap_pool *zswap_pool_current(void);
389
390static void __zswap_pool_empty(struct percpu_ref *ref)
391{
392 struct zswap_pool *pool;
393
394 pool = container_of(ref, typeof(*pool), ref);
395
396 spin_lock_bh(&zswap_pools_lock);
397
398 WARN_ON(pool == zswap_pool_current());
399
400 list_del_rcu(&pool->list);
401
402 INIT_WORK(&pool->release_work, __zswap_pool_release);
403 schedule_work(&pool->release_work);
404
405 spin_unlock_bh(&zswap_pools_lock);
406}
407
408static int __must_check zswap_pool_tryget(struct zswap_pool *pool)
409{
410 if (!pool)
411 return 0;
412
413 return percpu_ref_tryget(&pool->ref);
414}
415
416/* The caller must already have a reference. */
417static void zswap_pool_get(struct zswap_pool *pool)
418{
419 percpu_ref_get(&pool->ref);
420}
421
422static void zswap_pool_put(struct zswap_pool *pool)
423{
424 percpu_ref_put(&pool->ref);
425}
426
427static struct zswap_pool *__zswap_pool_current(void)
428{
429 struct zswap_pool *pool;
430
431 pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
432 WARN_ONCE(!pool && zswap_has_pool,
433 "%s: no page storage pool!\n", __func__);
434
435 return pool;
436}
437
438static struct zswap_pool *zswap_pool_current(void)
439{
440 assert_spin_locked(&zswap_pools_lock);
441
442 return __zswap_pool_current();
443}
444
445static struct zswap_pool *zswap_pool_current_get(void)
446{
447 struct zswap_pool *pool;
448
449 rcu_read_lock();
450
451 pool = __zswap_pool_current();
452 if (!zswap_pool_tryget(pool))
453 pool = NULL;
454
455 rcu_read_unlock();
456
457 return pool;
458}
459
460/* type and compressor must be null-terminated */
461static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
462{
463 struct zswap_pool *pool;
464
465 assert_spin_locked(&zswap_pools_lock);
466
467 list_for_each_entry_rcu(pool, &zswap_pools, list) {
468 if (strcmp(pool->tfm_name, compressor))
469 continue;
470 if (strcmp(zpool_get_type(pool->zpool), type))
471 continue;
472 /* if we can't get it, it's about to be destroyed */
473 if (!zswap_pool_tryget(pool))
474 continue;
475 return pool;
476 }
477
478 return NULL;
479}
480
481static unsigned long zswap_max_pages(void)
482{
483 return totalram_pages() * zswap_max_pool_percent / 100;
484}
485
486static unsigned long zswap_accept_thr_pages(void)
487{
488 return zswap_max_pages() * zswap_accept_thr_percent / 100;
489}
490
491unsigned long zswap_total_pages(void)
492{
493 struct zswap_pool *pool;
494 unsigned long total = 0;
495
496 rcu_read_lock();
497 list_for_each_entry_rcu(pool, &zswap_pools, list)
498 total += zpool_get_total_pages(pool->zpool);
499 rcu_read_unlock();
500
501 return total;
502}
503
504static bool zswap_check_limits(void)
505{
506 unsigned long cur_pages = zswap_total_pages();
507 unsigned long max_pages = zswap_max_pages();
508
509 if (cur_pages >= max_pages) {
510 zswap_pool_limit_hit++;
511 zswap_pool_reached_full = true;
512 } else if (zswap_pool_reached_full &&
513 cur_pages <= zswap_accept_thr_pages()) {
514 zswap_pool_reached_full = false;
515 }
516 return zswap_pool_reached_full;
517}
518
519/*********************************
520* param callbacks
521**********************************/
522
523static bool zswap_pool_changed(const char *s, const struct kernel_param *kp)
524{
525 /* no change required */
526 if (!strcmp(s, *(char **)kp->arg) && zswap_has_pool)
527 return false;
528 return true;
529}
530
531/* val must be a null-terminated string */
532static int __zswap_param_set(const char *val, const struct kernel_param *kp,
533 char *type, char *compressor)
534{
535 struct zswap_pool *pool, *put_pool = NULL;
536 char *s = strstrip((char *)val);
537 int ret = 0;
538 bool new_pool = false;
539
540 mutex_lock(&zswap_init_lock);
541 switch (zswap_init_state) {
542 case ZSWAP_UNINIT:
543 /* if this is load-time (pre-init) param setting,
544 * don't create a pool; that's done during init.
545 */
546 ret = param_set_charp(s, kp);
547 break;
548 case ZSWAP_INIT_SUCCEED:
549 new_pool = zswap_pool_changed(s, kp);
550 break;
551 case ZSWAP_INIT_FAILED:
552 pr_err("can't set param, initialization failed\n");
553 ret = -ENODEV;
554 }
555 mutex_unlock(&zswap_init_lock);
556
557 /* no need to create a new pool, return directly */
558 if (!new_pool)
559 return ret;
560
561 if (!type) {
562 if (!zpool_has_pool(s)) {
563 pr_err("zpool %s not available\n", s);
564 return -ENOENT;
565 }
566 type = s;
567 } else if (!compressor) {
568 if (!crypto_has_acomp(s, 0, 0)) {
569 pr_err("compressor %s not available\n", s);
570 return -ENOENT;
571 }
572 compressor = s;
573 } else {
574 WARN_ON(1);
575 return -EINVAL;
576 }
577
578 spin_lock_bh(&zswap_pools_lock);
579
580 pool = zswap_pool_find_get(type, compressor);
581 if (pool) {
582 zswap_pool_debug("using existing", pool);
583 WARN_ON(pool == zswap_pool_current());
584 list_del_rcu(&pool->list);
585 }
586
587 spin_unlock_bh(&zswap_pools_lock);
588
589 if (!pool)
590 pool = zswap_pool_create(type, compressor);
591 else {
592 /*
593 * Restore the initial ref dropped by percpu_ref_kill()
594 * when the pool was decommissioned and switch it again
595 * to percpu mode.
596 */
597 percpu_ref_resurrect(&pool->ref);
598
599 /* Drop the ref from zswap_pool_find_get(). */
600 zswap_pool_put(pool);
601 }
602
603 if (pool)
604 ret = param_set_charp(s, kp);
605 else
606 ret = -EINVAL;
607
608 spin_lock_bh(&zswap_pools_lock);
609
610 if (!ret) {
611 put_pool = zswap_pool_current();
612 list_add_rcu(&pool->list, &zswap_pools);
613 zswap_has_pool = true;
614 } else if (pool) {
615 /* add the possibly pre-existing pool to the end of the pools
616 * list; if it's new (and empty) then it'll be removed and
617 * destroyed by the put after we drop the lock
618 */
619 list_add_tail_rcu(&pool->list, &zswap_pools);
620 put_pool = pool;
621 }
622
623 spin_unlock_bh(&zswap_pools_lock);
624
625 if (!zswap_has_pool && !pool) {
626 /* if initial pool creation failed, and this pool creation also
627 * failed, maybe both compressor and zpool params were bad.
628 * Allow changing this param, so pool creation will succeed
629 * when the other param is changed. We already verified this
630 * param is ok in the zpool_has_pool() or crypto_has_acomp()
631 * checks above.
632 */
633 ret = param_set_charp(s, kp);
634 }
635
636 /* drop the ref from either the old current pool,
637 * or the new pool we failed to add
638 */
639 if (put_pool)
640 percpu_ref_kill(&put_pool->ref);
641
642 return ret;
643}
644
645static int zswap_compressor_param_set(const char *val,
646 const struct kernel_param *kp)
647{
648 return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
649}
650
651static int zswap_zpool_param_set(const char *val,
652 const struct kernel_param *kp)
653{
654 return __zswap_param_set(val, kp, NULL, zswap_compressor);
655}
656
657static int zswap_enabled_param_set(const char *val,
658 const struct kernel_param *kp)
659{
660 int ret = -ENODEV;
661
662 /* if this is load-time (pre-init) param setting, only set param. */
663 if (system_state != SYSTEM_RUNNING)
664 return param_set_bool(val, kp);
665
666 mutex_lock(&zswap_init_lock);
667 switch (zswap_init_state) {
668 case ZSWAP_UNINIT:
669 if (zswap_setup())
670 break;
671 fallthrough;
672 case ZSWAP_INIT_SUCCEED:
673 if (!zswap_has_pool)
674 pr_err("can't enable, no pool configured\n");
675 else
676 ret = param_set_bool(val, kp);
677 break;
678 case ZSWAP_INIT_FAILED:
679 pr_err("can't enable, initialization failed\n");
680 }
681 mutex_unlock(&zswap_init_lock);
682
683 return ret;
684}
685
686/*********************************
687* lru functions
688**********************************/
689
690/* should be called under RCU */
691#ifdef CONFIG_MEMCG
692static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
693{
694 return entry->objcg ? obj_cgroup_memcg(entry->objcg) : NULL;
695}
696#else
697static inline struct mem_cgroup *mem_cgroup_from_entry(struct zswap_entry *entry)
698{
699 return NULL;
700}
701#endif
702
703static inline int entry_to_nid(struct zswap_entry *entry)
704{
705 return page_to_nid(virt_to_page(entry));
706}
707
708static void zswap_lru_add(struct list_lru *list_lru, struct zswap_entry *entry)
709{
710 int nid = entry_to_nid(entry);
711 struct mem_cgroup *memcg;
712
713 /*
714 * Note that it is safe to use rcu_read_lock() here, even in the face of
715 * concurrent memcg offlining:
716 *
717 * 1. list_lru_add() is called before list_lru_one is dead. The
718 * new entry will be reparented to memcg's parent's list_lru.
719 * 2. list_lru_add() is called after list_lru_one is dead. The
720 * new entry will be added directly to memcg's parent's list_lru.
721 *
722 * Similar reasoning holds for list_lru_del().
723 */
724 rcu_read_lock();
725 memcg = mem_cgroup_from_entry(entry);
726 /* will always succeed */
727 list_lru_add(list_lru, &entry->lru, nid, memcg);
728 rcu_read_unlock();
729}
730
731static void zswap_lru_del(struct list_lru *list_lru, struct zswap_entry *entry)
732{
733 int nid = entry_to_nid(entry);
734 struct mem_cgroup *memcg;
735
736 rcu_read_lock();
737 memcg = mem_cgroup_from_entry(entry);
738 /* will always succeed */
739 list_lru_del(list_lru, &entry->lru, nid, memcg);
740 rcu_read_unlock();
741}
742
743void zswap_lruvec_state_init(struct lruvec *lruvec)
744{
745 atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0);
746}
747
748void zswap_folio_swapin(struct folio *folio)
749{
750 struct lruvec *lruvec;
751
752 if (folio) {
753 lruvec = folio_lruvec(folio);
754 atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins);
755 }
756}
757
758/*
759 * This function should be called when a memcg is being offlined.
760 *
761 * Since the global shrinker shrink_worker() may hold a reference
762 * of the memcg, we must check and release the reference in
763 * zswap_next_shrink.
764 *
765 * shrink_worker() must handle the case where this function releases
766 * the reference of memcg being shrunk.
767 */
768void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg)
769{
770 /* lock out zswap shrinker walking memcg tree */
771 spin_lock(&zswap_shrink_lock);
772 if (zswap_next_shrink == memcg) {
773 do {
774 zswap_next_shrink = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
775 } while (zswap_next_shrink && !mem_cgroup_online(zswap_next_shrink));
776 }
777 spin_unlock(&zswap_shrink_lock);
778}
779
780/*********************************
781* zswap entry functions
782**********************************/
783static struct kmem_cache *zswap_entry_cache;
784
785static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid)
786{
787 struct zswap_entry *entry;
788 entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid);
789 if (!entry)
790 return NULL;
791 return entry;
792}
793
794static void zswap_entry_cache_free(struct zswap_entry *entry)
795{
796 kmem_cache_free(zswap_entry_cache, entry);
797}
798
799/*
800 * Carries out the common pattern of freeing and entry's zpool allocation,
801 * freeing the entry itself, and decrementing the number of stored pages.
802 */
803static void zswap_entry_free(struct zswap_entry *entry)
804{
805 zswap_lru_del(&zswap_list_lru, entry);
806 zpool_free(entry->pool->zpool, entry->handle);
807 zswap_pool_put(entry->pool);
808 if (entry->objcg) {
809 obj_cgroup_uncharge_zswap(entry->objcg, entry->length);
810 obj_cgroup_put(entry->objcg);
811 }
812 zswap_entry_cache_free(entry);
813 atomic_long_dec(&zswap_stored_pages);
814}
815
816/*********************************
817* compressed storage functions
818**********************************/
819static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node)
820{
821 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
822 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
823 struct crypto_acomp *acomp = NULL;
824 struct acomp_req *req = NULL;
825 u8 *buffer = NULL;
826 int ret;
827
828 buffer = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
829 if (!buffer) {
830 ret = -ENOMEM;
831 goto fail;
832 }
833
834 acomp = crypto_alloc_acomp_node(pool->tfm_name, 0, 0, cpu_to_node(cpu));
835 if (IS_ERR(acomp)) {
836 pr_err("could not alloc crypto acomp %s : %ld\n",
837 pool->tfm_name, PTR_ERR(acomp));
838 ret = PTR_ERR(acomp);
839 goto fail;
840 }
841
842 req = acomp_request_alloc(acomp);
843 if (!req) {
844 pr_err("could not alloc crypto acomp_request %s\n",
845 pool->tfm_name);
846 ret = -ENOMEM;
847 goto fail;
848 }
849
850 /*
851 * Only hold the mutex after completing allocations, otherwise we may
852 * recurse into zswap through reclaim and attempt to hold the mutex
853 * again resulting in a deadlock.
854 */
855 mutex_lock(&acomp_ctx->mutex);
856 crypto_init_wait(&acomp_ctx->wait);
857
858 /*
859 * if the backend of acomp is async zip, crypto_req_done() will wakeup
860 * crypto_wait_req(); if the backend of acomp is scomp, the callback
861 * won't be called, crypto_wait_req() will return without blocking.
862 */
863 acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
864 crypto_req_done, &acomp_ctx->wait);
865
866 acomp_ctx->buffer = buffer;
867 acomp_ctx->acomp = acomp;
868 acomp_ctx->is_sleepable = acomp_is_async(acomp);
869 acomp_ctx->req = req;
870 mutex_unlock(&acomp_ctx->mutex);
871 return 0;
872
873fail:
874 if (acomp)
875 crypto_free_acomp(acomp);
876 kfree(buffer);
877 return ret;
878}
879
880static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node)
881{
882 struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node);
883 struct crypto_acomp_ctx *acomp_ctx = per_cpu_ptr(pool->acomp_ctx, cpu);
884
885 mutex_lock(&acomp_ctx->mutex);
886 if (!IS_ERR_OR_NULL(acomp_ctx)) {
887 if (!IS_ERR_OR_NULL(acomp_ctx->req))
888 acomp_request_free(acomp_ctx->req);
889 acomp_ctx->req = NULL;
890 if (!IS_ERR_OR_NULL(acomp_ctx->acomp))
891 crypto_free_acomp(acomp_ctx->acomp);
892 kfree(acomp_ctx->buffer);
893 }
894 mutex_unlock(&acomp_ctx->mutex);
895
896 return 0;
897}
898
899static struct crypto_acomp_ctx *acomp_ctx_get_cpu_lock(struct zswap_pool *pool)
900{
901 struct crypto_acomp_ctx *acomp_ctx;
902
903 for (;;) {
904 acomp_ctx = raw_cpu_ptr(pool->acomp_ctx);
905 mutex_lock(&acomp_ctx->mutex);
906 if (likely(acomp_ctx->req))
907 return acomp_ctx;
908 /*
909 * It is possible that we were migrated to a different CPU after
910 * getting the per-CPU ctx but before the mutex was acquired. If
911 * the old CPU got offlined, zswap_cpu_comp_dead() could have
912 * already freed ctx->req (among other things) and set it to
913 * NULL. Just try again on the new CPU that we ended up on.
914 */
915 mutex_unlock(&acomp_ctx->mutex);
916 }
917}
918
919static void acomp_ctx_put_unlock(struct crypto_acomp_ctx *acomp_ctx)
920{
921 mutex_unlock(&acomp_ctx->mutex);
922}
923
924static bool zswap_compress(struct page *page, struct zswap_entry *entry,
925 struct zswap_pool *pool)
926{
927 struct crypto_acomp_ctx *acomp_ctx;
928 struct scatterlist input, output;
929 int comp_ret = 0, alloc_ret = 0;
930 unsigned int dlen = PAGE_SIZE;
931 unsigned long handle;
932 struct zpool *zpool;
933 char *buf;
934 gfp_t gfp;
935 u8 *dst;
936
937 acomp_ctx = acomp_ctx_get_cpu_lock(pool);
938 dst = acomp_ctx->buffer;
939 sg_init_table(&input, 1);
940 sg_set_page(&input, page, PAGE_SIZE, 0);
941
942 /*
943 * We need PAGE_SIZE * 2 here since there maybe over-compression case,
944 * and hardware-accelerators may won't check the dst buffer size, so
945 * giving the dst buffer with enough length to avoid buffer overflow.
946 */
947 sg_init_one(&output, dst, PAGE_SIZE * 2);
948 acomp_request_set_params(acomp_ctx->req, &input, &output, PAGE_SIZE, dlen);
949
950 /*
951 * it maybe looks a little bit silly that we send an asynchronous request,
952 * then wait for its completion synchronously. This makes the process look
953 * synchronous in fact.
954 * Theoretically, acomp supports users send multiple acomp requests in one
955 * acomp instance, then get those requests done simultaneously. but in this
956 * case, zswap actually does store and load page by page, there is no
957 * existing method to send the second page before the first page is done
958 * in one thread doing zwap.
959 * but in different threads running on different cpu, we have different
960 * acomp instance, so multiple threads can do (de)compression in parallel.
961 */
962 comp_ret = crypto_wait_req(crypto_acomp_compress(acomp_ctx->req), &acomp_ctx->wait);
963 dlen = acomp_ctx->req->dlen;
964 if (comp_ret)
965 goto unlock;
966
967 zpool = pool->zpool;
968 gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
969 if (zpool_malloc_support_movable(zpool))
970 gfp |= __GFP_HIGHMEM | __GFP_MOVABLE;
971 alloc_ret = zpool_malloc(zpool, dlen, gfp, &handle);
972 if (alloc_ret)
973 goto unlock;
974
975 buf = zpool_map_handle(zpool, handle, ZPOOL_MM_WO);
976 memcpy(buf, dst, dlen);
977 zpool_unmap_handle(zpool, handle);
978
979 entry->handle = handle;
980 entry->length = dlen;
981
982unlock:
983 if (comp_ret == -ENOSPC || alloc_ret == -ENOSPC)
984 zswap_reject_compress_poor++;
985 else if (comp_ret)
986 zswap_reject_compress_fail++;
987 else if (alloc_ret)
988 zswap_reject_alloc_fail++;
989
990 acomp_ctx_put_unlock(acomp_ctx);
991 return comp_ret == 0 && alloc_ret == 0;
992}
993
994static void zswap_decompress(struct zswap_entry *entry, struct folio *folio)
995{
996 struct zpool *zpool = entry->pool->zpool;
997 struct scatterlist input, output;
998 struct crypto_acomp_ctx *acomp_ctx;
999 u8 *src;
1000
1001 acomp_ctx = acomp_ctx_get_cpu_lock(entry->pool);
1002 src = zpool_map_handle(zpool, entry->handle, ZPOOL_MM_RO);
1003 /*
1004 * If zpool_map_handle is atomic, we cannot reliably utilize its mapped buffer
1005 * to do crypto_acomp_decompress() which might sleep. In such cases, we must
1006 * resort to copying the buffer to a temporary one.
1007 * Meanwhile, zpool_map_handle() might return a non-linearly mapped buffer,
1008 * such as a kmap address of high memory or even ever a vmap address.
1009 * However, sg_init_one is only equipped to handle linearly mapped low memory.
1010 * In such cases, we also must copy the buffer to a temporary and lowmem one.
1011 */
1012 if ((acomp_ctx->is_sleepable && !zpool_can_sleep_mapped(zpool)) ||
1013 !virt_addr_valid(src)) {
1014 memcpy(acomp_ctx->buffer, src, entry->length);
1015 src = acomp_ctx->buffer;
1016 zpool_unmap_handle(zpool, entry->handle);
1017 }
1018
1019 sg_init_one(&input, src, entry->length);
1020 sg_init_table(&output, 1);
1021 sg_set_folio(&output, folio, PAGE_SIZE, 0);
1022 acomp_request_set_params(acomp_ctx->req, &input, &output, entry->length, PAGE_SIZE);
1023 BUG_ON(crypto_wait_req(crypto_acomp_decompress(acomp_ctx->req), &acomp_ctx->wait));
1024 BUG_ON(acomp_ctx->req->dlen != PAGE_SIZE);
1025
1026 if (src != acomp_ctx->buffer)
1027 zpool_unmap_handle(zpool, entry->handle);
1028 acomp_ctx_put_unlock(acomp_ctx);
1029}
1030
1031/*********************************
1032* writeback code
1033**********************************/
1034/*
1035 * Attempts to free an entry by adding a folio to the swap cache,
1036 * decompressing the entry data into the folio, and issuing a
1037 * bio write to write the folio back to the swap device.
1038 *
1039 * This can be thought of as a "resumed writeback" of the folio
1040 * to the swap device. We are basically resuming the same swap
1041 * writeback path that was intercepted with the zswap_store()
1042 * in the first place. After the folio has been decompressed into
1043 * the swap cache, the compressed version stored by zswap can be
1044 * freed.
1045 */
1046static int zswap_writeback_entry(struct zswap_entry *entry,
1047 swp_entry_t swpentry)
1048{
1049 struct xarray *tree;
1050 pgoff_t offset = swp_offset(swpentry);
1051 struct folio *folio;
1052 struct mempolicy *mpol;
1053 bool folio_was_allocated;
1054 struct writeback_control wbc = {
1055 .sync_mode = WB_SYNC_NONE,
1056 };
1057
1058 /* try to allocate swap cache folio */
1059 mpol = get_task_policy(current);
1060 folio = __read_swap_cache_async(swpentry, GFP_KERNEL, mpol,
1061 NO_INTERLEAVE_INDEX, &folio_was_allocated, true);
1062 if (!folio)
1063 return -ENOMEM;
1064
1065 /*
1066 * Found an existing folio, we raced with swapin or concurrent
1067 * shrinker. We generally writeback cold folios from zswap, and
1068 * swapin means the folio just became hot, so skip this folio.
1069 * For unlikely concurrent shrinker case, it will be unlinked
1070 * and freed when invalidated by the concurrent shrinker anyway.
1071 */
1072 if (!folio_was_allocated) {
1073 folio_put(folio);
1074 return -EEXIST;
1075 }
1076
1077 /*
1078 * folio is locked, and the swapcache is now secured against
1079 * concurrent swapping to and from the slot, and concurrent
1080 * swapoff so we can safely dereference the zswap tree here.
1081 * Verify that the swap entry hasn't been invalidated and recycled
1082 * behind our backs, to avoid overwriting a new swap folio with
1083 * old compressed data. Only when this is successful can the entry
1084 * be dereferenced.
1085 */
1086 tree = swap_zswap_tree(swpentry);
1087 if (entry != xa_cmpxchg(tree, offset, entry, NULL, GFP_KERNEL)) {
1088 delete_from_swap_cache(folio);
1089 folio_unlock(folio);
1090 folio_put(folio);
1091 return -ENOMEM;
1092 }
1093
1094 zswap_decompress(entry, folio);
1095
1096 count_vm_event(ZSWPWB);
1097 if (entry->objcg)
1098 count_objcg_events(entry->objcg, ZSWPWB, 1);
1099
1100 zswap_entry_free(entry);
1101
1102 /* folio is up to date */
1103 folio_mark_uptodate(folio);
1104
1105 /* move it to the tail of the inactive list after end_writeback */
1106 folio_set_reclaim(folio);
1107
1108 /* start writeback */
1109 __swap_writepage(folio, &wbc);
1110 folio_put(folio);
1111
1112 return 0;
1113}
1114
1115/*********************************
1116* shrinker functions
1117**********************************/
1118/*
1119 * The dynamic shrinker is modulated by the following factors:
1120 *
1121 * 1. Each zswap entry has a referenced bit, which the shrinker unsets (giving
1122 * the entry a second chance) before rotating it in the LRU list. If the
1123 * entry is considered again by the shrinker, with its referenced bit unset,
1124 * it is written back. The writeback rate as a result is dynamically
1125 * adjusted by the pool activities - if the pool is dominated by new entries
1126 * (i.e lots of recent zswapouts), these entries will be protected and
1127 * the writeback rate will slow down. On the other hand, if the pool has a
1128 * lot of stagnant entries, these entries will be reclaimed immediately,
1129 * effectively increasing the writeback rate.
1130 *
1131 * 2. Swapins counter: If we observe swapins, it is a sign that we are
1132 * overshrinking and should slow down. We maintain a swapins counter, which
1133 * is consumed and subtract from the number of eligible objects on the LRU
1134 * in zswap_shrinker_count().
1135 *
1136 * 3. Compression ratio. The better the workload compresses, the less gains we
1137 * can expect from writeback. We scale down the number of objects available
1138 * for reclaim by this ratio.
1139 */
1140static enum lru_status shrink_memcg_cb(struct list_head *item, struct list_lru_one *l,
1141 void *arg)
1142{
1143 struct zswap_entry *entry = container_of(item, struct zswap_entry, lru);
1144 bool *encountered_page_in_swapcache = (bool *)arg;
1145 swp_entry_t swpentry;
1146 enum lru_status ret = LRU_REMOVED_RETRY;
1147 int writeback_result;
1148
1149 /*
1150 * Second chance algorithm: if the entry has its referenced bit set, give it
1151 * a second chance. Only clear the referenced bit and rotate it in the
1152 * zswap's LRU list.
1153 */
1154 if (entry->referenced) {
1155 entry->referenced = false;
1156 return LRU_ROTATE;
1157 }
1158
1159 /*
1160 * As soon as we drop the LRU lock, the entry can be freed by
1161 * a concurrent invalidation. This means the following:
1162 *
1163 * 1. We extract the swp_entry_t to the stack, allowing
1164 * zswap_writeback_entry() to pin the swap entry and
1165 * then validate the zwap entry against that swap entry's
1166 * tree using pointer value comparison. Only when that
1167 * is successful can the entry be dereferenced.
1168 *
1169 * 2. Usually, objects are taken off the LRU for reclaim. In
1170 * this case this isn't possible, because if reclaim fails
1171 * for whatever reason, we have no means of knowing if the
1172 * entry is alive to put it back on the LRU.
1173 *
1174 * So rotate it before dropping the lock. If the entry is
1175 * written back or invalidated, the free path will unlink
1176 * it. For failures, rotation is the right thing as well.
1177 *
1178 * Temporary failures, where the same entry should be tried
1179 * again immediately, almost never happen for this shrinker.
1180 * We don't do any trylocking; -ENOMEM comes closest,
1181 * but that's extremely rare and doesn't happen spuriously
1182 * either. Don't bother distinguishing this case.
1183 */
1184 list_move_tail(item, &l->list);
1185
1186 /*
1187 * Once the lru lock is dropped, the entry might get freed. The
1188 * swpentry is copied to the stack, and entry isn't deref'd again
1189 * until the entry is verified to still be alive in the tree.
1190 */
1191 swpentry = entry->swpentry;
1192
1193 /*
1194 * It's safe to drop the lock here because we return either
1195 * LRU_REMOVED_RETRY or LRU_RETRY.
1196 */
1197 spin_unlock(&l->lock);
1198
1199 writeback_result = zswap_writeback_entry(entry, swpentry);
1200
1201 if (writeback_result) {
1202 zswap_reject_reclaim_fail++;
1203 ret = LRU_RETRY;
1204
1205 /*
1206 * Encountering a page already in swap cache is a sign that we are shrinking
1207 * into the warmer region. We should terminate shrinking (if we're in the dynamic
1208 * shrinker context).
1209 */
1210 if (writeback_result == -EEXIST && encountered_page_in_swapcache) {
1211 ret = LRU_STOP;
1212 *encountered_page_in_swapcache = true;
1213 }
1214 } else {
1215 zswap_written_back_pages++;
1216 }
1217
1218 return ret;
1219}
1220
1221static unsigned long zswap_shrinker_scan(struct shrinker *shrinker,
1222 struct shrink_control *sc)
1223{
1224 unsigned long shrink_ret;
1225 bool encountered_page_in_swapcache = false;
1226
1227 if (!zswap_shrinker_enabled ||
1228 !mem_cgroup_zswap_writeback_enabled(sc->memcg)) {
1229 sc->nr_scanned = 0;
1230 return SHRINK_STOP;
1231 }
1232
1233 shrink_ret = list_lru_shrink_walk(&zswap_list_lru, sc, &shrink_memcg_cb,
1234 &encountered_page_in_swapcache);
1235
1236 if (encountered_page_in_swapcache)
1237 return SHRINK_STOP;
1238
1239 return shrink_ret ? shrink_ret : SHRINK_STOP;
1240}
1241
1242static unsigned long zswap_shrinker_count(struct shrinker *shrinker,
1243 struct shrink_control *sc)
1244{
1245 struct mem_cgroup *memcg = sc->memcg;
1246 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid));
1247 atomic_long_t *nr_disk_swapins =
1248 &lruvec->zswap_lruvec_state.nr_disk_swapins;
1249 unsigned long nr_backing, nr_stored, nr_freeable, nr_disk_swapins_cur,
1250 nr_remain;
1251
1252 if (!zswap_shrinker_enabled || !mem_cgroup_zswap_writeback_enabled(memcg))
1253 return 0;
1254
1255 /*
1256 * The shrinker resumes swap writeback, which will enter block
1257 * and may enter fs. XXX: Harmonize with vmscan.c __GFP_FS
1258 * rules (may_enter_fs()), which apply on a per-folio basis.
1259 */
1260 if (!gfp_has_io_fs(sc->gfp_mask))
1261 return 0;
1262
1263 /*
1264 * For memcg, use the cgroup-wide ZSWAP stats since we don't
1265 * have them per-node and thus per-lruvec. Careful if memcg is
1266 * runtime-disabled: we can get sc->memcg == NULL, which is ok
1267 * for the lruvec, but not for memcg_page_state().
1268 *
1269 * Without memcg, use the zswap pool-wide metrics.
1270 */
1271 if (!mem_cgroup_disabled()) {
1272 mem_cgroup_flush_stats(memcg);
1273 nr_backing = memcg_page_state(memcg, MEMCG_ZSWAP_B) >> PAGE_SHIFT;
1274 nr_stored = memcg_page_state(memcg, MEMCG_ZSWAPPED);
1275 } else {
1276 nr_backing = zswap_total_pages();
1277 nr_stored = atomic_long_read(&zswap_stored_pages);
1278 }
1279
1280 if (!nr_stored)
1281 return 0;
1282
1283 nr_freeable = list_lru_shrink_count(&zswap_list_lru, sc);
1284 if (!nr_freeable)
1285 return 0;
1286
1287 /*
1288 * Subtract from the lru size the number of pages that are recently swapped
1289 * in from disk. The idea is that had we protect the zswap's LRU by this
1290 * amount of pages, these disk swapins would not have happened.
1291 */
1292 nr_disk_swapins_cur = atomic_long_read(nr_disk_swapins);
1293 do {
1294 if (nr_freeable >= nr_disk_swapins_cur)
1295 nr_remain = 0;
1296 else
1297 nr_remain = nr_disk_swapins_cur - nr_freeable;
1298 } while (!atomic_long_try_cmpxchg(
1299 nr_disk_swapins, &nr_disk_swapins_cur, nr_remain));
1300
1301 nr_freeable -= nr_disk_swapins_cur - nr_remain;
1302 if (!nr_freeable)
1303 return 0;
1304
1305 /*
1306 * Scale the number of freeable pages by the memory saving factor.
1307 * This ensures that the better zswap compresses memory, the fewer
1308 * pages we will evict to swap (as it will otherwise incur IO for
1309 * relatively small memory saving).
1310 */
1311 return mult_frac(nr_freeable, nr_backing, nr_stored);
1312}
1313
1314static struct shrinker *zswap_alloc_shrinker(void)
1315{
1316 struct shrinker *shrinker;
1317
1318 shrinker =
1319 shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, "mm-zswap");
1320 if (!shrinker)
1321 return NULL;
1322
1323 shrinker->scan_objects = zswap_shrinker_scan;
1324 shrinker->count_objects = zswap_shrinker_count;
1325 shrinker->batch = 0;
1326 shrinker->seeks = DEFAULT_SEEKS;
1327 return shrinker;
1328}
1329
1330static int shrink_memcg(struct mem_cgroup *memcg)
1331{
1332 int nid, shrunk = 0, scanned = 0;
1333
1334 if (!mem_cgroup_zswap_writeback_enabled(memcg))
1335 return -ENOENT;
1336
1337 /*
1338 * Skip zombies because their LRUs are reparented and we would be
1339 * reclaiming from the parent instead of the dead memcg.
1340 */
1341 if (memcg && !mem_cgroup_online(memcg))
1342 return -ENOENT;
1343
1344 for_each_node_state(nid, N_NORMAL_MEMORY) {
1345 unsigned long nr_to_walk = 1;
1346
1347 shrunk += list_lru_walk_one(&zswap_list_lru, nid, memcg,
1348 &shrink_memcg_cb, NULL, &nr_to_walk);
1349 scanned += 1 - nr_to_walk;
1350 }
1351
1352 if (!scanned)
1353 return -ENOENT;
1354
1355 return shrunk ? 0 : -EAGAIN;
1356}
1357
1358static void shrink_worker(struct work_struct *w)
1359{
1360 struct mem_cgroup *memcg;
1361 int ret, failures = 0, attempts = 0;
1362 unsigned long thr;
1363
1364 /* Reclaim down to the accept threshold */
1365 thr = zswap_accept_thr_pages();
1366
1367 /*
1368 * Global reclaim will select cgroup in a round-robin fashion from all
1369 * online memcgs, but memcgs that have no pages in zswap and
1370 * writeback-disabled memcgs (memory.zswap.writeback=0) are not
1371 * candidates for shrinking.
1372 *
1373 * Shrinking will be aborted if we encounter the following
1374 * MAX_RECLAIM_RETRIES times:
1375 * - No writeback-candidate memcgs found in a memcg tree walk.
1376 * - Shrinking a writeback-candidate memcg failed.
1377 *
1378 * We save iteration cursor memcg into zswap_next_shrink,
1379 * which can be modified by the offline memcg cleaner
1380 * zswap_memcg_offline_cleanup().
1381 *
1382 * Since the offline cleaner is called only once, we cannot leave an
1383 * offline memcg reference in zswap_next_shrink.
1384 * We can rely on the cleaner only if we get online memcg under lock.
1385 *
1386 * If we get an offline memcg, we cannot determine if the cleaner has
1387 * already been called or will be called later. We must put back the
1388 * reference before returning from this function. Otherwise, the
1389 * offline memcg left in zswap_next_shrink will hold the reference
1390 * until the next run of shrink_worker().
1391 */
1392 do {
1393 /*
1394 * Start shrinking from the next memcg after zswap_next_shrink.
1395 * When the offline cleaner has already advanced the cursor,
1396 * advancing the cursor here overlooks one memcg, but this
1397 * should be negligibly rare.
1398 *
1399 * If we get an online memcg, keep the extra reference in case
1400 * the original one obtained by mem_cgroup_iter() is dropped by
1401 * zswap_memcg_offline_cleanup() while we are shrinking the
1402 * memcg.
1403 */
1404 spin_lock(&zswap_shrink_lock);
1405 do {
1406 memcg = mem_cgroup_iter(NULL, zswap_next_shrink, NULL);
1407 zswap_next_shrink = memcg;
1408 } while (memcg && !mem_cgroup_tryget_online(memcg));
1409 spin_unlock(&zswap_shrink_lock);
1410
1411 if (!memcg) {
1412 /*
1413 * Continue shrinking without incrementing failures if
1414 * we found candidate memcgs in the last tree walk.
1415 */
1416 if (!attempts && ++failures == MAX_RECLAIM_RETRIES)
1417 break;
1418
1419 attempts = 0;
1420 goto resched;
1421 }
1422
1423 ret = shrink_memcg(memcg);
1424 /* drop the extra reference */
1425 mem_cgroup_put(memcg);
1426
1427 /*
1428 * There are no writeback-candidate pages in the memcg.
1429 * This is not an issue as long as we can find another memcg
1430 * with pages in zswap. Skip this without incrementing attempts
1431 * and failures.
1432 */
1433 if (ret == -ENOENT)
1434 continue;
1435 ++attempts;
1436
1437 if (ret && ++failures == MAX_RECLAIM_RETRIES)
1438 break;
1439resched:
1440 cond_resched();
1441 } while (zswap_total_pages() > thr);
1442}
1443
1444/*********************************
1445* main API
1446**********************************/
1447
1448static bool zswap_store_page(struct page *page,
1449 struct obj_cgroup *objcg,
1450 struct zswap_pool *pool)
1451{
1452 swp_entry_t page_swpentry = page_swap_entry(page);
1453 struct zswap_entry *entry, *old;
1454
1455 /* allocate entry */
1456 entry = zswap_entry_cache_alloc(GFP_KERNEL, page_to_nid(page));
1457 if (!entry) {
1458 zswap_reject_kmemcache_fail++;
1459 return false;
1460 }
1461
1462 if (!zswap_compress(page, entry, pool))
1463 goto compress_failed;
1464
1465 old = xa_store(swap_zswap_tree(page_swpentry),
1466 swp_offset(page_swpentry),
1467 entry, GFP_KERNEL);
1468 if (xa_is_err(old)) {
1469 int err = xa_err(old);
1470
1471 WARN_ONCE(err != -ENOMEM, "unexpected xarray error: %d\n", err);
1472 zswap_reject_alloc_fail++;
1473 goto store_failed;
1474 }
1475
1476 /*
1477 * We may have had an existing entry that became stale when
1478 * the folio was redirtied and now the new version is being
1479 * swapped out. Get rid of the old.
1480 */
1481 if (old)
1482 zswap_entry_free(old);
1483
1484 /*
1485 * The entry is successfully compressed and stored in the tree, there is
1486 * no further possibility of failure. Grab refs to the pool and objcg,
1487 * charge zswap memory, and increment zswap_stored_pages.
1488 * The opposite actions will be performed by zswap_entry_free()
1489 * when the entry is removed from the tree.
1490 */
1491 zswap_pool_get(pool);
1492 if (objcg) {
1493 obj_cgroup_get(objcg);
1494 obj_cgroup_charge_zswap(objcg, entry->length);
1495 }
1496 atomic_long_inc(&zswap_stored_pages);
1497
1498 /*
1499 * We finish initializing the entry while it's already in xarray.
1500 * This is safe because:
1501 *
1502 * 1. Concurrent stores and invalidations are excluded by folio lock.
1503 *
1504 * 2. Writeback is excluded by the entry not being on the LRU yet.
1505 * The publishing order matters to prevent writeback from seeing
1506 * an incoherent entry.
1507 */
1508 entry->pool = pool;
1509 entry->swpentry = page_swpentry;
1510 entry->objcg = objcg;
1511 entry->referenced = true;
1512 if (entry->length) {
1513 INIT_LIST_HEAD(&entry->lru);
1514 zswap_lru_add(&zswap_list_lru, entry);
1515 }
1516
1517 return true;
1518
1519store_failed:
1520 zpool_free(pool->zpool, entry->handle);
1521compress_failed:
1522 zswap_entry_cache_free(entry);
1523 return false;
1524}
1525
1526bool zswap_store(struct folio *folio)
1527{
1528 long nr_pages = folio_nr_pages(folio);
1529 swp_entry_t swp = folio->swap;
1530 struct obj_cgroup *objcg = NULL;
1531 struct mem_cgroup *memcg = NULL;
1532 struct zswap_pool *pool;
1533 bool ret = false;
1534 long index;
1535
1536 VM_WARN_ON_ONCE(!folio_test_locked(folio));
1537 VM_WARN_ON_ONCE(!folio_test_swapcache(folio));
1538
1539 if (!zswap_enabled)
1540 goto check_old;
1541
1542 objcg = get_obj_cgroup_from_folio(folio);
1543 if (objcg && !obj_cgroup_may_zswap(objcg)) {
1544 memcg = get_mem_cgroup_from_objcg(objcg);
1545 if (shrink_memcg(memcg)) {
1546 mem_cgroup_put(memcg);
1547 goto put_objcg;
1548 }
1549 mem_cgroup_put(memcg);
1550 }
1551
1552 if (zswap_check_limits())
1553 goto put_objcg;
1554
1555 pool = zswap_pool_current_get();
1556 if (!pool)
1557 goto put_objcg;
1558
1559 if (objcg) {
1560 memcg = get_mem_cgroup_from_objcg(objcg);
1561 if (memcg_list_lru_alloc(memcg, &zswap_list_lru, GFP_KERNEL)) {
1562 mem_cgroup_put(memcg);
1563 goto put_pool;
1564 }
1565 mem_cgroup_put(memcg);
1566 }
1567
1568 for (index = 0; index < nr_pages; ++index) {
1569 struct page *page = folio_page(folio, index);
1570
1571 if (!zswap_store_page(page, objcg, pool))
1572 goto put_pool;
1573 }
1574
1575 if (objcg)
1576 count_objcg_events(objcg, ZSWPOUT, nr_pages);
1577
1578 count_vm_events(ZSWPOUT, nr_pages);
1579
1580 ret = true;
1581
1582put_pool:
1583 zswap_pool_put(pool);
1584put_objcg:
1585 obj_cgroup_put(objcg);
1586 if (!ret && zswap_pool_reached_full)
1587 queue_work(shrink_wq, &zswap_shrink_work);
1588check_old:
1589 /*
1590 * If the zswap store fails or zswap is disabled, we must invalidate
1591 * the possibly stale entries which were previously stored at the
1592 * offsets corresponding to each page of the folio. Otherwise,
1593 * writeback could overwrite the new data in the swapfile.
1594 */
1595 if (!ret) {
1596 unsigned type = swp_type(swp);
1597 pgoff_t offset = swp_offset(swp);
1598 struct zswap_entry *entry;
1599 struct xarray *tree;
1600
1601 for (index = 0; index < nr_pages; ++index) {
1602 tree = swap_zswap_tree(swp_entry(type, offset + index));
1603 entry = xa_erase(tree, offset + index);
1604 if (entry)
1605 zswap_entry_free(entry);
1606 }
1607 }
1608
1609 return ret;
1610}
1611
1612bool zswap_load(struct folio *folio)
1613{
1614 swp_entry_t swp = folio->swap;
1615 pgoff_t offset = swp_offset(swp);
1616 bool swapcache = folio_test_swapcache(folio);
1617 struct xarray *tree = swap_zswap_tree(swp);
1618 struct zswap_entry *entry;
1619
1620 VM_WARN_ON_ONCE(!folio_test_locked(folio));
1621
1622 if (zswap_never_enabled())
1623 return false;
1624
1625 /*
1626 * Large folios should not be swapped in while zswap is being used, as
1627 * they are not properly handled. Zswap does not properly load large
1628 * folios, and a large folio may only be partially in zswap.
1629 *
1630 * Return true without marking the folio uptodate so that an IO error is
1631 * emitted (e.g. do_swap_page() will sigbus).
1632 */
1633 if (WARN_ON_ONCE(folio_test_large(folio)))
1634 return true;
1635
1636 /*
1637 * When reading into the swapcache, invalidate our entry. The
1638 * swapcache can be the authoritative owner of the page and
1639 * its mappings, and the pressure that results from having two
1640 * in-memory copies outweighs any benefits of caching the
1641 * compression work.
1642 *
1643 * (Most swapins go through the swapcache. The notable
1644 * exception is the singleton fault on SWP_SYNCHRONOUS_IO
1645 * files, which reads into a private page and may free it if
1646 * the fault fails. We remain the primary owner of the entry.)
1647 */
1648 if (swapcache)
1649 entry = xa_erase(tree, offset);
1650 else
1651 entry = xa_load(tree, offset);
1652
1653 if (!entry)
1654 return false;
1655
1656 zswap_decompress(entry, folio);
1657
1658 count_vm_event(ZSWPIN);
1659 if (entry->objcg)
1660 count_objcg_events(entry->objcg, ZSWPIN, 1);
1661
1662 if (swapcache) {
1663 zswap_entry_free(entry);
1664 folio_mark_dirty(folio);
1665 }
1666
1667 folio_mark_uptodate(folio);
1668 return true;
1669}
1670
1671void zswap_invalidate(swp_entry_t swp)
1672{
1673 pgoff_t offset = swp_offset(swp);
1674 struct xarray *tree = swap_zswap_tree(swp);
1675 struct zswap_entry *entry;
1676
1677 if (xa_empty(tree))
1678 return;
1679
1680 entry = xa_erase(tree, offset);
1681 if (entry)
1682 zswap_entry_free(entry);
1683}
1684
1685int zswap_swapon(int type, unsigned long nr_pages)
1686{
1687 struct xarray *trees, *tree;
1688 unsigned int nr, i;
1689
1690 nr = DIV_ROUND_UP(nr_pages, SWAP_ADDRESS_SPACE_PAGES);
1691 trees = kvcalloc(nr, sizeof(*tree), GFP_KERNEL);
1692 if (!trees) {
1693 pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1694 return -ENOMEM;
1695 }
1696
1697 for (i = 0; i < nr; i++)
1698 xa_init(trees + i);
1699
1700 nr_zswap_trees[type] = nr;
1701 zswap_trees[type] = trees;
1702 return 0;
1703}
1704
1705void zswap_swapoff(int type)
1706{
1707 struct xarray *trees = zswap_trees[type];
1708 unsigned int i;
1709
1710 if (!trees)
1711 return;
1712
1713 /* try_to_unuse() invalidated all the entries already */
1714 for (i = 0; i < nr_zswap_trees[type]; i++)
1715 WARN_ON_ONCE(!xa_empty(trees + i));
1716
1717 kvfree(trees);
1718 nr_zswap_trees[type] = 0;
1719 zswap_trees[type] = NULL;
1720}
1721
1722/*********************************
1723* debugfs functions
1724**********************************/
1725#ifdef CONFIG_DEBUG_FS
1726#include <linux/debugfs.h>
1727
1728static struct dentry *zswap_debugfs_root;
1729
1730static int debugfs_get_total_size(void *data, u64 *val)
1731{
1732 *val = zswap_total_pages() * PAGE_SIZE;
1733 return 0;
1734}
1735DEFINE_DEBUGFS_ATTRIBUTE(total_size_fops, debugfs_get_total_size, NULL, "%llu\n");
1736
1737static int debugfs_get_stored_pages(void *data, u64 *val)
1738{
1739 *val = atomic_long_read(&zswap_stored_pages);
1740 return 0;
1741}
1742DEFINE_DEBUGFS_ATTRIBUTE(stored_pages_fops, debugfs_get_stored_pages, NULL, "%llu\n");
1743
1744static int zswap_debugfs_init(void)
1745{
1746 if (!debugfs_initialized())
1747 return -ENODEV;
1748
1749 zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1750
1751 debugfs_create_u64("pool_limit_hit", 0444,
1752 zswap_debugfs_root, &zswap_pool_limit_hit);
1753 debugfs_create_u64("reject_reclaim_fail", 0444,
1754 zswap_debugfs_root, &zswap_reject_reclaim_fail);
1755 debugfs_create_u64("reject_alloc_fail", 0444,
1756 zswap_debugfs_root, &zswap_reject_alloc_fail);
1757 debugfs_create_u64("reject_kmemcache_fail", 0444,
1758 zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1759 debugfs_create_u64("reject_compress_fail", 0444,
1760 zswap_debugfs_root, &zswap_reject_compress_fail);
1761 debugfs_create_u64("reject_compress_poor", 0444,
1762 zswap_debugfs_root, &zswap_reject_compress_poor);
1763 debugfs_create_u64("written_back_pages", 0444,
1764 zswap_debugfs_root, &zswap_written_back_pages);
1765 debugfs_create_file("pool_total_size", 0444,
1766 zswap_debugfs_root, NULL, &total_size_fops);
1767 debugfs_create_file("stored_pages", 0444,
1768 zswap_debugfs_root, NULL, &stored_pages_fops);
1769
1770 return 0;
1771}
1772#else
1773static int zswap_debugfs_init(void)
1774{
1775 return 0;
1776}
1777#endif
1778
1779/*********************************
1780* module init and exit
1781**********************************/
1782static int zswap_setup(void)
1783{
1784 struct zswap_pool *pool;
1785 int ret;
1786
1787 zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
1788 if (!zswap_entry_cache) {
1789 pr_err("entry cache creation failed\n");
1790 goto cache_fail;
1791 }
1792
1793 ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE,
1794 "mm/zswap_pool:prepare",
1795 zswap_cpu_comp_prepare,
1796 zswap_cpu_comp_dead);
1797 if (ret)
1798 goto hp_fail;
1799
1800 shrink_wq = alloc_workqueue("zswap-shrink",
1801 WQ_UNBOUND|WQ_MEM_RECLAIM, 1);
1802 if (!shrink_wq)
1803 goto shrink_wq_fail;
1804
1805 zswap_shrinker = zswap_alloc_shrinker();
1806 if (!zswap_shrinker)
1807 goto shrinker_fail;
1808 if (list_lru_init_memcg(&zswap_list_lru, zswap_shrinker))
1809 goto lru_fail;
1810 shrinker_register(zswap_shrinker);
1811
1812 INIT_WORK(&zswap_shrink_work, shrink_worker);
1813
1814 pool = __zswap_pool_create_fallback();
1815 if (pool) {
1816 pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1817 zpool_get_type(pool->zpool));
1818 list_add(&pool->list, &zswap_pools);
1819 zswap_has_pool = true;
1820 static_branch_enable(&zswap_ever_enabled);
1821 } else {
1822 pr_err("pool creation failed\n");
1823 zswap_enabled = false;
1824 }
1825
1826 if (zswap_debugfs_init())
1827 pr_warn("debugfs initialization failed\n");
1828 zswap_init_state = ZSWAP_INIT_SUCCEED;
1829 return 0;
1830
1831lru_fail:
1832 shrinker_free(zswap_shrinker);
1833shrinker_fail:
1834 destroy_workqueue(shrink_wq);
1835shrink_wq_fail:
1836 cpuhp_remove_multi_state(CPUHP_MM_ZSWP_POOL_PREPARE);
1837hp_fail:
1838 kmem_cache_destroy(zswap_entry_cache);
1839cache_fail:
1840 /* if built-in, we aren't unloaded on failure; don't allow use */
1841 zswap_init_state = ZSWAP_INIT_FAILED;
1842 zswap_enabled = false;
1843 return -ENOMEM;
1844}
1845
1846static int __init zswap_init(void)
1847{
1848 if (!zswap_enabled)
1849 return 0;
1850 return zswap_setup();
1851}
1852/* must be late so crypto has time to come up */
1853late_initcall(zswap_init);
1854
1855MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1856MODULE_DESCRIPTION("Compressed cache for swap pages");