1/*
   2 * zswap.c - zswap driver file
   3 *
   4 * zswap is a backend for frontswap that takes pages that are in the process
   5 * of being swapped out and attempts to compress and store them in a
   6 * RAM-based memory pool.  This can result in a significant I/O reduction on
   7 * the swap device and, in the case where decompressing from RAM is faster
   8 * than reading from the swap device, can also improve workload performance.
   9 *
  10 * Copyright (C) 2012  Seth Jennings <sjenning@linux.vnet.ibm.com>
  11 *
  12 * This program is free software; you can redistribute it and/or
  13 * modify it under the terms of the GNU General Public License
  14 * as published by the Free Software Foundation; either version 2
  15 * of the License, or (at your option) any later version.
  16 *
  17 * This program is distributed in the hope that it will be useful,
  18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 * GNU General Public License for more details.
  21*/
  22
  23#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  24
  25#include <linux/module.h>
  26#include <linux/cpu.h>
  27#include <linux/highmem.h>
  28#include <linux/slab.h>
  29#include <linux/spinlock.h>
  30#include <linux/types.h>
  31#include <linux/atomic.h>
  32#include <linux/frontswap.h>
  33#include <linux/rbtree.h>
  34#include <linux/swap.h>
  35#include <linux/crypto.h>
  36#include <linux/mempool.h>
  37#include <linux/zpool.h>
  38
  39#include <linux/mm_types.h>
  40#include <linux/page-flags.h>
  41#include <linux/swapops.h>
  42#include <linux/writeback.h>
  43#include <linux/pagemap.h>
  44
  45/*********************************
  46* statistics
  47**********************************/
  48/* Total bytes used by the compressed storage */
  49static u64 zswap_pool_total_size;
  50/* The number of compressed pages currently stored in zswap */
  51static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
  52
  53/*
  54 * The statistics below are not protected from concurrent access for
  55 * performance reasons so they may not be a 100% accurate.  However,
  56 * they do provide useful information on roughly how many times a
  57 * certain event is occurring.
  58*/
  59
  60/* Pool limit was hit (see zswap_max_pool_percent) */
  61static u64 zswap_pool_limit_hit;
  62/* Pages written back when pool limit was reached */
  63static u64 zswap_written_back_pages;
  64/* Store failed due to a reclaim failure after pool limit was reached */
  65static u64 zswap_reject_reclaim_fail;
  66/* Compressed page was too big for the allocator to (optimally) store */
  67static u64 zswap_reject_compress_poor;
  68/* Store failed because underlying allocator could not get memory */
  69static u64 zswap_reject_alloc_fail;
  70/* Store failed because the entry metadata could not be allocated (rare) */
  71static u64 zswap_reject_kmemcache_fail;
  72/* Duplicate store was encountered (rare) */
  73static u64 zswap_duplicate_entry;
  74
  75/*********************************
  76* tunables
  77**********************************/
 
 
 
  78
  79/* Enable/disable zswap (disabled by default) */
  80static bool zswap_enabled;
  81module_param_named(enabled, zswap_enabled, bool, 0644);
  82
  83/* Crypto compressor to use */
  84#define ZSWAP_COMPRESSOR_DEFAULT "lzo"
  85static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
  86static int zswap_compressor_param_set(const char *,
  87				      const struct kernel_param *);
  88static struct kernel_param_ops zswap_compressor_param_ops = {
  89	.set =		zswap_compressor_param_set,
  90	.get =		param_get_charp,
  91	.free =		param_free_charp,
  92};
  93module_param_cb(compressor, &zswap_compressor_param_ops,
  94		&zswap_compressor, 0644);
  95
  96/* Compressed storage zpool to use */
  97#define ZSWAP_ZPOOL_DEFAULT "zbud"
  98static char *zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
  99static int zswap_zpool_param_set(const char *, const struct kernel_param *);
 100static struct kernel_param_ops zswap_zpool_param_ops = {
 101	.set =		zswap_zpool_param_set,
 102	.get =		param_get_charp,
 103	.free =		param_free_charp,
 104};
 105module_param_cb(zpool, &zswap_zpool_param_ops, &zswap_zpool_type, 0644);
 106
 107/* The maximum percentage of memory that the compressed pool can occupy */
 108static unsigned int zswap_max_pool_percent = 20;
 109module_param_named(max_pool_percent, zswap_max_pool_percent, uint, 0644);
 
 
 
 
 110
 111/*********************************
 112* data structures
 113**********************************/
 
 
 114
 115struct zswap_pool {
 116	struct zpool *zpool;
 117	struct crypto_comp * __percpu *tfm;
 118	struct kref kref;
 119	struct list_head list;
 120	struct rcu_head rcu_head;
 121	struct notifier_block notifier;
 122	char tfm_name[CRYPTO_MAX_ALG_NAME];
 123};
 124
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 125/*
 126 * struct zswap_entry
 127 *
 128 * This structure contains the metadata for tracking a single compressed
 129 * page within zswap.
 130 *
 131 * rbnode - links the entry into red-black tree for the appropriate swap type
 132 * offset - the swap offset for the entry.  Index into the red-black tree.
 133 * refcount - the number of outstanding reference to the entry. This is needed
 134 *            to protect against premature freeing of the entry by code
 135 *            concurrent calls to load, invalidate, and writeback.  The lock
 136 *            for the zswap_tree structure that contains the entry must
 137 *            be held while changing the refcount.  Since the lock must
 138 *            be held, there is no reason to also make refcount atomic.
 
 
 139 * length - the length in bytes of the compressed page data.  Needed during
 140 *          decompression
 141 * pool - the zswap_pool the entry's data is in
 142 * handle - zpool allocation handle that stores the compressed page data
 143 */
 144struct zswap_entry {
 145	struct rb_node rbnode;
 146	pgoff_t offset;
 147	int refcount;
 148	unsigned int length;
 149	struct zswap_pool *pool;
 150	unsigned long handle;
 151};
 152
 153struct zswap_header {
 154	swp_entry_t swpentry;
 155};
 156
 157/*
 158 * The tree lock in the zswap_tree struct protects a few things:
 159 * - the rbtree
 160 * - the refcount field of each entry in the tree
 161 */
 162struct zswap_tree {
 163	struct rb_root rbroot;
 164	spinlock_t lock;
 165};
 166
 167static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
 168
 169/* RCU-protected iteration */
 170static LIST_HEAD(zswap_pools);
 171/* protects zswap_pools list modification */
 172static DEFINE_SPINLOCK(zswap_pools_lock);
 173/* pool counter to provide unique names to zpool */
 174static atomic_t zswap_pools_count = ATOMIC_INIT(0);
 175
 176/* used by param callback function */
 177static bool zswap_init_started;
 178
 179/*********************************
 180* helpers and fwd declarations
 181**********************************/
 182
 183#define zswap_pool_debug(msg, p)				\
 184	pr_debug("%s pool %s/%s\n", msg, (p)->tfm_name,		\
 185		 zpool_get_type((p)->zpool))
 186
 187static int zswap_writeback_entry(struct zpool *pool, unsigned long handle);
 188static int zswap_pool_get(struct zswap_pool *pool);
 189static void zswap_pool_put(struct zswap_pool *pool);
 190
 191static const struct zpool_ops zswap_zpool_ops = {
 192	.evict = zswap_writeback_entry
 193};
 194
 195static bool zswap_is_full(void)
 196{
 197	return totalram_pages * zswap_max_pool_percent / 100 <
 198		DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
 199}
 200
 201static void zswap_update_total_size(void)
 202{
 203	struct zswap_pool *pool;
 204	u64 total = 0;
 205
 206	rcu_read_lock();
 207
 208	list_for_each_entry_rcu(pool, &zswap_pools, list)
 209		total += zpool_get_total_size(pool->zpool);
 210
 211	rcu_read_unlock();
 212
 213	zswap_pool_total_size = total;
 214}
 215
 216/*********************************
 217* zswap entry functions
 218**********************************/
 219static struct kmem_cache *zswap_entry_cache;
 220
 221static int __init zswap_entry_cache_create(void)
 222{
 223	zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
 224	return zswap_entry_cache == NULL;
 225}
 226
 227static void __init zswap_entry_cache_destroy(void)
 228{
 229	kmem_cache_destroy(zswap_entry_cache);
 230}
 231
 232static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
 233{
 234	struct zswap_entry *entry;
 235	entry = kmem_cache_alloc(zswap_entry_cache, gfp);
 236	if (!entry)
 237		return NULL;
 238	entry->refcount = 1;
 239	RB_CLEAR_NODE(&entry->rbnode);
 240	return entry;
 241}
 242
 243static void zswap_entry_cache_free(struct zswap_entry *entry)
 244{
 245	kmem_cache_free(zswap_entry_cache, entry);
 246}
 247
 248/*********************************
 249* rbtree functions
 250**********************************/
 251static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
 252{
 253	struct rb_node *node = root->rb_node;
 254	struct zswap_entry *entry;
 255
 256	while (node) {
 257		entry = rb_entry(node, struct zswap_entry, rbnode);
 258		if (entry->offset > offset)
 259			node = node->rb_left;
 260		else if (entry->offset < offset)
 261			node = node->rb_right;
 262		else
 263			return entry;
 264	}
 265	return NULL;
 266}
 267
 268/*
 269 * In the case that a entry with the same offset is found, a pointer to
 270 * the existing entry is stored in dupentry and the function returns -EEXIST
 271 */
 272static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
 273			struct zswap_entry **dupentry)
 274{
 275	struct rb_node **link = &root->rb_node, *parent = NULL;
 276	struct zswap_entry *myentry;
 277
 278	while (*link) {
 279		parent = *link;
 280		myentry = rb_entry(parent, struct zswap_entry, rbnode);
 281		if (myentry->offset > entry->offset)
 282			link = &(*link)->rb_left;
 283		else if (myentry->offset < entry->offset)
 284			link = &(*link)->rb_right;
 285		else {
 286			*dupentry = myentry;
 287			return -EEXIST;
 288		}
 289	}
 290	rb_link_node(&entry->rbnode, parent, link);
 291	rb_insert_color(&entry->rbnode, root);
 292	return 0;
 293}
 294
 295static void zswap_rb_erase(struct rb_root *root, struct zswap_entry *entry)
 296{
 297	if (!RB_EMPTY_NODE(&entry->rbnode)) {
 298		rb_erase(&entry->rbnode, root);
 299		RB_CLEAR_NODE(&entry->rbnode);
 300	}
 301}
 302
 303/*
 304 * Carries out the common pattern of freeing and entry's zpool allocation,
 305 * freeing the entry itself, and decrementing the number of stored pages.
 306 */
 307static void zswap_free_entry(struct zswap_entry *entry)
 308{
 309	zpool_free(entry->pool->zpool, entry->handle);
 310	zswap_pool_put(entry->pool);
 311	zswap_entry_cache_free(entry);
 312	atomic_dec(&zswap_stored_pages);
 313	zswap_update_total_size();
 314}
 315
 316/* caller must hold the tree lock */
 317static void zswap_entry_get(struct zswap_entry *entry)
 318{
 319	entry->refcount++;
 320}
 321
 322/* caller must hold the tree lock
 323* remove from the tree and free it, if nobody reference the entry
 324*/
 325static void zswap_entry_put(struct zswap_tree *tree,
 326			struct zswap_entry *entry)
 327{
 328	int refcount = --entry->refcount;
 329
 330	BUG_ON(refcount < 0);
 331	if (refcount == 0) {
 332		zswap_rb_erase(&tree->rbroot, entry);
 333		zswap_free_entry(entry);
 334	}
 335}
 336
 337/* caller must hold the tree lock */
 338static struct zswap_entry *zswap_entry_find_get(struct rb_root *root,
 339				pgoff_t offset)
 340{
 341	struct zswap_entry *entry;
 342
 343	entry = zswap_rb_search(root, offset);
 344	if (entry)
 345		zswap_entry_get(entry);
 346
 347	return entry;
 348}
 349
 350/*********************************
 351* per-cpu code
 352**********************************/
 353static DEFINE_PER_CPU(u8 *, zswap_dstmem);
 354
 355static int __zswap_cpu_dstmem_notifier(unsigned long action, unsigned long cpu)
 356{
 
 357	u8 *dst;
 358
 359	switch (action) {
 360	case CPU_UP_PREPARE:
 361		dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu));
 
 
 
 
 
 
 362		if (!dst) {
 363			pr_err("can't allocate compressor buffer\n");
 
 
 364			return NOTIFY_BAD;
 365		}
 366		per_cpu(zswap_dstmem, cpu) = dst;
 367		break;
 368	case CPU_DEAD:
 369	case CPU_UP_CANCELED:
 
 
 
 
 
 370		dst = per_cpu(zswap_dstmem, cpu);
 371		kfree(dst);
 372		per_cpu(zswap_dstmem, cpu) = NULL;
 373		break;
 374	default:
 375		break;
 376	}
 377	return NOTIFY_OK;
 378}
 379
 380static int zswap_cpu_dstmem_notifier(struct notifier_block *nb,
 381				     unsigned long action, void *pcpu)
 382{
 383	return __zswap_cpu_dstmem_notifier(action, (unsigned long)pcpu);
 
 384}
 385
 386static struct notifier_block zswap_dstmem_notifier = {
 387	.notifier_call =	zswap_cpu_dstmem_notifier,
 388};
 389
 390static int __init zswap_cpu_dstmem_init(void)
 391{
 392	unsigned long cpu;
 393
 394	cpu_notifier_register_begin();
 395	for_each_online_cpu(cpu)
 396		if (__zswap_cpu_dstmem_notifier(CPU_UP_PREPARE, cpu) ==
 397		    NOTIFY_BAD)
 398			goto cleanup;
 399	__register_cpu_notifier(&zswap_dstmem_notifier);
 400	cpu_notifier_register_done();
 401	return 0;
 402
 403cleanup:
 404	for_each_online_cpu(cpu)
 405		__zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
 406	cpu_notifier_register_done();
 407	return -ENOMEM;
 408}
 409
 410static void zswap_cpu_dstmem_destroy(void)
 411{
 412	unsigned long cpu;
 413
 414	cpu_notifier_register_begin();
 415	for_each_online_cpu(cpu)
 416		__zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu);
 417	__unregister_cpu_notifier(&zswap_dstmem_notifier);
 418	cpu_notifier_register_done();
 419}
 420
 421static int __zswap_cpu_comp_notifier(struct zswap_pool *pool,
 422				     unsigned long action, unsigned long cpu)
 423{
 424	struct crypto_comp *tfm;
 425
 426	switch (action) {
 427	case CPU_UP_PREPARE:
 428		if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu)))
 429			break;
 430		tfm = crypto_alloc_comp(pool->tfm_name, 0, 0);
 431		if (IS_ERR_OR_NULL(tfm)) {
 432			pr_err("could not alloc crypto comp %s : %ld\n",
 433			       pool->tfm_name, PTR_ERR(tfm));
 434			return NOTIFY_BAD;
 435		}
 436		*per_cpu_ptr(pool->tfm, cpu) = tfm;
 437		break;
 438	case CPU_DEAD:
 439	case CPU_UP_CANCELED:
 440		tfm = *per_cpu_ptr(pool->tfm, cpu);
 441		if (!IS_ERR_OR_NULL(tfm))
 442			crypto_free_comp(tfm);
 443		*per_cpu_ptr(pool->tfm, cpu) = NULL;
 444		break;
 445	default:
 446		break;
 447	}
 448	return NOTIFY_OK;
 449}
 450
 451static int zswap_cpu_comp_notifier(struct notifier_block *nb,
 452				   unsigned long action, void *pcpu)
 453{
 454	unsigned long cpu = (unsigned long)pcpu;
 455	struct zswap_pool *pool = container_of(nb, typeof(*pool), notifier);
 456
 457	return __zswap_cpu_comp_notifier(pool, action, cpu);
 458}
 459
 460static int zswap_cpu_comp_init(struct zswap_pool *pool)
 461{
 462	unsigned long cpu;
 463
 464	memset(&pool->notifier, 0, sizeof(pool->notifier));
 465	pool->notifier.notifier_call = zswap_cpu_comp_notifier;
 466
 467	cpu_notifier_register_begin();
 468	for_each_online_cpu(cpu)
 469		if (__zswap_cpu_comp_notifier(pool, CPU_UP_PREPARE, cpu) ==
 470		    NOTIFY_BAD)
 471			goto cleanup;
 472	__register_cpu_notifier(&pool->notifier);
 473	cpu_notifier_register_done();
 474	return 0;
 475
 476cleanup:
 477	for_each_online_cpu(cpu)
 478		__zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
 479	cpu_notifier_register_done();
 480	return -ENOMEM;
 481}
 482
 483static void zswap_cpu_comp_destroy(struct zswap_pool *pool)
 484{
 485	unsigned long cpu;
 486
 487	cpu_notifier_register_begin();
 488	for_each_online_cpu(cpu)
 489		__zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu);
 490	__unregister_cpu_notifier(&pool->notifier);
 491	cpu_notifier_register_done();
 492}
 493
 494/*********************************
 495* pool functions
 496**********************************/
 497
 498static struct zswap_pool *__zswap_pool_current(void)
 499{
 500	struct zswap_pool *pool;
 501
 502	pool = list_first_or_null_rcu(&zswap_pools, typeof(*pool), list);
 503	WARN_ON(!pool);
 504
 505	return pool;
 506}
 507
 508static struct zswap_pool *zswap_pool_current(void)
 509{
 510	assert_spin_locked(&zswap_pools_lock);
 511
 512	return __zswap_pool_current();
 513}
 514
 515static struct zswap_pool *zswap_pool_current_get(void)
 516{
 517	struct zswap_pool *pool;
 518
 519	rcu_read_lock();
 520
 521	pool = __zswap_pool_current();
 522	if (!pool || !zswap_pool_get(pool))
 523		pool = NULL;
 524
 525	rcu_read_unlock();
 526
 527	return pool;
 528}
 529
 530static struct zswap_pool *zswap_pool_last_get(void)
 531{
 532	struct zswap_pool *pool, *last = NULL;
 533
 534	rcu_read_lock();
 535
 536	list_for_each_entry_rcu(pool, &zswap_pools, list)
 537		last = pool;
 538	if (!WARN_ON(!last) && !zswap_pool_get(last))
 539		last = NULL;
 540
 541	rcu_read_unlock();
 542
 543	return last;
 544}
 545
 546/* type and compressor must be null-terminated */
 547static struct zswap_pool *zswap_pool_find_get(char *type, char *compressor)
 548{
 549	struct zswap_pool *pool;
 550
 551	assert_spin_locked(&zswap_pools_lock);
 552
 553	list_for_each_entry_rcu(pool, &zswap_pools, list) {
 554		if (strcmp(pool->tfm_name, compressor))
 555			continue;
 556		if (strcmp(zpool_get_type(pool->zpool), type))
 557			continue;
 558		/* if we can't get it, it's about to be destroyed */
 559		if (!zswap_pool_get(pool))
 560			continue;
 561		return pool;
 562	}
 563
 564	return NULL;
 565}
 566
 567static struct zswap_pool *zswap_pool_create(char *type, char *compressor)
 568{
 569	struct zswap_pool *pool;
 570	char name[38]; /* 'zswap' + 32 char (max) num + \0 */
 571	gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM;
 572
 573	pool = kzalloc(sizeof(*pool), GFP_KERNEL);
 574	if (!pool) {
 575		pr_err("pool alloc failed\n");
 576		return NULL;
 577	}
 578
 579	/* unique name for each pool specifically required by zsmalloc */
 580	snprintf(name, 38, "zswap%x", atomic_inc_return(&zswap_pools_count));
 581
 582	pool->zpool = zpool_create_pool(type, name, gfp, &zswap_zpool_ops);
 583	if (!pool->zpool) {
 584		pr_err("%s zpool not available\n", type);
 585		goto error;
 586	}
 587	pr_debug("using %s zpool\n", zpool_get_type(pool->zpool));
 588
 589	strlcpy(pool->tfm_name, compressor, sizeof(pool->tfm_name));
 590	pool->tfm = alloc_percpu(struct crypto_comp *);
 591	if (!pool->tfm) {
 592		pr_err("percpu alloc failed\n");
 593		goto error;
 594	}
 595
 596	if (zswap_cpu_comp_init(pool))
 597		goto error;
 598	pr_debug("using %s compressor\n", pool->tfm_name);
 599
 600	/* being the current pool takes 1 ref; this func expects the
 601	 * caller to always add the new pool as the current pool
 602	 */
 603	kref_init(&pool->kref);
 604	INIT_LIST_HEAD(&pool->list);
 605
 606	zswap_pool_debug("created", pool);
 607
 608	return pool;
 609
 610error:
 611	free_percpu(pool->tfm);
 612	if (pool->zpool)
 613		zpool_destroy_pool(pool->zpool);
 614	kfree(pool);
 615	return NULL;
 616}
 617
 618static __init struct zswap_pool *__zswap_pool_create_fallback(void)
 619{
 620	if (!crypto_has_comp(zswap_compressor, 0, 0)) {
 621		if (!strcmp(zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT)) {
 622			pr_err("default compressor %s not available\n",
 623			       zswap_compressor);
 624			return NULL;
 625		}
 626		pr_err("compressor %s not available, using default %s\n",
 627		       zswap_compressor, ZSWAP_COMPRESSOR_DEFAULT);
 628		param_free_charp(&zswap_compressor);
 629		zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
 630	}
 631	if (!zpool_has_pool(zswap_zpool_type)) {
 632		if (!strcmp(zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT)) {
 633			pr_err("default zpool %s not available\n",
 634			       zswap_zpool_type);
 635			return NULL;
 636		}
 637		pr_err("zpool %s not available, using default %s\n",
 638		       zswap_zpool_type, ZSWAP_ZPOOL_DEFAULT);
 639		param_free_charp(&zswap_zpool_type);
 640		zswap_zpool_type = ZSWAP_ZPOOL_DEFAULT;
 641	}
 642
 643	return zswap_pool_create(zswap_zpool_type, zswap_compressor);
 644}
 645
 646static void zswap_pool_destroy(struct zswap_pool *pool)
 647{
 648	zswap_pool_debug("destroying", pool);
 649
 650	zswap_cpu_comp_destroy(pool);
 651	free_percpu(pool->tfm);
 652	zpool_destroy_pool(pool->zpool);
 653	kfree(pool);
 654}
 655
 656static int __must_check zswap_pool_get(struct zswap_pool *pool)
 657{
 658	return kref_get_unless_zero(&pool->kref);
 659}
 660
 661static void __zswap_pool_release(struct rcu_head *head)
 662{
 663	struct zswap_pool *pool = container_of(head, typeof(*pool), rcu_head);
 664
 665	/* nobody should have been able to get a kref... */
 666	WARN_ON(kref_get_unless_zero(&pool->kref));
 667
 668	/* pool is now off zswap_pools list and has no references. */
 669	zswap_pool_destroy(pool);
 670}
 671
 672static void __zswap_pool_empty(struct kref *kref)
 673{
 674	struct zswap_pool *pool;
 675
 676	pool = container_of(kref, typeof(*pool), kref);
 677
 678	spin_lock(&zswap_pools_lock);
 679
 680	WARN_ON(pool == zswap_pool_current());
 681
 682	list_del_rcu(&pool->list);
 683	call_rcu(&pool->rcu_head, __zswap_pool_release);
 684
 685	spin_unlock(&zswap_pools_lock);
 686}
 687
 688static void zswap_pool_put(struct zswap_pool *pool)
 689{
 690	kref_put(&pool->kref, __zswap_pool_empty);
 691}
 692
 693/*********************************
 694* param callbacks
 695**********************************/
 696
 697/* val must be a null-terminated string */
 698static int __zswap_param_set(const char *val, const struct kernel_param *kp,
 699			     char *type, char *compressor)
 700{
 701	struct zswap_pool *pool, *put_pool = NULL;
 702	char *s = strstrip((char *)val);
 703	int ret;
 704
 705	/* no change required */
 706	if (!strcmp(s, *(char **)kp->arg))
 707		return 0;
 708
 709	/* if this is load-time (pre-init) param setting,
 710	 * don't create a pool; that's done during init.
 711	 */
 712	if (!zswap_init_started)
 713		return param_set_charp(s, kp);
 714
 715	if (!type) {
 716		if (!zpool_has_pool(s)) {
 717			pr_err("zpool %s not available\n", s);
 718			return -ENOENT;
 719		}
 720		type = s;
 721	} else if (!compressor) {
 722		if (!crypto_has_comp(s, 0, 0)) {
 723			pr_err("compressor %s not available\n", s);
 724			return -ENOENT;
 725		}
 726		compressor = s;
 727	} else {
 728		WARN_ON(1);
 729		return -EINVAL;
 730	}
 731
 732	spin_lock(&zswap_pools_lock);
 733
 734	pool = zswap_pool_find_get(type, compressor);
 735	if (pool) {
 736		zswap_pool_debug("using existing", pool);
 737		list_del_rcu(&pool->list);
 738	} else {
 739		spin_unlock(&zswap_pools_lock);
 740		pool = zswap_pool_create(type, compressor);
 741		spin_lock(&zswap_pools_lock);
 742	}
 743
 744	if (pool)
 745		ret = param_set_charp(s, kp);
 746	else
 747		ret = -EINVAL;
 748
 749	if (!ret) {
 750		put_pool = zswap_pool_current();
 751		list_add_rcu(&pool->list, &zswap_pools);
 752	} else if (pool) {
 753		/* add the possibly pre-existing pool to the end of the pools
 754		 * list; if it's new (and empty) then it'll be removed and
 755		 * destroyed by the put after we drop the lock
 756		 */
 757		list_add_tail_rcu(&pool->list, &zswap_pools);
 758		put_pool = pool;
 759	}
 760
 761	spin_unlock(&zswap_pools_lock);
 762
 763	/* drop the ref from either the old current pool,
 764	 * or the new pool we failed to add
 765	 */
 766	if (put_pool)
 767		zswap_pool_put(put_pool);
 768
 769	return ret;
 770}
 771
 772static int zswap_compressor_param_set(const char *val,
 773				      const struct kernel_param *kp)
 774{
 775	return __zswap_param_set(val, kp, zswap_zpool_type, NULL);
 776}
 777
 778static int zswap_zpool_param_set(const char *val,
 779				 const struct kernel_param *kp)
 780{
 781	return __zswap_param_set(val, kp, NULL, zswap_compressor);
 
 782}
 783
 784/*********************************
 785* writeback code
 786**********************************/
 787/* return enum for zswap_get_swap_cache_page */
 788enum zswap_get_swap_ret {
 789	ZSWAP_SWAPCACHE_NEW,
 790	ZSWAP_SWAPCACHE_EXIST,
 791	ZSWAP_SWAPCACHE_FAIL,
 792};
 793
 794/*
 795 * zswap_get_swap_cache_page
 796 *
 797 * This is an adaption of read_swap_cache_async()
 798 *
 799 * This function tries to find a page with the given swap entry
 800 * in the swapper_space address space (the swap cache).  If the page
 801 * is found, it is returned in retpage.  Otherwise, a page is allocated,
 802 * added to the swap cache, and returned in retpage.
 803 *
 804 * If success, the swap cache page is returned in retpage
 805 * Returns ZSWAP_SWAPCACHE_EXIST if page was already in the swap cache
 806 * Returns ZSWAP_SWAPCACHE_NEW if the new page needs to be populated,
 807 *     the new page is added to swapcache and locked
 808 * Returns ZSWAP_SWAPCACHE_FAIL on error
 809 */
 810static int zswap_get_swap_cache_page(swp_entry_t entry,
 811				struct page **retpage)
 812{
 813	bool page_was_allocated;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 814
 815	*retpage = __read_swap_cache_async(entry, GFP_KERNEL,
 816			NULL, 0, &page_was_allocated);
 817	if (page_was_allocated)
 818		return ZSWAP_SWAPCACHE_NEW;
 819	if (!*retpage)
 820		return ZSWAP_SWAPCACHE_FAIL;
 
 821	return ZSWAP_SWAPCACHE_EXIST;
 822}
 823
 824/*
 825 * Attempts to free an entry by adding a page to the swap cache,
 826 * decompressing the entry data into the page, and issuing a
 827 * bio write to write the page back to the swap device.
 828 *
 829 * This can be thought of as a "resumed writeback" of the page
 830 * to the swap device.  We are basically resuming the same swap
 831 * writeback path that was intercepted with the frontswap_store()
 832 * in the first place.  After the page has been decompressed into
 833 * the swap cache, the compressed version stored by zswap can be
 834 * freed.
 835 */
 836static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
 837{
 838	struct zswap_header *zhdr;
 839	swp_entry_t swpentry;
 840	struct zswap_tree *tree;
 841	pgoff_t offset;
 842	struct zswap_entry *entry;
 843	struct page *page;
 844	struct crypto_comp *tfm;
 845	u8 *src, *dst;
 846	unsigned int dlen;
 847	int ret;
 848	struct writeback_control wbc = {
 849		.sync_mode = WB_SYNC_NONE,
 850	};
 851
 852	/* extract swpentry from data */
 853	zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
 854	swpentry = zhdr->swpentry; /* here */
 855	zpool_unmap_handle(pool, handle);
 856	tree = zswap_trees[swp_type(swpentry)];
 857	offset = swp_offset(swpentry);
 858
 859	/* find and ref zswap entry */
 860	spin_lock(&tree->lock);
 861	entry = zswap_entry_find_get(&tree->rbroot, offset);
 862	if (!entry) {
 863		/* entry was invalidated */
 864		spin_unlock(&tree->lock);
 865		return 0;
 866	}
 867	spin_unlock(&tree->lock);
 868	BUG_ON(offset != entry->offset);
 869
 870	/* try to allocate swap cache page */
 871	switch (zswap_get_swap_cache_page(swpentry, &page)) {
 872	case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
 873		ret = -ENOMEM;
 874		goto fail;
 875
 876	case ZSWAP_SWAPCACHE_EXIST:
 877		/* page is already in the swap cache, ignore for now */
 878		put_page(page);
 879		ret = -EEXIST;
 880		goto fail;
 881
 882	case ZSWAP_SWAPCACHE_NEW: /* page is locked */
 883		/* decompress */
 884		dlen = PAGE_SIZE;
 885		src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
 886				ZPOOL_MM_RO) + sizeof(struct zswap_header);
 887		dst = kmap_atomic(page);
 888		tfm = *get_cpu_ptr(entry->pool->tfm);
 889		ret = crypto_comp_decompress(tfm, src, entry->length,
 890					     dst, &dlen);
 891		put_cpu_ptr(entry->pool->tfm);
 892		kunmap_atomic(dst);
 893		zpool_unmap_handle(entry->pool->zpool, entry->handle);
 894		BUG_ON(ret);
 895		BUG_ON(dlen != PAGE_SIZE);
 896
 897		/* page is up to date */
 898		SetPageUptodate(page);
 899	}
 900
 901	/* move it to the tail of the inactive list after end_writeback */
 902	SetPageReclaim(page);
 903
 904	/* start writeback */
 905	__swap_writepage(page, &wbc, end_swap_bio_write);
 906	put_page(page);
 907	zswap_written_back_pages++;
 908
 909	spin_lock(&tree->lock);
 910	/* drop local reference */
 911	zswap_entry_put(tree, entry);
 912
 913	/*
 914	* There are two possible situations for entry here:
 915	* (1) refcount is 1(normal case),  entry is valid and on the tree
 916	* (2) refcount is 0, entry is freed and not on the tree
 917	*     because invalidate happened during writeback
 918	*  search the tree and free the entry if find entry
 919	*/
 920	if (entry == zswap_rb_search(&tree->rbroot, offset))
 921		zswap_entry_put(tree, entry);
 922	spin_unlock(&tree->lock);
 923
 924	goto end;
 925
 926	/*
 927	* if we get here due to ZSWAP_SWAPCACHE_EXIST
 928	* a load may happening concurrently
 929	* it is safe and okay to not free the entry
 930	* if we free the entry in the following put
 931	* it it either okay to return !0
 932	*/
 933fail:
 934	spin_lock(&tree->lock);
 935	zswap_entry_put(tree, entry);
 936	spin_unlock(&tree->lock);
 937
 938end:
 939	return ret;
 940}
 941
 942static int zswap_shrink(void)
 943{
 944	struct zswap_pool *pool;
 945	int ret;
 946
 947	pool = zswap_pool_last_get();
 948	if (!pool)
 949		return -ENOENT;
 950
 951	ret = zpool_shrink(pool->zpool, 1, NULL);
 952
 953	zswap_pool_put(pool);
 954
 955	return ret;
 956}
 957
 958/*********************************
 959* frontswap hooks
 960**********************************/
 961/* attempts to compress and store an single page */
 962static int zswap_frontswap_store(unsigned type, pgoff_t offset,
 963				struct page *page)
 964{
 965	struct zswap_tree *tree = zswap_trees[type];
 966	struct zswap_entry *entry, *dupentry;
 967	struct crypto_comp *tfm;
 968	int ret;
 969	unsigned int dlen = PAGE_SIZE, len;
 970	unsigned long handle;
 971	char *buf;
 972	u8 *src, *dst;
 973	struct zswap_header *zhdr;
 974
 975	if (!zswap_enabled || !tree) {
 976		ret = -ENODEV;
 977		goto reject;
 978	}
 979
 980	/* reclaim space if needed */
 981	if (zswap_is_full()) {
 982		zswap_pool_limit_hit++;
 983		if (zswap_shrink()) {
 984			zswap_reject_reclaim_fail++;
 985			ret = -ENOMEM;
 986			goto reject;
 987		}
 988	}
 989
 990	/* allocate entry */
 991	entry = zswap_entry_cache_alloc(GFP_KERNEL);
 992	if (!entry) {
 993		zswap_reject_kmemcache_fail++;
 994		ret = -ENOMEM;
 995		goto reject;
 996	}
 997
 998	/* if entry is successfully added, it keeps the reference */
 999	entry->pool = zswap_pool_current_get();
1000	if (!entry->pool) {
1001		ret = -EINVAL;
1002		goto freepage;
1003	}
1004
1005	/* compress */
1006	dst = get_cpu_var(zswap_dstmem);
1007	tfm = *get_cpu_ptr(entry->pool->tfm);
1008	src = kmap_atomic(page);
1009	ret = crypto_comp_compress(tfm, src, PAGE_SIZE, dst, &dlen);
1010	kunmap_atomic(src);
1011	put_cpu_ptr(entry->pool->tfm);
1012	if (ret) {
1013		ret = -EINVAL;
1014		goto put_dstmem;
1015	}
1016
1017	/* store */
1018	len = dlen + sizeof(struct zswap_header);
1019	ret = zpool_malloc(entry->pool->zpool, len,
1020			   __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM,
1021			   &handle);
1022	if (ret == -ENOSPC) {
1023		zswap_reject_compress_poor++;
1024		goto put_dstmem;
1025	}
1026	if (ret) {
1027		zswap_reject_alloc_fail++;
1028		goto put_dstmem;
1029	}
1030	zhdr = zpool_map_handle(entry->pool->zpool, handle, ZPOOL_MM_RW);
1031	zhdr->swpentry = swp_entry(type, offset);
1032	buf = (u8 *)(zhdr + 1);
1033	memcpy(buf, dst, dlen);
1034	zpool_unmap_handle(entry->pool->zpool, handle);
1035	put_cpu_var(zswap_dstmem);
1036
1037	/* populate entry */
1038	entry->offset = offset;
1039	entry->handle = handle;
1040	entry->length = dlen;
1041
1042	/* map */
1043	spin_lock(&tree->lock);
1044	do {
1045		ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
1046		if (ret == -EEXIST) {
1047			zswap_duplicate_entry++;
1048			/* remove from rbtree */
1049			zswap_rb_erase(&tree->rbroot, dupentry);
1050			zswap_entry_put(tree, dupentry);
1051		}
1052	} while (ret == -EEXIST);
1053	spin_unlock(&tree->lock);
1054
1055	/* update stats */
1056	atomic_inc(&zswap_stored_pages);
1057	zswap_update_total_size();
1058
1059	return 0;
1060
1061put_dstmem:
1062	put_cpu_var(zswap_dstmem);
1063	zswap_pool_put(entry->pool);
1064freepage:
 
1065	zswap_entry_cache_free(entry);
1066reject:
1067	return ret;
1068}
1069
1070/*
1071 * returns 0 if the page was successfully decompressed
1072 * return -1 on entry not found or error
1073*/
1074static int zswap_frontswap_load(unsigned type, pgoff_t offset,
1075				struct page *page)
1076{
1077	struct zswap_tree *tree = zswap_trees[type];
1078	struct zswap_entry *entry;
1079	struct crypto_comp *tfm;
1080	u8 *src, *dst;
1081	unsigned int dlen;
1082	int ret;
1083
1084	/* find */
1085	spin_lock(&tree->lock);
1086	entry = zswap_entry_find_get(&tree->rbroot, offset);
1087	if (!entry) {
1088		/* entry was written back */
1089		spin_unlock(&tree->lock);
1090		return -1;
1091	}
1092	spin_unlock(&tree->lock);
1093
1094	/* decompress */
1095	dlen = PAGE_SIZE;
1096	src = (u8 *)zpool_map_handle(entry->pool->zpool, entry->handle,
1097			ZPOOL_MM_RO) + sizeof(struct zswap_header);
1098	dst = kmap_atomic(page);
1099	tfm = *get_cpu_ptr(entry->pool->tfm);
1100	ret = crypto_comp_decompress(tfm, src, entry->length, dst, &dlen);
1101	put_cpu_ptr(entry->pool->tfm);
1102	kunmap_atomic(dst);
1103	zpool_unmap_handle(entry->pool->zpool, entry->handle);
1104	BUG_ON(ret);
1105
1106	spin_lock(&tree->lock);
1107	zswap_entry_put(tree, entry);
1108	spin_unlock(&tree->lock);
1109
1110	return 0;
1111}
1112
1113/* frees an entry in zswap */
1114static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
1115{
1116	struct zswap_tree *tree = zswap_trees[type];
1117	struct zswap_entry *entry;
1118
1119	/* find */
1120	spin_lock(&tree->lock);
1121	entry = zswap_rb_search(&tree->rbroot, offset);
1122	if (!entry) {
1123		/* entry was written back */
1124		spin_unlock(&tree->lock);
1125		return;
1126	}
1127
1128	/* remove from rbtree */
1129	zswap_rb_erase(&tree->rbroot, entry);
1130
1131	/* drop the initial reference from entry creation */
1132	zswap_entry_put(tree, entry);
1133
1134	spin_unlock(&tree->lock);
1135}
1136
1137/* frees all zswap entries for the given swap type */
1138static void zswap_frontswap_invalidate_area(unsigned type)
1139{
1140	struct zswap_tree *tree = zswap_trees[type];
1141	struct zswap_entry *entry, *n;
1142
1143	if (!tree)
1144		return;
1145
1146	/* walk the tree and free everything */
1147	spin_lock(&tree->lock);
1148	rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode)
1149		zswap_free_entry(entry);
1150	tree->rbroot = RB_ROOT;
1151	spin_unlock(&tree->lock);
1152	kfree(tree);
1153	zswap_trees[type] = NULL;
1154}
1155
 
 
 
 
1156static void zswap_frontswap_init(unsigned type)
1157{
1158	struct zswap_tree *tree;
1159
1160	tree = kzalloc(sizeof(struct zswap_tree), GFP_KERNEL);
1161	if (!tree) {
1162		pr_err("alloc failed, zswap disabled for swap type %d\n", type);
1163		return;
1164	}
1165
1166	tree->rbroot = RB_ROOT;
1167	spin_lock_init(&tree->lock);
1168	zswap_trees[type] = tree;
1169}
1170
1171static struct frontswap_ops zswap_frontswap_ops = {
1172	.store = zswap_frontswap_store,
1173	.load = zswap_frontswap_load,
1174	.invalidate_page = zswap_frontswap_invalidate_page,
1175	.invalidate_area = zswap_frontswap_invalidate_area,
1176	.init = zswap_frontswap_init
1177};
1178
1179/*********************************
1180* debugfs functions
1181**********************************/
1182#ifdef CONFIG_DEBUG_FS
1183#include <linux/debugfs.h>
1184
1185static struct dentry *zswap_debugfs_root;
1186
1187static int __init zswap_debugfs_init(void)
1188{
1189	if (!debugfs_initialized())
1190		return -ENODEV;
1191
1192	zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
1193	if (!zswap_debugfs_root)
1194		return -ENOMEM;
1195
1196	debugfs_create_u64("pool_limit_hit", S_IRUGO,
1197			zswap_debugfs_root, &zswap_pool_limit_hit);
1198	debugfs_create_u64("reject_reclaim_fail", S_IRUGO,
1199			zswap_debugfs_root, &zswap_reject_reclaim_fail);
1200	debugfs_create_u64("reject_alloc_fail", S_IRUGO,
1201			zswap_debugfs_root, &zswap_reject_alloc_fail);
1202	debugfs_create_u64("reject_kmemcache_fail", S_IRUGO,
1203			zswap_debugfs_root, &zswap_reject_kmemcache_fail);
1204	debugfs_create_u64("reject_compress_poor", S_IRUGO,
1205			zswap_debugfs_root, &zswap_reject_compress_poor);
1206	debugfs_create_u64("written_back_pages", S_IRUGO,
1207			zswap_debugfs_root, &zswap_written_back_pages);
1208	debugfs_create_u64("duplicate_entry", S_IRUGO,
1209			zswap_debugfs_root, &zswap_duplicate_entry);
1210	debugfs_create_u64("pool_total_size", S_IRUGO,
1211			zswap_debugfs_root, &zswap_pool_total_size);
1212	debugfs_create_atomic_t("stored_pages", S_IRUGO,
1213			zswap_debugfs_root, &zswap_stored_pages);
1214
1215	return 0;
1216}
1217
1218static void __exit zswap_debugfs_exit(void)
1219{
1220	debugfs_remove_recursive(zswap_debugfs_root);
1221}
1222#else
1223static int __init zswap_debugfs_init(void)
1224{
1225	return 0;
1226}
1227
1228static void __exit zswap_debugfs_exit(void) { }
1229#endif
1230
1231/*********************************
1232* module init and exit
1233**********************************/
1234static int __init init_zswap(void)
1235{
1236	struct zswap_pool *pool;
 
1237
1238	zswap_init_started = true;
1239
1240	if (zswap_entry_cache_create()) {
1241		pr_err("entry cache creation failed\n");
1242		goto cache_fail;
 
1243	}
1244
1245	if (zswap_cpu_dstmem_init()) {
1246		pr_err("dstmem alloc failed\n");
1247		goto dstmem_fail;
1248	}
1249
1250	pool = __zswap_pool_create_fallback();
1251	if (!pool) {
1252		pr_err("pool creation failed\n");
1253		goto pool_fail;
 
 
1254	}
1255	pr_info("loaded using pool %s/%s\n", pool->tfm_name,
1256		zpool_get_type(pool->zpool));
1257
1258	list_add(&pool->list, &zswap_pools);
1259
1260	frontswap_register_ops(&zswap_frontswap_ops);
1261	if (zswap_debugfs_init())
1262		pr_warn("debugfs initialization failed\n");
1263	return 0;
1264
1265pool_fail:
1266	zswap_cpu_dstmem_destroy();
1267dstmem_fail:
1268	zswap_entry_cache_destroy();
1269cache_fail:
 
1270	return -ENOMEM;
1271}
1272/* must be late so crypto has time to come up */
1273late_initcall(init_zswap);
1274
1275MODULE_LICENSE("GPL");
1276MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
1277MODULE_DESCRIPTION("Compressed cache for swap pages");