xref: /openbmc/linux/mm/zswap.c (revision cd5d5810)
1 /*
2  * zswap.c - zswap driver file
3  *
4  * zswap is a backend for frontswap that takes pages that are in the process
5  * of being swapped out and attempts to compress and store them in a
6  * RAM-based memory pool.  This can result in a significant I/O reduction on
7  * the swap device and, in the case where decompressing from RAM is faster
8  * than reading from the swap device, can also improve workload performance.
9  *
10  * Copyright (C) 2012  Seth Jennings <sjenning@linux.vnet.ibm.com>
11  *
12  * This program is free software; you can redistribute it and/or
13  * modify it under the terms of the GNU General Public License
14  * as published by the Free Software Foundation; either version 2
15  * of the License, or (at your option) any later version.
16  *
17  * This program is distributed in the hope that it will be useful,
18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
20  * GNU General Public License for more details.
21 */
22 
23 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24 
25 #include <linux/module.h>
26 #include <linux/cpu.h>
27 #include <linux/highmem.h>
28 #include <linux/slab.h>
29 #include <linux/spinlock.h>
30 #include <linux/types.h>
31 #include <linux/atomic.h>
32 #include <linux/frontswap.h>
33 #include <linux/rbtree.h>
34 #include <linux/swap.h>
35 #include <linux/crypto.h>
36 #include <linux/mempool.h>
37 #include <linux/zbud.h>
38 
39 #include <linux/mm_types.h>
40 #include <linux/page-flags.h>
41 #include <linux/swapops.h>
42 #include <linux/writeback.h>
43 #include <linux/pagemap.h>
44 
45 /*********************************
46 * statistics
47 **********************************/
48 /* Number of memory pages used by the compressed pool */
49 static u64 zswap_pool_pages;
50 /* The number of compressed pages currently stored in zswap */
51 static atomic_t zswap_stored_pages = ATOMIC_INIT(0);
52 
53 /*
54  * The statistics below are not protected from concurrent access for
55  * performance reasons so they may not be a 100% accurate.  However,
56  * they do provide useful information on roughly how many times a
57  * certain event is occurring.
58 */
59 
60 /* Pool limit was hit (see zswap_max_pool_percent) */
61 static u64 zswap_pool_limit_hit;
62 /* Pages written back when pool limit was reached */
63 static u64 zswap_written_back_pages;
64 /* Store failed due to a reclaim failure after pool limit was reached */
65 static u64 zswap_reject_reclaim_fail;
66 /* Compressed page was too big for the allocator to (optimally) store */
67 static u64 zswap_reject_compress_poor;
68 /* Store failed because underlying allocator could not get memory */
69 static u64 zswap_reject_alloc_fail;
70 /* Store failed because the entry metadata could not be allocated (rare) */
71 static u64 zswap_reject_kmemcache_fail;
72 /* Duplicate store was encountered (rare) */
73 static u64 zswap_duplicate_entry;
74 
75 /*********************************
76 * tunables
77 **********************************/
78 /* Enable/disable zswap (disabled by default, fixed at boot for now) */
79 static bool zswap_enabled __read_mostly;
80 module_param_named(enabled, zswap_enabled, bool, 0);
81 
82 /* Compressor to be used by zswap (fixed at boot for now) */
83 #define ZSWAP_COMPRESSOR_DEFAULT "lzo"
84 static char *zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
85 module_param_named(compressor, zswap_compressor, charp, 0);
86 
87 /* The maximum percentage of memory that the compressed pool can occupy */
88 static unsigned int zswap_max_pool_percent = 20;
89 module_param_named(max_pool_percent,
90 			zswap_max_pool_percent, uint, 0644);
91 
92 /*********************************
93 * compression functions
94 **********************************/
95 /* per-cpu compression transforms */
96 static struct crypto_comp * __percpu *zswap_comp_pcpu_tfms;
97 
98 enum comp_op {
99 	ZSWAP_COMPOP_COMPRESS,
100 	ZSWAP_COMPOP_DECOMPRESS
101 };
102 
103 static int zswap_comp_op(enum comp_op op, const u8 *src, unsigned int slen,
104 				u8 *dst, unsigned int *dlen)
105 {
106 	struct crypto_comp *tfm;
107 	int ret;
108 
109 	tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, get_cpu());
110 	switch (op) {
111 	case ZSWAP_COMPOP_COMPRESS:
112 		ret = crypto_comp_compress(tfm, src, slen, dst, dlen);
113 		break;
114 	case ZSWAP_COMPOP_DECOMPRESS:
115 		ret = crypto_comp_decompress(tfm, src, slen, dst, dlen);
116 		break;
117 	default:
118 		ret = -EINVAL;
119 	}
120 
121 	put_cpu();
122 	return ret;
123 }
124 
125 static int __init zswap_comp_init(void)
126 {
127 	if (!crypto_has_comp(zswap_compressor, 0, 0)) {
128 		pr_info("%s compressor not available\n", zswap_compressor);
129 		/* fall back to default compressor */
130 		zswap_compressor = ZSWAP_COMPRESSOR_DEFAULT;
131 		if (!crypto_has_comp(zswap_compressor, 0, 0))
132 			/* can't even load the default compressor */
133 			return -ENODEV;
134 	}
135 	pr_info("using %s compressor\n", zswap_compressor);
136 
137 	/* alloc percpu transforms */
138 	zswap_comp_pcpu_tfms = alloc_percpu(struct crypto_comp *);
139 	if (!zswap_comp_pcpu_tfms)
140 		return -ENOMEM;
141 	return 0;
142 }
143 
144 static void zswap_comp_exit(void)
145 {
146 	/* free percpu transforms */
147 	if (zswap_comp_pcpu_tfms)
148 		free_percpu(zswap_comp_pcpu_tfms);
149 }
150 
151 /*********************************
152 * data structures
153 **********************************/
154 /*
155  * struct zswap_entry
156  *
157  * This structure contains the metadata for tracking a single compressed
158  * page within zswap.
159  *
160  * rbnode - links the entry into red-black tree for the appropriate swap type
161  * refcount - the number of outstanding reference to the entry. This is needed
162  *            to protect against premature freeing of the entry by code
163  *            concurent calls to load, invalidate, and writeback.  The lock
164  *            for the zswap_tree structure that contains the entry must
165  *            be held while changing the refcount.  Since the lock must
166  *            be held, there is no reason to also make refcount atomic.
167  * offset - the swap offset for the entry.  Index into the red-black tree.
168  * handle - zsmalloc allocation handle that stores the compressed page data
169  * length - the length in bytes of the compressed page data.  Needed during
170  *           decompression
171  */
172 struct zswap_entry {
173 	struct rb_node rbnode;
174 	pgoff_t offset;
175 	int refcount;
176 	unsigned int length;
177 	unsigned long handle;
178 };
179 
180 struct zswap_header {
181 	swp_entry_t swpentry;
182 };
183 
184 /*
185  * The tree lock in the zswap_tree struct protects a few things:
186  * - the rbtree
187  * - the refcount field of each entry in the tree
188  */
189 struct zswap_tree {
190 	struct rb_root rbroot;
191 	spinlock_t lock;
192 	struct zbud_pool *pool;
193 };
194 
195 static struct zswap_tree *zswap_trees[MAX_SWAPFILES];
196 
197 /*********************************
198 * zswap entry functions
199 **********************************/
200 static struct kmem_cache *zswap_entry_cache;
201 
202 static int zswap_entry_cache_create(void)
203 {
204 	zswap_entry_cache = KMEM_CACHE(zswap_entry, 0);
205 	return (zswap_entry_cache == NULL);
206 }
207 
208 static void zswap_entry_cache_destory(void)
209 {
210 	kmem_cache_destroy(zswap_entry_cache);
211 }
212 
213 static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp)
214 {
215 	struct zswap_entry *entry;
216 	entry = kmem_cache_alloc(zswap_entry_cache, gfp);
217 	if (!entry)
218 		return NULL;
219 	entry->refcount = 1;
220 	return entry;
221 }
222 
223 static void zswap_entry_cache_free(struct zswap_entry *entry)
224 {
225 	kmem_cache_free(zswap_entry_cache, entry);
226 }
227 
228 /* caller must hold the tree lock */
229 static void zswap_entry_get(struct zswap_entry *entry)
230 {
231 	entry->refcount++;
232 }
233 
234 /* caller must hold the tree lock */
235 static int zswap_entry_put(struct zswap_entry *entry)
236 {
237 	entry->refcount--;
238 	return entry->refcount;
239 }
240 
241 /*********************************
242 * rbtree functions
243 **********************************/
244 static struct zswap_entry *zswap_rb_search(struct rb_root *root, pgoff_t offset)
245 {
246 	struct rb_node *node = root->rb_node;
247 	struct zswap_entry *entry;
248 
249 	while (node) {
250 		entry = rb_entry(node, struct zswap_entry, rbnode);
251 		if (entry->offset > offset)
252 			node = node->rb_left;
253 		else if (entry->offset < offset)
254 			node = node->rb_right;
255 		else
256 			return entry;
257 	}
258 	return NULL;
259 }
260 
261 /*
262  * In the case that a entry with the same offset is found, a pointer to
263  * the existing entry is stored in dupentry and the function returns -EEXIST
264  */
265 static int zswap_rb_insert(struct rb_root *root, struct zswap_entry *entry,
266 			struct zswap_entry **dupentry)
267 {
268 	struct rb_node **link = &root->rb_node, *parent = NULL;
269 	struct zswap_entry *myentry;
270 
271 	while (*link) {
272 		parent = *link;
273 		myentry = rb_entry(parent, struct zswap_entry, rbnode);
274 		if (myentry->offset > entry->offset)
275 			link = &(*link)->rb_left;
276 		else if (myentry->offset < entry->offset)
277 			link = &(*link)->rb_right;
278 		else {
279 			*dupentry = myentry;
280 			return -EEXIST;
281 		}
282 	}
283 	rb_link_node(&entry->rbnode, parent, link);
284 	rb_insert_color(&entry->rbnode, root);
285 	return 0;
286 }
287 
288 /*********************************
289 * per-cpu code
290 **********************************/
291 static DEFINE_PER_CPU(u8 *, zswap_dstmem);
292 
293 static int __zswap_cpu_notifier(unsigned long action, unsigned long cpu)
294 {
295 	struct crypto_comp *tfm;
296 	u8 *dst;
297 
298 	switch (action) {
299 	case CPU_UP_PREPARE:
300 		tfm = crypto_alloc_comp(zswap_compressor, 0, 0);
301 		if (IS_ERR(tfm)) {
302 			pr_err("can't allocate compressor transform\n");
303 			return NOTIFY_BAD;
304 		}
305 		*per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = tfm;
306 		dst = kmalloc(PAGE_SIZE * 2, GFP_KERNEL);
307 		if (!dst) {
308 			pr_err("can't allocate compressor buffer\n");
309 			crypto_free_comp(tfm);
310 			*per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL;
311 			return NOTIFY_BAD;
312 		}
313 		per_cpu(zswap_dstmem, cpu) = dst;
314 		break;
315 	case CPU_DEAD:
316 	case CPU_UP_CANCELED:
317 		tfm = *per_cpu_ptr(zswap_comp_pcpu_tfms, cpu);
318 		if (tfm) {
319 			crypto_free_comp(tfm);
320 			*per_cpu_ptr(zswap_comp_pcpu_tfms, cpu) = NULL;
321 		}
322 		dst = per_cpu(zswap_dstmem, cpu);
323 		kfree(dst);
324 		per_cpu(zswap_dstmem, cpu) = NULL;
325 		break;
326 	default:
327 		break;
328 	}
329 	return NOTIFY_OK;
330 }
331 
332 static int zswap_cpu_notifier(struct notifier_block *nb,
333 				unsigned long action, void *pcpu)
334 {
335 	unsigned long cpu = (unsigned long)pcpu;
336 	return __zswap_cpu_notifier(action, cpu);
337 }
338 
339 static struct notifier_block zswap_cpu_notifier_block = {
340 	.notifier_call = zswap_cpu_notifier
341 };
342 
343 static int zswap_cpu_init(void)
344 {
345 	unsigned long cpu;
346 
347 	get_online_cpus();
348 	for_each_online_cpu(cpu)
349 		if (__zswap_cpu_notifier(CPU_UP_PREPARE, cpu) != NOTIFY_OK)
350 			goto cleanup;
351 	register_cpu_notifier(&zswap_cpu_notifier_block);
352 	put_online_cpus();
353 	return 0;
354 
355 cleanup:
356 	for_each_online_cpu(cpu)
357 		__zswap_cpu_notifier(CPU_UP_CANCELED, cpu);
358 	put_online_cpus();
359 	return -ENOMEM;
360 }
361 
362 /*********************************
363 * helpers
364 **********************************/
365 static bool zswap_is_full(void)
366 {
367 	return (totalram_pages * zswap_max_pool_percent / 100 <
368 		zswap_pool_pages);
369 }
370 
371 /*
372  * Carries out the common pattern of freeing and entry's zsmalloc allocation,
373  * freeing the entry itself, and decrementing the number of stored pages.
374  */
375 static void zswap_free_entry(struct zswap_tree *tree, struct zswap_entry *entry)
376 {
377 	zbud_free(tree->pool, entry->handle);
378 	zswap_entry_cache_free(entry);
379 	atomic_dec(&zswap_stored_pages);
380 	zswap_pool_pages = zbud_get_pool_size(tree->pool);
381 }
382 
383 /*********************************
384 * writeback code
385 **********************************/
386 /* return enum for zswap_get_swap_cache_page */
387 enum zswap_get_swap_ret {
388 	ZSWAP_SWAPCACHE_NEW,
389 	ZSWAP_SWAPCACHE_EXIST,
390 	ZSWAP_SWAPCACHE_NOMEM
391 };
392 
393 /*
394  * zswap_get_swap_cache_page
395  *
396  * This is an adaption of read_swap_cache_async()
397  *
398  * This function tries to find a page with the given swap entry
399  * in the swapper_space address space (the swap cache).  If the page
400  * is found, it is returned in retpage.  Otherwise, a page is allocated,
401  * added to the swap cache, and returned in retpage.
402  *
403  * If success, the swap cache page is returned in retpage
404  * Returns 0 if page was already in the swap cache, page is not locked
405  * Returns 1 if the new page needs to be populated, page is locked
406  * Returns <0 on error
407  */
408 static int zswap_get_swap_cache_page(swp_entry_t entry,
409 				struct page **retpage)
410 {
411 	struct page *found_page, *new_page = NULL;
412 	struct address_space *swapper_space = swap_address_space(entry);
413 	int err;
414 
415 	*retpage = NULL;
416 	do {
417 		/*
418 		 * First check the swap cache.  Since this is normally
419 		 * called after lookup_swap_cache() failed, re-calling
420 		 * that would confuse statistics.
421 		 */
422 		found_page = find_get_page(swapper_space, entry.val);
423 		if (found_page)
424 			break;
425 
426 		/*
427 		 * Get a new page to read into from swap.
428 		 */
429 		if (!new_page) {
430 			new_page = alloc_page(GFP_KERNEL);
431 			if (!new_page)
432 				break; /* Out of memory */
433 		}
434 
435 		/*
436 		 * call radix_tree_preload() while we can wait.
437 		 */
438 		err = radix_tree_preload(GFP_KERNEL);
439 		if (err)
440 			break;
441 
442 		/*
443 		 * Swap entry may have been freed since our caller observed it.
444 		 */
445 		err = swapcache_prepare(entry);
446 		if (err == -EEXIST) { /* seems racy */
447 			radix_tree_preload_end();
448 			continue;
449 		}
450 		if (err) { /* swp entry is obsolete ? */
451 			radix_tree_preload_end();
452 			break;
453 		}
454 
455 		/* May fail (-ENOMEM) if radix-tree node allocation failed. */
456 		__set_page_locked(new_page);
457 		SetPageSwapBacked(new_page);
458 		err = __add_to_swap_cache(new_page, entry);
459 		if (likely(!err)) {
460 			radix_tree_preload_end();
461 			lru_cache_add_anon(new_page);
462 			*retpage = new_page;
463 			return ZSWAP_SWAPCACHE_NEW;
464 		}
465 		radix_tree_preload_end();
466 		ClearPageSwapBacked(new_page);
467 		__clear_page_locked(new_page);
468 		/*
469 		 * add_to_swap_cache() doesn't return -EEXIST, so we can safely
470 		 * clear SWAP_HAS_CACHE flag.
471 		 */
472 		swapcache_free(entry, NULL);
473 	} while (err != -ENOMEM);
474 
475 	if (new_page)
476 		page_cache_release(new_page);
477 	if (!found_page)
478 		return ZSWAP_SWAPCACHE_NOMEM;
479 	*retpage = found_page;
480 	return ZSWAP_SWAPCACHE_EXIST;
481 }
482 
483 /*
484  * Attempts to free an entry by adding a page to the swap cache,
485  * decompressing the entry data into the page, and issuing a
486  * bio write to write the page back to the swap device.
487  *
488  * This can be thought of as a "resumed writeback" of the page
489  * to the swap device.  We are basically resuming the same swap
490  * writeback path that was intercepted with the frontswap_store()
491  * in the first place.  After the page has been decompressed into
492  * the swap cache, the compressed version stored by zswap can be
493  * freed.
494  */
495 static int zswap_writeback_entry(struct zbud_pool *pool, unsigned long handle)
496 {
497 	struct zswap_header *zhdr;
498 	swp_entry_t swpentry;
499 	struct zswap_tree *tree;
500 	pgoff_t offset;
501 	struct zswap_entry *entry;
502 	struct page *page;
503 	u8 *src, *dst;
504 	unsigned int dlen;
505 	int ret, refcount;
506 	struct writeback_control wbc = {
507 		.sync_mode = WB_SYNC_NONE,
508 	};
509 
510 	/* extract swpentry from data */
511 	zhdr = zbud_map(pool, handle);
512 	swpentry = zhdr->swpentry; /* here */
513 	zbud_unmap(pool, handle);
514 	tree = zswap_trees[swp_type(swpentry)];
515 	offset = swp_offset(swpentry);
516 	BUG_ON(pool != tree->pool);
517 
518 	/* find and ref zswap entry */
519 	spin_lock(&tree->lock);
520 	entry = zswap_rb_search(&tree->rbroot, offset);
521 	if (!entry) {
522 		/* entry was invalidated */
523 		spin_unlock(&tree->lock);
524 		return 0;
525 	}
526 	zswap_entry_get(entry);
527 	spin_unlock(&tree->lock);
528 	BUG_ON(offset != entry->offset);
529 
530 	/* try to allocate swap cache page */
531 	switch (zswap_get_swap_cache_page(swpentry, &page)) {
532 	case ZSWAP_SWAPCACHE_NOMEM: /* no memory */
533 		ret = -ENOMEM;
534 		goto fail;
535 
536 	case ZSWAP_SWAPCACHE_EXIST: /* page is unlocked */
537 		/* page is already in the swap cache, ignore for now */
538 		page_cache_release(page);
539 		ret = -EEXIST;
540 		goto fail;
541 
542 	case ZSWAP_SWAPCACHE_NEW: /* page is locked */
543 		/* decompress */
544 		dlen = PAGE_SIZE;
545 		src = (u8 *)zbud_map(tree->pool, entry->handle) +
546 			sizeof(struct zswap_header);
547 		dst = kmap_atomic(page);
548 		ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src,
549 				entry->length, dst, &dlen);
550 		kunmap_atomic(dst);
551 		zbud_unmap(tree->pool, entry->handle);
552 		BUG_ON(ret);
553 		BUG_ON(dlen != PAGE_SIZE);
554 
555 		/* page is up to date */
556 		SetPageUptodate(page);
557 	}
558 
559 	/* start writeback */
560 	__swap_writepage(page, &wbc, end_swap_bio_write);
561 	page_cache_release(page);
562 	zswap_written_back_pages++;
563 
564 	spin_lock(&tree->lock);
565 
566 	/* drop local reference */
567 	zswap_entry_put(entry);
568 	/* drop the initial reference from entry creation */
569 	refcount = zswap_entry_put(entry);
570 
571 	/*
572 	 * There are three possible values for refcount here:
573 	 * (1) refcount is 1, load is in progress, unlink from rbtree,
574 	 *     load will free
575 	 * (2) refcount is 0, (normal case) entry is valid,
576 	 *     remove from rbtree and free entry
577 	 * (3) refcount is -1, invalidate happened during writeback,
578 	 *     free entry
579 	 */
580 	if (refcount >= 0) {
581 		/* no invalidate yet, remove from rbtree */
582 		rb_erase(&entry->rbnode, &tree->rbroot);
583 	}
584 	spin_unlock(&tree->lock);
585 	if (refcount <= 0) {
586 		/* free the entry */
587 		zswap_free_entry(tree, entry);
588 		return 0;
589 	}
590 	return -EAGAIN;
591 
592 fail:
593 	spin_lock(&tree->lock);
594 	zswap_entry_put(entry);
595 	spin_unlock(&tree->lock);
596 	return ret;
597 }
598 
599 /*********************************
600 * frontswap hooks
601 **********************************/
602 /* attempts to compress and store an single page */
603 static int zswap_frontswap_store(unsigned type, pgoff_t offset,
604 				struct page *page)
605 {
606 	struct zswap_tree *tree = zswap_trees[type];
607 	struct zswap_entry *entry, *dupentry;
608 	int ret;
609 	unsigned int dlen = PAGE_SIZE, len;
610 	unsigned long handle;
611 	char *buf;
612 	u8 *src, *dst;
613 	struct zswap_header *zhdr;
614 
615 	if (!tree) {
616 		ret = -ENODEV;
617 		goto reject;
618 	}
619 
620 	/* reclaim space if needed */
621 	if (zswap_is_full()) {
622 		zswap_pool_limit_hit++;
623 		if (zbud_reclaim_page(tree->pool, 8)) {
624 			zswap_reject_reclaim_fail++;
625 			ret = -ENOMEM;
626 			goto reject;
627 		}
628 	}
629 
630 	/* allocate entry */
631 	entry = zswap_entry_cache_alloc(GFP_KERNEL);
632 	if (!entry) {
633 		zswap_reject_kmemcache_fail++;
634 		ret = -ENOMEM;
635 		goto reject;
636 	}
637 
638 	/* compress */
639 	dst = get_cpu_var(zswap_dstmem);
640 	src = kmap_atomic(page);
641 	ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen);
642 	kunmap_atomic(src);
643 	if (ret) {
644 		ret = -EINVAL;
645 		goto freepage;
646 	}
647 
648 	/* store */
649 	len = dlen + sizeof(struct zswap_header);
650 	ret = zbud_alloc(tree->pool, len, __GFP_NORETRY | __GFP_NOWARN,
651 		&handle);
652 	if (ret == -ENOSPC) {
653 		zswap_reject_compress_poor++;
654 		goto freepage;
655 	}
656 	if (ret) {
657 		zswap_reject_alloc_fail++;
658 		goto freepage;
659 	}
660 	zhdr = zbud_map(tree->pool, handle);
661 	zhdr->swpentry = swp_entry(type, offset);
662 	buf = (u8 *)(zhdr + 1);
663 	memcpy(buf, dst, dlen);
664 	zbud_unmap(tree->pool, handle);
665 	put_cpu_var(zswap_dstmem);
666 
667 	/* populate entry */
668 	entry->offset = offset;
669 	entry->handle = handle;
670 	entry->length = dlen;
671 
672 	/* map */
673 	spin_lock(&tree->lock);
674 	do {
675 		ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
676 		if (ret == -EEXIST) {
677 			zswap_duplicate_entry++;
678 			/* remove from rbtree */
679 			rb_erase(&dupentry->rbnode, &tree->rbroot);
680 			if (!zswap_entry_put(dupentry)) {
681 				/* free */
682 				zswap_free_entry(tree, dupentry);
683 			}
684 		}
685 	} while (ret == -EEXIST);
686 	spin_unlock(&tree->lock);
687 
688 	/* update stats */
689 	atomic_inc(&zswap_stored_pages);
690 	zswap_pool_pages = zbud_get_pool_size(tree->pool);
691 
692 	return 0;
693 
694 freepage:
695 	put_cpu_var(zswap_dstmem);
696 	zswap_entry_cache_free(entry);
697 reject:
698 	return ret;
699 }
700 
701 /*
702  * returns 0 if the page was successfully decompressed
703  * return -1 on entry not found or error
704 */
705 static int zswap_frontswap_load(unsigned type, pgoff_t offset,
706 				struct page *page)
707 {
708 	struct zswap_tree *tree = zswap_trees[type];
709 	struct zswap_entry *entry;
710 	u8 *src, *dst;
711 	unsigned int dlen;
712 	int refcount, ret;
713 
714 	/* find */
715 	spin_lock(&tree->lock);
716 	entry = zswap_rb_search(&tree->rbroot, offset);
717 	if (!entry) {
718 		/* entry was written back */
719 		spin_unlock(&tree->lock);
720 		return -1;
721 	}
722 	zswap_entry_get(entry);
723 	spin_unlock(&tree->lock);
724 
725 	/* decompress */
726 	dlen = PAGE_SIZE;
727 	src = (u8 *)zbud_map(tree->pool, entry->handle) +
728 			sizeof(struct zswap_header);
729 	dst = kmap_atomic(page);
730 	ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, entry->length,
731 		dst, &dlen);
732 	kunmap_atomic(dst);
733 	zbud_unmap(tree->pool, entry->handle);
734 	BUG_ON(ret);
735 
736 	spin_lock(&tree->lock);
737 	refcount = zswap_entry_put(entry);
738 	if (likely(refcount)) {
739 		spin_unlock(&tree->lock);
740 		return 0;
741 	}
742 	spin_unlock(&tree->lock);
743 
744 	/*
745 	 * We don't have to unlink from the rbtree because
746 	 * zswap_writeback_entry() or zswap_frontswap_invalidate page()
747 	 * has already done this for us if we are the last reference.
748 	 */
749 	/* free */
750 
751 	zswap_free_entry(tree, entry);
752 
753 	return 0;
754 }
755 
756 /* frees an entry in zswap */
757 static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
758 {
759 	struct zswap_tree *tree = zswap_trees[type];
760 	struct zswap_entry *entry;
761 	int refcount;
762 
763 	/* find */
764 	spin_lock(&tree->lock);
765 	entry = zswap_rb_search(&tree->rbroot, offset);
766 	if (!entry) {
767 		/* entry was written back */
768 		spin_unlock(&tree->lock);
769 		return;
770 	}
771 
772 	/* remove from rbtree */
773 	rb_erase(&entry->rbnode, &tree->rbroot);
774 
775 	/* drop the initial reference from entry creation */
776 	refcount = zswap_entry_put(entry);
777 
778 	spin_unlock(&tree->lock);
779 
780 	if (refcount) {
781 		/* writeback in progress, writeback will free */
782 		return;
783 	}
784 
785 	/* free */
786 	zswap_free_entry(tree, entry);
787 }
788 
789 /* frees all zswap entries for the given swap type */
790 static void zswap_frontswap_invalidate_area(unsigned type)
791 {
792 	struct zswap_tree *tree = zswap_trees[type];
793 	struct zswap_entry *entry, *n;
794 
795 	if (!tree)
796 		return;
797 
798 	/* walk the tree and free everything */
799 	spin_lock(&tree->lock);
800 	rbtree_postorder_for_each_entry_safe(entry, n, &tree->rbroot, rbnode) {
801 		zbud_free(tree->pool, entry->handle);
802 		zswap_entry_cache_free(entry);
803 		atomic_dec(&zswap_stored_pages);
804 	}
805 	tree->rbroot = RB_ROOT;
806 	spin_unlock(&tree->lock);
807 
808 	zbud_destroy_pool(tree->pool);
809 	kfree(tree);
810 	zswap_trees[type] = NULL;
811 }
812 
813 static struct zbud_ops zswap_zbud_ops = {
814 	.evict = zswap_writeback_entry
815 };
816 
817 static void zswap_frontswap_init(unsigned type)
818 {
819 	struct zswap_tree *tree;
820 
821 	tree = kzalloc(sizeof(struct zswap_tree), GFP_KERNEL);
822 	if (!tree)
823 		goto err;
824 	tree->pool = zbud_create_pool(GFP_KERNEL, &zswap_zbud_ops);
825 	if (!tree->pool)
826 		goto freetree;
827 	tree->rbroot = RB_ROOT;
828 	spin_lock_init(&tree->lock);
829 	zswap_trees[type] = tree;
830 	return;
831 
832 freetree:
833 	kfree(tree);
834 err:
835 	pr_err("alloc failed, zswap disabled for swap type %d\n", type);
836 }
837 
838 static struct frontswap_ops zswap_frontswap_ops = {
839 	.store = zswap_frontswap_store,
840 	.load = zswap_frontswap_load,
841 	.invalidate_page = zswap_frontswap_invalidate_page,
842 	.invalidate_area = zswap_frontswap_invalidate_area,
843 	.init = zswap_frontswap_init
844 };
845 
846 /*********************************
847 * debugfs functions
848 **********************************/
849 #ifdef CONFIG_DEBUG_FS
850 #include <linux/debugfs.h>
851 
852 static struct dentry *zswap_debugfs_root;
853 
854 static int __init zswap_debugfs_init(void)
855 {
856 	if (!debugfs_initialized())
857 		return -ENODEV;
858 
859 	zswap_debugfs_root = debugfs_create_dir("zswap", NULL);
860 	if (!zswap_debugfs_root)
861 		return -ENOMEM;
862 
863 	debugfs_create_u64("pool_limit_hit", S_IRUGO,
864 			zswap_debugfs_root, &zswap_pool_limit_hit);
865 	debugfs_create_u64("reject_reclaim_fail", S_IRUGO,
866 			zswap_debugfs_root, &zswap_reject_reclaim_fail);
867 	debugfs_create_u64("reject_alloc_fail", S_IRUGO,
868 			zswap_debugfs_root, &zswap_reject_alloc_fail);
869 	debugfs_create_u64("reject_kmemcache_fail", S_IRUGO,
870 			zswap_debugfs_root, &zswap_reject_kmemcache_fail);
871 	debugfs_create_u64("reject_compress_poor", S_IRUGO,
872 			zswap_debugfs_root, &zswap_reject_compress_poor);
873 	debugfs_create_u64("written_back_pages", S_IRUGO,
874 			zswap_debugfs_root, &zswap_written_back_pages);
875 	debugfs_create_u64("duplicate_entry", S_IRUGO,
876 			zswap_debugfs_root, &zswap_duplicate_entry);
877 	debugfs_create_u64("pool_pages", S_IRUGO,
878 			zswap_debugfs_root, &zswap_pool_pages);
879 	debugfs_create_atomic_t("stored_pages", S_IRUGO,
880 			zswap_debugfs_root, &zswap_stored_pages);
881 
882 	return 0;
883 }
884 
885 static void __exit zswap_debugfs_exit(void)
886 {
887 	debugfs_remove_recursive(zswap_debugfs_root);
888 }
889 #else
890 static int __init zswap_debugfs_init(void)
891 {
892 	return 0;
893 }
894 
895 static void __exit zswap_debugfs_exit(void) { }
896 #endif
897 
898 /*********************************
899 * module init and exit
900 **********************************/
901 static int __init init_zswap(void)
902 {
903 	if (!zswap_enabled)
904 		return 0;
905 
906 	pr_info("loading zswap\n");
907 	if (zswap_entry_cache_create()) {
908 		pr_err("entry cache creation failed\n");
909 		goto error;
910 	}
911 	if (zswap_comp_init()) {
912 		pr_err("compressor initialization failed\n");
913 		goto compfail;
914 	}
915 	if (zswap_cpu_init()) {
916 		pr_err("per-cpu initialization failed\n");
917 		goto pcpufail;
918 	}
919 	frontswap_register_ops(&zswap_frontswap_ops);
920 	if (zswap_debugfs_init())
921 		pr_warn("debugfs initialization failed\n");
922 	return 0;
923 pcpufail:
924 	zswap_comp_exit();
925 compfail:
926 	zswap_entry_cache_destory();
927 error:
928 	return -ENOMEM;
929 }
930 /* must be late so crypto has time to come up */
931 late_initcall(init_zswap);
932 
933 MODULE_LICENSE("GPL");
934 MODULE_AUTHOR("Seth Jennings <sjenning@linux.vnet.ibm.com>");
935 MODULE_DESCRIPTION("Compressed cache for swap pages");
936