xref: /openbmc/linux/fs/f2fs/compress.c (revision f20c7d91)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * f2fs compress support
4  *
5  * Copyright (c) 2019 Chao Yu <chao@kernel.org>
6  */
7 
8 #include <linux/fs.h>
9 #include <linux/f2fs_fs.h>
10 #include <linux/writeback.h>
11 #include <linux/backing-dev.h>
12 #include <linux/lzo.h>
13 #include <linux/lz4.h>
14 #include <linux/zstd.h>
15 
16 #include "f2fs.h"
17 #include "node.h"
18 #include <trace/events/f2fs.h>
19 
20 struct f2fs_compress_ops {
21 	int (*init_compress_ctx)(struct compress_ctx *cc);
22 	void (*destroy_compress_ctx)(struct compress_ctx *cc);
23 	int (*compress_pages)(struct compress_ctx *cc);
24 	int (*init_decompress_ctx)(struct decompress_io_ctx *dic);
25 	void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic);
26 	int (*decompress_pages)(struct decompress_io_ctx *dic);
27 };
28 
29 static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index)
30 {
31 	return index & (cc->cluster_size - 1);
32 }
33 
34 static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index)
35 {
36 	return index >> cc->log_cluster_size;
37 }
38 
39 static pgoff_t start_idx_of_cluster(struct compress_ctx *cc)
40 {
41 	return cc->cluster_idx << cc->log_cluster_size;
42 }
43 
44 bool f2fs_is_compressed_page(struct page *page)
45 {
46 	if (!PagePrivate(page))
47 		return false;
48 	if (!page_private(page))
49 		return false;
50 	if (IS_ATOMIC_WRITTEN_PAGE(page) || IS_DUMMY_WRITTEN_PAGE(page))
51 		return false;
52 	f2fs_bug_on(F2FS_M_SB(page->mapping),
53 		*((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC);
54 	return true;
55 }
56 
57 static void f2fs_set_compressed_page(struct page *page,
58 		struct inode *inode, pgoff_t index, void *data)
59 {
60 	SetPagePrivate(page);
61 	set_page_private(page, (unsigned long)data);
62 
63 	/* i_crypto_info and iv index */
64 	page->index = index;
65 	page->mapping = inode->i_mapping;
66 }
67 
68 static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock)
69 {
70 	int i;
71 
72 	for (i = 0; i < len; i++) {
73 		if (!cc->rpages[i])
74 			continue;
75 		if (unlock)
76 			unlock_page(cc->rpages[i]);
77 		else
78 			put_page(cc->rpages[i]);
79 	}
80 }
81 
82 static void f2fs_put_rpages(struct compress_ctx *cc)
83 {
84 	f2fs_drop_rpages(cc, cc->cluster_size, false);
85 }
86 
87 static void f2fs_unlock_rpages(struct compress_ctx *cc, int len)
88 {
89 	f2fs_drop_rpages(cc, len, true);
90 }
91 
92 static void f2fs_put_rpages_mapping(struct address_space *mapping,
93 				pgoff_t start, int len)
94 {
95 	int i;
96 
97 	for (i = 0; i < len; i++) {
98 		struct page *page = find_get_page(mapping, start + i);
99 
100 		put_page(page);
101 		put_page(page);
102 	}
103 }
104 
105 static void f2fs_put_rpages_wbc(struct compress_ctx *cc,
106 		struct writeback_control *wbc, bool redirty, int unlock)
107 {
108 	unsigned int i;
109 
110 	for (i = 0; i < cc->cluster_size; i++) {
111 		if (!cc->rpages[i])
112 			continue;
113 		if (redirty)
114 			redirty_page_for_writepage(wbc, cc->rpages[i]);
115 		f2fs_put_page(cc->rpages[i], unlock);
116 	}
117 }
118 
119 struct page *f2fs_compress_control_page(struct page *page)
120 {
121 	return ((struct compress_io_ctx *)page_private(page))->rpages[0];
122 }
123 
124 int f2fs_init_compress_ctx(struct compress_ctx *cc)
125 {
126 	struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
127 
128 	if (cc->nr_rpages)
129 		return 0;
130 
131 	cc->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
132 					cc->log_cluster_size, GFP_NOFS);
133 	return cc->rpages ? 0 : -ENOMEM;
134 }
135 
136 void f2fs_destroy_compress_ctx(struct compress_ctx *cc)
137 {
138 	kfree(cc->rpages);
139 	cc->rpages = NULL;
140 	cc->nr_rpages = 0;
141 	cc->nr_cpages = 0;
142 	cc->cluster_idx = NULL_CLUSTER;
143 }
144 
145 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page)
146 {
147 	unsigned int cluster_ofs;
148 
149 	if (!f2fs_cluster_can_merge_page(cc, page->index))
150 		f2fs_bug_on(F2FS_I_SB(cc->inode), 1);
151 
152 	cluster_ofs = offset_in_cluster(cc, page->index);
153 	cc->rpages[cluster_ofs] = page;
154 	cc->nr_rpages++;
155 	cc->cluster_idx = cluster_idx(cc, page->index);
156 }
157 
158 #ifdef CONFIG_F2FS_FS_LZO
159 static int lzo_init_compress_ctx(struct compress_ctx *cc)
160 {
161 	cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
162 				LZO1X_MEM_COMPRESS, GFP_NOFS);
163 	if (!cc->private)
164 		return -ENOMEM;
165 
166 	cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size);
167 	return 0;
168 }
169 
170 static void lzo_destroy_compress_ctx(struct compress_ctx *cc)
171 {
172 	kvfree(cc->private);
173 	cc->private = NULL;
174 }
175 
176 static int lzo_compress_pages(struct compress_ctx *cc)
177 {
178 	int ret;
179 
180 	ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
181 					&cc->clen, cc->private);
182 	if (ret != LZO_E_OK) {
183 		printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n",
184 				KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret);
185 		return -EIO;
186 	}
187 	return 0;
188 }
189 
190 static int lzo_decompress_pages(struct decompress_io_ctx *dic)
191 {
192 	int ret;
193 
194 	ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen,
195 						dic->rbuf, &dic->rlen);
196 	if (ret != LZO_E_OK) {
197 		printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n",
198 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
199 		return -EIO;
200 	}
201 
202 	if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) {
203 		printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, "
204 					"expected:%lu\n", KERN_ERR,
205 					F2FS_I_SB(dic->inode)->sb->s_id,
206 					dic->rlen,
207 					PAGE_SIZE << dic->log_cluster_size);
208 		return -EIO;
209 	}
210 	return 0;
211 }
212 
213 static const struct f2fs_compress_ops f2fs_lzo_ops = {
214 	.init_compress_ctx	= lzo_init_compress_ctx,
215 	.destroy_compress_ctx	= lzo_destroy_compress_ctx,
216 	.compress_pages		= lzo_compress_pages,
217 	.decompress_pages	= lzo_decompress_pages,
218 };
219 #endif
220 
221 #ifdef CONFIG_F2FS_FS_LZ4
222 static int lz4_init_compress_ctx(struct compress_ctx *cc)
223 {
224 	cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
225 				LZ4_MEM_COMPRESS, GFP_NOFS);
226 	if (!cc->private)
227 		return -ENOMEM;
228 
229 	/*
230 	 * we do not change cc->clen to LZ4_compressBound(inputsize) to
231 	 * adapt worst compress case, because lz4 compressor can handle
232 	 * output budget properly.
233 	 */
234 	cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
235 	return 0;
236 }
237 
238 static void lz4_destroy_compress_ctx(struct compress_ctx *cc)
239 {
240 	kvfree(cc->private);
241 	cc->private = NULL;
242 }
243 
244 static int lz4_compress_pages(struct compress_ctx *cc)
245 {
246 	int len;
247 
248 	len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen,
249 						cc->clen, cc->private);
250 	if (!len)
251 		return -EAGAIN;
252 
253 	cc->clen = len;
254 	return 0;
255 }
256 
257 static int lz4_decompress_pages(struct decompress_io_ctx *dic)
258 {
259 	int ret;
260 
261 	ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf,
262 						dic->clen, dic->rlen);
263 	if (ret < 0) {
264 		printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n",
265 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret);
266 		return -EIO;
267 	}
268 
269 	if (ret != PAGE_SIZE << dic->log_cluster_size) {
270 		printk_ratelimited("%sF2FS-fs (%s): lz4 invalid rlen:%zu, "
271 					"expected:%lu\n", KERN_ERR,
272 					F2FS_I_SB(dic->inode)->sb->s_id,
273 					dic->rlen,
274 					PAGE_SIZE << dic->log_cluster_size);
275 		return -EIO;
276 	}
277 	return 0;
278 }
279 
280 static const struct f2fs_compress_ops f2fs_lz4_ops = {
281 	.init_compress_ctx	= lz4_init_compress_ctx,
282 	.destroy_compress_ctx	= lz4_destroy_compress_ctx,
283 	.compress_pages		= lz4_compress_pages,
284 	.decompress_pages	= lz4_decompress_pages,
285 };
286 #endif
287 
288 #ifdef CONFIG_F2FS_FS_ZSTD
289 #define F2FS_ZSTD_DEFAULT_CLEVEL	1
290 
291 static int zstd_init_compress_ctx(struct compress_ctx *cc)
292 {
293 	ZSTD_parameters params;
294 	ZSTD_CStream *stream;
295 	void *workspace;
296 	unsigned int workspace_size;
297 
298 	params = ZSTD_getParams(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen, 0);
299 	workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams);
300 
301 	workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode),
302 					workspace_size, GFP_NOFS);
303 	if (!workspace)
304 		return -ENOMEM;
305 
306 	stream = ZSTD_initCStream(params, 0, workspace, workspace_size);
307 	if (!stream) {
308 		printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initCStream failed\n",
309 				KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
310 				__func__);
311 		kvfree(workspace);
312 		return -EIO;
313 	}
314 
315 	cc->private = workspace;
316 	cc->private2 = stream;
317 
318 	cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE;
319 	return 0;
320 }
321 
322 static void zstd_destroy_compress_ctx(struct compress_ctx *cc)
323 {
324 	kvfree(cc->private);
325 	cc->private = NULL;
326 	cc->private2 = NULL;
327 }
328 
329 static int zstd_compress_pages(struct compress_ctx *cc)
330 {
331 	ZSTD_CStream *stream = cc->private2;
332 	ZSTD_inBuffer inbuf;
333 	ZSTD_outBuffer outbuf;
334 	int src_size = cc->rlen;
335 	int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE;
336 	int ret;
337 
338 	inbuf.pos = 0;
339 	inbuf.src = cc->rbuf;
340 	inbuf.size = src_size;
341 
342 	outbuf.pos = 0;
343 	outbuf.dst = cc->cbuf->cdata;
344 	outbuf.size = dst_size;
345 
346 	ret = ZSTD_compressStream(stream, &outbuf, &inbuf);
347 	if (ZSTD_isError(ret)) {
348 		printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n",
349 				KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
350 				__func__, ZSTD_getErrorCode(ret));
351 		return -EIO;
352 	}
353 
354 	ret = ZSTD_endStream(stream, &outbuf);
355 	if (ZSTD_isError(ret)) {
356 		printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_endStream returned %d\n",
357 				KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id,
358 				__func__, ZSTD_getErrorCode(ret));
359 		return -EIO;
360 	}
361 
362 	/*
363 	 * there is compressed data remained in intermediate buffer due to
364 	 * no more space in cbuf.cdata
365 	 */
366 	if (ret)
367 		return -EAGAIN;
368 
369 	cc->clen = outbuf.pos;
370 	return 0;
371 }
372 
373 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic)
374 {
375 	ZSTD_DStream *stream;
376 	void *workspace;
377 	unsigned int workspace_size;
378 
379 	workspace_size = ZSTD_DStreamWorkspaceBound(MAX_COMPRESS_WINDOW_SIZE);
380 
381 	workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode),
382 					workspace_size, GFP_NOFS);
383 	if (!workspace)
384 		return -ENOMEM;
385 
386 	stream = ZSTD_initDStream(MAX_COMPRESS_WINDOW_SIZE,
387 					workspace, workspace_size);
388 	if (!stream) {
389 		printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initDStream failed\n",
390 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,
391 				__func__);
392 		kvfree(workspace);
393 		return -EIO;
394 	}
395 
396 	dic->private = workspace;
397 	dic->private2 = stream;
398 
399 	return 0;
400 }
401 
402 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic)
403 {
404 	kvfree(dic->private);
405 	dic->private = NULL;
406 	dic->private2 = NULL;
407 }
408 
409 static int zstd_decompress_pages(struct decompress_io_ctx *dic)
410 {
411 	ZSTD_DStream *stream = dic->private2;
412 	ZSTD_inBuffer inbuf;
413 	ZSTD_outBuffer outbuf;
414 	int ret;
415 
416 	inbuf.pos = 0;
417 	inbuf.src = dic->cbuf->cdata;
418 	inbuf.size = dic->clen;
419 
420 	outbuf.pos = 0;
421 	outbuf.dst = dic->rbuf;
422 	outbuf.size = dic->rlen;
423 
424 	ret = ZSTD_decompressStream(stream, &outbuf, &inbuf);
425 	if (ZSTD_isError(ret)) {
426 		printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n",
427 				KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id,
428 				__func__, ZSTD_getErrorCode(ret));
429 		return -EIO;
430 	}
431 
432 	if (dic->rlen != outbuf.pos) {
433 		printk_ratelimited("%sF2FS-fs (%s): %s ZSTD invalid rlen:%zu, "
434 				"expected:%lu\n", KERN_ERR,
435 				F2FS_I_SB(dic->inode)->sb->s_id,
436 				__func__, dic->rlen,
437 				PAGE_SIZE << dic->log_cluster_size);
438 		return -EIO;
439 	}
440 
441 	return 0;
442 }
443 
444 static const struct f2fs_compress_ops f2fs_zstd_ops = {
445 	.init_compress_ctx	= zstd_init_compress_ctx,
446 	.destroy_compress_ctx	= zstd_destroy_compress_ctx,
447 	.compress_pages		= zstd_compress_pages,
448 	.init_decompress_ctx	= zstd_init_decompress_ctx,
449 	.destroy_decompress_ctx	= zstd_destroy_decompress_ctx,
450 	.decompress_pages	= zstd_decompress_pages,
451 };
452 #endif
453 
454 #ifdef CONFIG_F2FS_FS_LZO
455 #ifdef CONFIG_F2FS_FS_LZORLE
456 static int lzorle_compress_pages(struct compress_ctx *cc)
457 {
458 	int ret;
459 
460 	ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
461 					&cc->clen, cc->private);
462 	if (ret != LZO_E_OK) {
463 		printk_ratelimited("%sF2FS-fs (%s): lzo-rle compress failed, ret:%d\n",
464 				KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret);
465 		return -EIO;
466 	}
467 	return 0;
468 }
469 
470 static const struct f2fs_compress_ops f2fs_lzorle_ops = {
471 	.init_compress_ctx	= lzo_init_compress_ctx,
472 	.destroy_compress_ctx	= lzo_destroy_compress_ctx,
473 	.compress_pages		= lzorle_compress_pages,
474 	.decompress_pages	= lzo_decompress_pages,
475 };
476 #endif
477 #endif
478 
479 static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = {
480 #ifdef CONFIG_F2FS_FS_LZO
481 	&f2fs_lzo_ops,
482 #else
483 	NULL,
484 #endif
485 #ifdef CONFIG_F2FS_FS_LZ4
486 	&f2fs_lz4_ops,
487 #else
488 	NULL,
489 #endif
490 #ifdef CONFIG_F2FS_FS_ZSTD
491 	&f2fs_zstd_ops,
492 #else
493 	NULL,
494 #endif
495 #if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE)
496 	&f2fs_lzorle_ops,
497 #else
498 	NULL,
499 #endif
500 };
501 
502 bool f2fs_is_compress_backend_ready(struct inode *inode)
503 {
504 	if (!f2fs_compressed_file(inode))
505 		return true;
506 	return f2fs_cops[F2FS_I(inode)->i_compress_algorithm];
507 }
508 
509 static mempool_t *compress_page_pool = NULL;
510 static int num_compress_pages = 512;
511 module_param(num_compress_pages, uint, 0444);
512 MODULE_PARM_DESC(num_compress_pages,
513 		"Number of intermediate compress pages to preallocate");
514 
515 int f2fs_init_compress_mempool(void)
516 {
517 	compress_page_pool = mempool_create_page_pool(num_compress_pages, 0);
518 	if (!compress_page_pool)
519 		return -ENOMEM;
520 
521 	return 0;
522 }
523 
524 void f2fs_destroy_compress_mempool(void)
525 {
526 	mempool_destroy(compress_page_pool);
527 }
528 
529 static struct page *f2fs_compress_alloc_page(void)
530 {
531 	struct page *page;
532 
533 	page = mempool_alloc(compress_page_pool, GFP_NOFS);
534 	lock_page(page);
535 
536 	return page;
537 }
538 
539 static void f2fs_compress_free_page(struct page *page)
540 {
541 	if (!page)
542 		return;
543 	set_page_private(page, (unsigned long)NULL);
544 	ClearPagePrivate(page);
545 	page->mapping = NULL;
546 	unlock_page(page);
547 	mempool_free(page, compress_page_pool);
548 }
549 
550 static int f2fs_compress_pages(struct compress_ctx *cc)
551 {
552 	struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
553 	struct f2fs_inode_info *fi = F2FS_I(cc->inode);
554 	const struct f2fs_compress_ops *cops =
555 				f2fs_cops[fi->i_compress_algorithm];
556 	unsigned int max_len, nr_cpages;
557 	int i, ret;
558 
559 	trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx,
560 				cc->cluster_size, fi->i_compress_algorithm);
561 
562 	if (cops->init_compress_ctx) {
563 		ret = cops->init_compress_ctx(cc);
564 		if (ret)
565 			goto out;
566 	}
567 
568 	max_len = COMPRESS_HEADER_SIZE + cc->clen;
569 	cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE);
570 
571 	cc->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
572 					cc->nr_cpages, GFP_NOFS);
573 	if (!cc->cpages) {
574 		ret = -ENOMEM;
575 		goto destroy_compress_ctx;
576 	}
577 
578 	for (i = 0; i < cc->nr_cpages; i++) {
579 		cc->cpages[i] = f2fs_compress_alloc_page();
580 		if (!cc->cpages[i]) {
581 			ret = -ENOMEM;
582 			goto out_free_cpages;
583 		}
584 	}
585 
586 	cc->rbuf = vmap(cc->rpages, cc->cluster_size, VM_MAP, PAGE_KERNEL_RO);
587 	if (!cc->rbuf) {
588 		ret = -ENOMEM;
589 		goto out_free_cpages;
590 	}
591 
592 	cc->cbuf = vmap(cc->cpages, cc->nr_cpages, VM_MAP, PAGE_KERNEL);
593 	if (!cc->cbuf) {
594 		ret = -ENOMEM;
595 		goto out_vunmap_rbuf;
596 	}
597 
598 	ret = cops->compress_pages(cc);
599 	if (ret)
600 		goto out_vunmap_cbuf;
601 
602 	max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE;
603 
604 	if (cc->clen > max_len) {
605 		ret = -EAGAIN;
606 		goto out_vunmap_cbuf;
607 	}
608 
609 	cc->cbuf->clen = cpu_to_le32(cc->clen);
610 
611 	for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++)
612 		cc->cbuf->reserved[i] = cpu_to_le32(0);
613 
614 	nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE);
615 
616 	/* zero out any unused part of the last page */
617 	memset(&cc->cbuf->cdata[cc->clen], 0,
618 	       (nr_cpages * PAGE_SIZE) - (cc->clen + COMPRESS_HEADER_SIZE));
619 
620 	vunmap(cc->cbuf);
621 	vunmap(cc->rbuf);
622 
623 	for (i = nr_cpages; i < cc->nr_cpages; i++) {
624 		f2fs_compress_free_page(cc->cpages[i]);
625 		cc->cpages[i] = NULL;
626 	}
627 
628 	if (cops->destroy_compress_ctx)
629 		cops->destroy_compress_ctx(cc);
630 
631 	cc->nr_cpages = nr_cpages;
632 
633 	trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
634 							cc->clen, ret);
635 	return 0;
636 
637 out_vunmap_cbuf:
638 	vunmap(cc->cbuf);
639 out_vunmap_rbuf:
640 	vunmap(cc->rbuf);
641 out_free_cpages:
642 	for (i = 0; i < cc->nr_cpages; i++) {
643 		if (cc->cpages[i])
644 			f2fs_compress_free_page(cc->cpages[i]);
645 	}
646 	kfree(cc->cpages);
647 	cc->cpages = NULL;
648 destroy_compress_ctx:
649 	if (cops->destroy_compress_ctx)
650 		cops->destroy_compress_ctx(cc);
651 out:
652 	trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx,
653 							cc->clen, ret);
654 	return ret;
655 }
656 
657 void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity)
658 {
659 	struct decompress_io_ctx *dic =
660 			(struct decompress_io_ctx *)page_private(page);
661 	struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode);
662 	struct f2fs_inode_info *fi= F2FS_I(dic->inode);
663 	const struct f2fs_compress_ops *cops =
664 			f2fs_cops[fi->i_compress_algorithm];
665 	int ret;
666 
667 	dec_page_count(sbi, F2FS_RD_DATA);
668 
669 	if (bio->bi_status || PageError(page))
670 		dic->failed = true;
671 
672 	if (refcount_dec_not_one(&dic->ref))
673 		return;
674 
675 	trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx,
676 				dic->cluster_size, fi->i_compress_algorithm);
677 
678 	/* submit partial compressed pages */
679 	if (dic->failed) {
680 		ret = -EIO;
681 		goto out_free_dic;
682 	}
683 
684 	if (cops->init_decompress_ctx) {
685 		ret = cops->init_decompress_ctx(dic);
686 		if (ret)
687 			goto out_free_dic;
688 	}
689 
690 	dic->rbuf = vmap(dic->tpages, dic->cluster_size, VM_MAP, PAGE_KERNEL);
691 	if (!dic->rbuf) {
692 		ret = -ENOMEM;
693 		goto destroy_decompress_ctx;
694 	}
695 
696 	dic->cbuf = vmap(dic->cpages, dic->nr_cpages, VM_MAP, PAGE_KERNEL_RO);
697 	if (!dic->cbuf) {
698 		ret = -ENOMEM;
699 		goto out_vunmap_rbuf;
700 	}
701 
702 	dic->clen = le32_to_cpu(dic->cbuf->clen);
703 	dic->rlen = PAGE_SIZE << dic->log_cluster_size;
704 
705 	if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) {
706 		ret = -EFSCORRUPTED;
707 		goto out_vunmap_cbuf;
708 	}
709 
710 	ret = cops->decompress_pages(dic);
711 
712 out_vunmap_cbuf:
713 	vunmap(dic->cbuf);
714 out_vunmap_rbuf:
715 	vunmap(dic->rbuf);
716 destroy_decompress_ctx:
717 	if (cops->destroy_decompress_ctx)
718 		cops->destroy_decompress_ctx(dic);
719 out_free_dic:
720 	if (verity)
721 		refcount_set(&dic->ref, dic->nr_cpages);
722 	if (!verity)
723 		f2fs_decompress_end_io(dic->rpages, dic->cluster_size,
724 								ret, false);
725 
726 	trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx,
727 							dic->clen, ret);
728 	if (!verity)
729 		f2fs_free_dic(dic);
730 }
731 
732 static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index)
733 {
734 	if (cc->cluster_idx == NULL_CLUSTER)
735 		return true;
736 	return cc->cluster_idx == cluster_idx(cc, index);
737 }
738 
739 bool f2fs_cluster_is_empty(struct compress_ctx *cc)
740 {
741 	return cc->nr_rpages == 0;
742 }
743 
744 static bool f2fs_cluster_is_full(struct compress_ctx *cc)
745 {
746 	return cc->cluster_size == cc->nr_rpages;
747 }
748 
749 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index)
750 {
751 	if (f2fs_cluster_is_empty(cc))
752 		return true;
753 	return is_page_in_cluster(cc, index);
754 }
755 
756 static bool __cluster_may_compress(struct compress_ctx *cc)
757 {
758 	struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
759 	loff_t i_size = i_size_read(cc->inode);
760 	unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE);
761 	int i;
762 
763 	for (i = 0; i < cc->cluster_size; i++) {
764 		struct page *page = cc->rpages[i];
765 
766 		f2fs_bug_on(sbi, !page);
767 
768 		if (unlikely(f2fs_cp_error(sbi)))
769 			return false;
770 		if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
771 			return false;
772 
773 		/* beyond EOF */
774 		if (page->index >= nr_pages)
775 			return false;
776 	}
777 	return true;
778 }
779 
780 static int __f2fs_cluster_blocks(struct compress_ctx *cc, bool compr)
781 {
782 	struct dnode_of_data dn;
783 	int ret;
784 
785 	set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
786 	ret = f2fs_get_dnode_of_data(&dn, start_idx_of_cluster(cc),
787 							LOOKUP_NODE);
788 	if (ret) {
789 		if (ret == -ENOENT)
790 			ret = 0;
791 		goto fail;
792 	}
793 
794 	if (dn.data_blkaddr == COMPRESS_ADDR) {
795 		int i;
796 
797 		ret = 1;
798 		for (i = 1; i < cc->cluster_size; i++) {
799 			block_t blkaddr;
800 
801 			blkaddr = data_blkaddr(dn.inode,
802 					dn.node_page, dn.ofs_in_node + i);
803 			if (compr) {
804 				if (__is_valid_data_blkaddr(blkaddr))
805 					ret++;
806 			} else {
807 				if (blkaddr != NULL_ADDR)
808 					ret++;
809 			}
810 		}
811 	}
812 fail:
813 	f2fs_put_dnode(&dn);
814 	return ret;
815 }
816 
817 /* return # of compressed blocks in compressed cluster */
818 static int f2fs_compressed_blocks(struct compress_ctx *cc)
819 {
820 	return __f2fs_cluster_blocks(cc, true);
821 }
822 
823 /* return # of valid blocks in compressed cluster */
824 static int f2fs_cluster_blocks(struct compress_ctx *cc, bool compr)
825 {
826 	return __f2fs_cluster_blocks(cc, false);
827 }
828 
829 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index)
830 {
831 	struct compress_ctx cc = {
832 		.inode = inode,
833 		.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
834 		.cluster_size = F2FS_I(inode)->i_cluster_size,
835 		.cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
836 	};
837 
838 	return f2fs_cluster_blocks(&cc, false);
839 }
840 
841 static bool cluster_may_compress(struct compress_ctx *cc)
842 {
843 	if (!f2fs_compressed_file(cc->inode))
844 		return false;
845 	if (f2fs_is_atomic_file(cc->inode))
846 		return false;
847 	if (f2fs_is_mmap_file(cc->inode))
848 		return false;
849 	if (!f2fs_cluster_is_full(cc))
850 		return false;
851 	if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode))))
852 		return false;
853 	return __cluster_may_compress(cc);
854 }
855 
856 static void set_cluster_writeback(struct compress_ctx *cc)
857 {
858 	int i;
859 
860 	for (i = 0; i < cc->cluster_size; i++) {
861 		if (cc->rpages[i])
862 			set_page_writeback(cc->rpages[i]);
863 	}
864 }
865 
866 static void set_cluster_dirty(struct compress_ctx *cc)
867 {
868 	int i;
869 
870 	for (i = 0; i < cc->cluster_size; i++)
871 		if (cc->rpages[i])
872 			set_page_dirty(cc->rpages[i]);
873 }
874 
875 static int prepare_compress_overwrite(struct compress_ctx *cc,
876 		struct page **pagep, pgoff_t index, void **fsdata)
877 {
878 	struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
879 	struct address_space *mapping = cc->inode->i_mapping;
880 	struct page *page;
881 	struct dnode_of_data dn;
882 	sector_t last_block_in_bio;
883 	unsigned fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT;
884 	pgoff_t start_idx = start_idx_of_cluster(cc);
885 	int i, ret;
886 	bool prealloc;
887 
888 retry:
889 	ret = f2fs_cluster_blocks(cc, false);
890 	if (ret <= 0)
891 		return ret;
892 
893 	/* compressed case */
894 	prealloc = (ret < cc->cluster_size);
895 
896 	ret = f2fs_init_compress_ctx(cc);
897 	if (ret)
898 		return ret;
899 
900 	/* keep page reference to avoid page reclaim */
901 	for (i = 0; i < cc->cluster_size; i++) {
902 		page = f2fs_pagecache_get_page(mapping, start_idx + i,
903 							fgp_flag, GFP_NOFS);
904 		if (!page) {
905 			ret = -ENOMEM;
906 			goto unlock_pages;
907 		}
908 
909 		if (PageUptodate(page))
910 			unlock_page(page);
911 		else
912 			f2fs_compress_ctx_add_page(cc, page);
913 	}
914 
915 	if (!f2fs_cluster_is_empty(cc)) {
916 		struct bio *bio = NULL;
917 
918 		ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size,
919 					&last_block_in_bio, false, true);
920 		f2fs_destroy_compress_ctx(cc);
921 		if (ret)
922 			goto release_pages;
923 		if (bio)
924 			f2fs_submit_bio(sbi, bio, DATA);
925 
926 		ret = f2fs_init_compress_ctx(cc);
927 		if (ret)
928 			goto release_pages;
929 	}
930 
931 	for (i = 0; i < cc->cluster_size; i++) {
932 		f2fs_bug_on(sbi, cc->rpages[i]);
933 
934 		page = find_lock_page(mapping, start_idx + i);
935 		f2fs_bug_on(sbi, !page);
936 
937 		f2fs_wait_on_page_writeback(page, DATA, true, true);
938 
939 		f2fs_compress_ctx_add_page(cc, page);
940 		f2fs_put_page(page, 0);
941 
942 		if (!PageUptodate(page)) {
943 			f2fs_unlock_rpages(cc, i + 1);
944 			f2fs_put_rpages_mapping(mapping, start_idx,
945 					cc->cluster_size);
946 			f2fs_destroy_compress_ctx(cc);
947 			goto retry;
948 		}
949 	}
950 
951 	if (prealloc) {
952 		__do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
953 
954 		set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
955 
956 		for (i = cc->cluster_size - 1; i > 0; i--) {
957 			ret = f2fs_get_block(&dn, start_idx + i);
958 			if (ret) {
959 				i = cc->cluster_size;
960 				break;
961 			}
962 
963 			if (dn.data_blkaddr != NEW_ADDR)
964 				break;
965 		}
966 
967 		__do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
968 	}
969 
970 	if (likely(!ret)) {
971 		*fsdata = cc->rpages;
972 		*pagep = cc->rpages[offset_in_cluster(cc, index)];
973 		return cc->cluster_size;
974 	}
975 
976 unlock_pages:
977 	f2fs_unlock_rpages(cc, i);
978 release_pages:
979 	f2fs_put_rpages_mapping(mapping, start_idx, i);
980 	f2fs_destroy_compress_ctx(cc);
981 	return ret;
982 }
983 
984 int f2fs_prepare_compress_overwrite(struct inode *inode,
985 		struct page **pagep, pgoff_t index, void **fsdata)
986 {
987 	struct compress_ctx cc = {
988 		.inode = inode,
989 		.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
990 		.cluster_size = F2FS_I(inode)->i_cluster_size,
991 		.cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size,
992 		.rpages = NULL,
993 		.nr_rpages = 0,
994 	};
995 
996 	return prepare_compress_overwrite(&cc, pagep, index, fsdata);
997 }
998 
999 bool f2fs_compress_write_end(struct inode *inode, void *fsdata,
1000 					pgoff_t index, unsigned copied)
1001 
1002 {
1003 	struct compress_ctx cc = {
1004 		.log_cluster_size = F2FS_I(inode)->i_log_cluster_size,
1005 		.cluster_size = F2FS_I(inode)->i_cluster_size,
1006 		.rpages = fsdata,
1007 	};
1008 	bool first_index = (index == cc.rpages[0]->index);
1009 
1010 	if (copied)
1011 		set_cluster_dirty(&cc);
1012 
1013 	f2fs_put_rpages_wbc(&cc, NULL, false, 1);
1014 	f2fs_destroy_compress_ctx(&cc);
1015 
1016 	return first_index;
1017 }
1018 
1019 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock)
1020 {
1021 	void *fsdata = NULL;
1022 	struct page *pagep;
1023 	int log_cluster_size = F2FS_I(inode)->i_log_cluster_size;
1024 	pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) <<
1025 							log_cluster_size;
1026 	int err;
1027 
1028 	err = f2fs_is_compressed_cluster(inode, start_idx);
1029 	if (err < 0)
1030 		return err;
1031 
1032 	/* truncate normal cluster */
1033 	if (!err)
1034 		return f2fs_do_truncate_blocks(inode, from, lock);
1035 
1036 	/* truncate compressed cluster */
1037 	err = f2fs_prepare_compress_overwrite(inode, &pagep,
1038 						start_idx, &fsdata);
1039 
1040 	/* should not be a normal cluster */
1041 	f2fs_bug_on(F2FS_I_SB(inode), err == 0);
1042 
1043 	if (err <= 0)
1044 		return err;
1045 
1046 	if (err > 0) {
1047 		struct page **rpages = fsdata;
1048 		int cluster_size = F2FS_I(inode)->i_cluster_size;
1049 		int i;
1050 
1051 		for (i = cluster_size - 1; i >= 0; i--) {
1052 			loff_t start = rpages[i]->index << PAGE_SHIFT;
1053 
1054 			if (from <= start) {
1055 				zero_user_segment(rpages[i], 0, PAGE_SIZE);
1056 			} else {
1057 				zero_user_segment(rpages[i], from - start,
1058 								PAGE_SIZE);
1059 				break;
1060 			}
1061 		}
1062 
1063 		f2fs_compress_write_end(inode, fsdata, start_idx, true);
1064 	}
1065 	return 0;
1066 }
1067 
1068 static int f2fs_write_compressed_pages(struct compress_ctx *cc,
1069 					int *submitted,
1070 					struct writeback_control *wbc,
1071 					enum iostat_type io_type)
1072 {
1073 	struct inode *inode = cc->inode;
1074 	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1075 	struct f2fs_inode_info *fi = F2FS_I(inode);
1076 	struct f2fs_io_info fio = {
1077 		.sbi = sbi,
1078 		.ino = cc->inode->i_ino,
1079 		.type = DATA,
1080 		.op = REQ_OP_WRITE,
1081 		.op_flags = wbc_to_write_flags(wbc),
1082 		.old_blkaddr = NEW_ADDR,
1083 		.page = NULL,
1084 		.encrypted_page = NULL,
1085 		.compressed_page = NULL,
1086 		.submitted = false,
1087 		.io_type = io_type,
1088 		.io_wbc = wbc,
1089 		.encrypted = f2fs_encrypted_file(cc->inode),
1090 	};
1091 	struct dnode_of_data dn;
1092 	struct node_info ni;
1093 	struct compress_io_ctx *cic;
1094 	pgoff_t start_idx = start_idx_of_cluster(cc);
1095 	unsigned int last_index = cc->cluster_size - 1;
1096 	loff_t psize;
1097 	int i, err;
1098 
1099 	if (!IS_NOQUOTA(inode) && !f2fs_trylock_op(sbi))
1100 		return -EAGAIN;
1101 
1102 	set_new_dnode(&dn, cc->inode, NULL, NULL, 0);
1103 
1104 	err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE);
1105 	if (err)
1106 		goto out_unlock_op;
1107 
1108 	for (i = 0; i < cc->cluster_size; i++) {
1109 		if (data_blkaddr(dn.inode, dn.node_page,
1110 					dn.ofs_in_node + i) == NULL_ADDR)
1111 			goto out_put_dnode;
1112 	}
1113 
1114 	psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT;
1115 
1116 	err = f2fs_get_node_info(fio.sbi, dn.nid, &ni);
1117 	if (err)
1118 		goto out_put_dnode;
1119 
1120 	fio.version = ni.version;
1121 
1122 	cic = f2fs_kzalloc(sbi, sizeof(struct compress_io_ctx), GFP_NOFS);
1123 	if (!cic)
1124 		goto out_put_dnode;
1125 
1126 	cic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1127 	cic->inode = inode;
1128 	refcount_set(&cic->ref, cc->nr_cpages);
1129 	cic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1130 			cc->log_cluster_size, GFP_NOFS);
1131 	if (!cic->rpages)
1132 		goto out_put_cic;
1133 
1134 	cic->nr_rpages = cc->cluster_size;
1135 
1136 	for (i = 0; i < cc->nr_cpages; i++) {
1137 		f2fs_set_compressed_page(cc->cpages[i], inode,
1138 					cc->rpages[i + 1]->index, cic);
1139 		fio.compressed_page = cc->cpages[i];
1140 		if (fio.encrypted) {
1141 			fio.page = cc->rpages[i + 1];
1142 			err = f2fs_encrypt_one_page(&fio);
1143 			if (err)
1144 				goto out_destroy_crypt;
1145 			cc->cpages[i] = fio.encrypted_page;
1146 		}
1147 	}
1148 
1149 	set_cluster_writeback(cc);
1150 
1151 	for (i = 0; i < cc->cluster_size; i++)
1152 		cic->rpages[i] = cc->rpages[i];
1153 
1154 	for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) {
1155 		block_t blkaddr;
1156 
1157 		blkaddr = f2fs_data_blkaddr(&dn);
1158 		fio.page = cc->rpages[i];
1159 		fio.old_blkaddr = blkaddr;
1160 
1161 		/* cluster header */
1162 		if (i == 0) {
1163 			if (blkaddr == COMPRESS_ADDR)
1164 				fio.compr_blocks++;
1165 			if (__is_valid_data_blkaddr(blkaddr))
1166 				f2fs_invalidate_blocks(sbi, blkaddr);
1167 			f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR);
1168 			goto unlock_continue;
1169 		}
1170 
1171 		if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr))
1172 			fio.compr_blocks++;
1173 
1174 		if (i > cc->nr_cpages) {
1175 			if (__is_valid_data_blkaddr(blkaddr)) {
1176 				f2fs_invalidate_blocks(sbi, blkaddr);
1177 				f2fs_update_data_blkaddr(&dn, NEW_ADDR);
1178 			}
1179 			goto unlock_continue;
1180 		}
1181 
1182 		f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR);
1183 
1184 		if (fio.encrypted)
1185 			fio.encrypted_page = cc->cpages[i - 1];
1186 		else
1187 			fio.compressed_page = cc->cpages[i - 1];
1188 
1189 		cc->cpages[i - 1] = NULL;
1190 		f2fs_outplace_write_data(&dn, &fio);
1191 		(*submitted)++;
1192 unlock_continue:
1193 		inode_dec_dirty_pages(cc->inode);
1194 		unlock_page(fio.page);
1195 	}
1196 
1197 	if (fio.compr_blocks)
1198 		f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false);
1199 	f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true);
1200 
1201 	set_inode_flag(cc->inode, FI_APPEND_WRITE);
1202 	if (cc->cluster_idx == 0)
1203 		set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1204 
1205 	f2fs_put_dnode(&dn);
1206 	if (!IS_NOQUOTA(inode))
1207 		f2fs_unlock_op(sbi);
1208 
1209 	spin_lock(&fi->i_size_lock);
1210 	if (fi->last_disk_size < psize)
1211 		fi->last_disk_size = psize;
1212 	spin_unlock(&fi->i_size_lock);
1213 
1214 	f2fs_put_rpages(cc);
1215 	f2fs_destroy_compress_ctx(cc);
1216 	return 0;
1217 
1218 out_destroy_crypt:
1219 	kfree(cic->rpages);
1220 
1221 	for (--i; i >= 0; i--)
1222 		fscrypt_finalize_bounce_page(&cc->cpages[i]);
1223 	for (i = 0; i < cc->nr_cpages; i++) {
1224 		if (!cc->cpages[i])
1225 			continue;
1226 		f2fs_put_page(cc->cpages[i], 1);
1227 	}
1228 out_put_cic:
1229 	kfree(cic);
1230 out_put_dnode:
1231 	f2fs_put_dnode(&dn);
1232 out_unlock_op:
1233 	if (!IS_NOQUOTA(inode))
1234 		f2fs_unlock_op(sbi);
1235 	return -EAGAIN;
1236 }
1237 
1238 void f2fs_compress_write_end_io(struct bio *bio, struct page *page)
1239 {
1240 	struct f2fs_sb_info *sbi = bio->bi_private;
1241 	struct compress_io_ctx *cic =
1242 			(struct compress_io_ctx *)page_private(page);
1243 	int i;
1244 
1245 	if (unlikely(bio->bi_status))
1246 		mapping_set_error(cic->inode->i_mapping, -EIO);
1247 
1248 	f2fs_compress_free_page(page);
1249 
1250 	dec_page_count(sbi, F2FS_WB_DATA);
1251 
1252 	if (refcount_dec_not_one(&cic->ref))
1253 		return;
1254 
1255 	for (i = 0; i < cic->nr_rpages; i++) {
1256 		WARN_ON(!cic->rpages[i]);
1257 		clear_cold_data(cic->rpages[i]);
1258 		end_page_writeback(cic->rpages[i]);
1259 	}
1260 
1261 	kfree(cic->rpages);
1262 	kfree(cic);
1263 }
1264 
1265 static int f2fs_write_raw_pages(struct compress_ctx *cc,
1266 					int *submitted,
1267 					struct writeback_control *wbc,
1268 					enum iostat_type io_type)
1269 {
1270 	struct address_space *mapping = cc->inode->i_mapping;
1271 	int _submitted, compr_blocks, ret;
1272 	int i = -1, err = 0;
1273 
1274 	compr_blocks = f2fs_compressed_blocks(cc);
1275 	if (compr_blocks < 0) {
1276 		err = compr_blocks;
1277 		goto out_err;
1278 	}
1279 
1280 	for (i = 0; i < cc->cluster_size; i++) {
1281 		if (!cc->rpages[i])
1282 			continue;
1283 retry_write:
1284 		if (cc->rpages[i]->mapping != mapping) {
1285 			unlock_page(cc->rpages[i]);
1286 			continue;
1287 		}
1288 
1289 		BUG_ON(!PageLocked(cc->rpages[i]));
1290 
1291 		ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted,
1292 						NULL, NULL, wbc, io_type,
1293 						compr_blocks);
1294 		if (ret) {
1295 			if (ret == AOP_WRITEPAGE_ACTIVATE) {
1296 				unlock_page(cc->rpages[i]);
1297 				ret = 0;
1298 			} else if (ret == -EAGAIN) {
1299 				/*
1300 				 * for quota file, just redirty left pages to
1301 				 * avoid deadlock caused by cluster update race
1302 				 * from foreground operation.
1303 				 */
1304 				if (IS_NOQUOTA(cc->inode)) {
1305 					err = 0;
1306 					goto out_err;
1307 				}
1308 				ret = 0;
1309 				cond_resched();
1310 				congestion_wait(BLK_RW_ASYNC,
1311 						DEFAULT_IO_TIMEOUT);
1312 				lock_page(cc->rpages[i]);
1313 				clear_page_dirty_for_io(cc->rpages[i]);
1314 				goto retry_write;
1315 			}
1316 			err = ret;
1317 			goto out_err;
1318 		}
1319 
1320 		*submitted += _submitted;
1321 	}
1322 	return 0;
1323 out_err:
1324 	for (++i; i < cc->cluster_size; i++) {
1325 		if (!cc->rpages[i])
1326 			continue;
1327 		redirty_page_for_writepage(wbc, cc->rpages[i]);
1328 		unlock_page(cc->rpages[i]);
1329 	}
1330 	return err;
1331 }
1332 
1333 int f2fs_write_multi_pages(struct compress_ctx *cc,
1334 					int *submitted,
1335 					struct writeback_control *wbc,
1336 					enum iostat_type io_type)
1337 {
1338 	struct f2fs_inode_info *fi = F2FS_I(cc->inode);
1339 	const struct f2fs_compress_ops *cops =
1340 			f2fs_cops[fi->i_compress_algorithm];
1341 	int err;
1342 
1343 	*submitted = 0;
1344 	if (cluster_may_compress(cc)) {
1345 		err = f2fs_compress_pages(cc);
1346 		if (err == -EAGAIN) {
1347 			goto write;
1348 		} else if (err) {
1349 			f2fs_put_rpages_wbc(cc, wbc, true, 1);
1350 			goto destroy_out;
1351 		}
1352 
1353 		err = f2fs_write_compressed_pages(cc, submitted,
1354 							wbc, io_type);
1355 		cops->destroy_compress_ctx(cc);
1356 		if (!err)
1357 			return 0;
1358 		f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN);
1359 	}
1360 write:
1361 	f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted);
1362 
1363 	err = f2fs_write_raw_pages(cc, submitted, wbc, io_type);
1364 	f2fs_put_rpages_wbc(cc, wbc, false, 0);
1365 destroy_out:
1366 	f2fs_destroy_compress_ctx(cc);
1367 	return err;
1368 }
1369 
1370 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc)
1371 {
1372 	struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode);
1373 	struct decompress_io_ctx *dic;
1374 	pgoff_t start_idx = start_idx_of_cluster(cc);
1375 	int i;
1376 
1377 	dic = f2fs_kzalloc(sbi, sizeof(struct decompress_io_ctx), GFP_NOFS);
1378 	if (!dic)
1379 		return ERR_PTR(-ENOMEM);
1380 
1381 	dic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) <<
1382 			cc->log_cluster_size, GFP_NOFS);
1383 	if (!dic->rpages) {
1384 		kfree(dic);
1385 		return ERR_PTR(-ENOMEM);
1386 	}
1387 
1388 	dic->magic = F2FS_COMPRESSED_PAGE_MAGIC;
1389 	dic->inode = cc->inode;
1390 	refcount_set(&dic->ref, cc->nr_cpages);
1391 	dic->cluster_idx = cc->cluster_idx;
1392 	dic->cluster_size = cc->cluster_size;
1393 	dic->log_cluster_size = cc->log_cluster_size;
1394 	dic->nr_cpages = cc->nr_cpages;
1395 	dic->failed = false;
1396 
1397 	for (i = 0; i < dic->cluster_size; i++)
1398 		dic->rpages[i] = cc->rpages[i];
1399 	dic->nr_rpages = cc->cluster_size;
1400 
1401 	dic->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1402 					dic->nr_cpages, GFP_NOFS);
1403 	if (!dic->cpages)
1404 		goto out_free;
1405 
1406 	for (i = 0; i < dic->nr_cpages; i++) {
1407 		struct page *page;
1408 
1409 		page = f2fs_compress_alloc_page();
1410 		if (!page)
1411 			goto out_free;
1412 
1413 		f2fs_set_compressed_page(page, cc->inode,
1414 					start_idx + i + 1, dic);
1415 		dic->cpages[i] = page;
1416 	}
1417 
1418 	dic->tpages = f2fs_kzalloc(sbi, sizeof(struct page *) *
1419 					dic->cluster_size, GFP_NOFS);
1420 	if (!dic->tpages)
1421 		goto out_free;
1422 
1423 	for (i = 0; i < dic->cluster_size; i++) {
1424 		if (cc->rpages[i]) {
1425 			dic->tpages[i] = cc->rpages[i];
1426 			continue;
1427 		}
1428 
1429 		dic->tpages[i] = f2fs_compress_alloc_page();
1430 		if (!dic->tpages[i])
1431 			goto out_free;
1432 	}
1433 
1434 	return dic;
1435 
1436 out_free:
1437 	f2fs_free_dic(dic);
1438 	return ERR_PTR(-ENOMEM);
1439 }
1440 
1441 void f2fs_free_dic(struct decompress_io_ctx *dic)
1442 {
1443 	int i;
1444 
1445 	if (dic->tpages) {
1446 		for (i = 0; i < dic->cluster_size; i++) {
1447 			if (dic->rpages[i])
1448 				continue;
1449 			if (!dic->tpages[i])
1450 				continue;
1451 			f2fs_compress_free_page(dic->tpages[i]);
1452 		}
1453 		kfree(dic->tpages);
1454 	}
1455 
1456 	if (dic->cpages) {
1457 		for (i = 0; i < dic->nr_cpages; i++) {
1458 			if (!dic->cpages[i])
1459 				continue;
1460 			f2fs_compress_free_page(dic->cpages[i]);
1461 		}
1462 		kfree(dic->cpages);
1463 	}
1464 
1465 	kfree(dic->rpages);
1466 	kfree(dic);
1467 }
1468 
1469 void f2fs_decompress_end_io(struct page **rpages,
1470 			unsigned int cluster_size, bool err, bool verity)
1471 {
1472 	int i;
1473 
1474 	for (i = 0; i < cluster_size; i++) {
1475 		struct page *rpage = rpages[i];
1476 
1477 		if (!rpage)
1478 			continue;
1479 
1480 		if (err || PageError(rpage))
1481 			goto clear_uptodate;
1482 
1483 		if (!verity || fsverity_verify_page(rpage)) {
1484 			SetPageUptodate(rpage);
1485 			goto unlock;
1486 		}
1487 clear_uptodate:
1488 		ClearPageUptodate(rpage);
1489 		ClearPageError(rpage);
1490 unlock:
1491 		unlock_page(rpage);
1492 	}
1493 }
1494