1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * f2fs compress support 4 * 5 * Copyright (c) 2019 Chao Yu <chao@kernel.org> 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/writeback.h> 11 #include <linux/backing-dev.h> 12 #include <linux/lzo.h> 13 #include <linux/lz4.h> 14 #include <linux/zstd.h> 15 16 #include "f2fs.h" 17 #include "node.h" 18 #include <trace/events/f2fs.h> 19 20 struct f2fs_compress_ops { 21 int (*init_compress_ctx)(struct compress_ctx *cc); 22 void (*destroy_compress_ctx)(struct compress_ctx *cc); 23 int (*compress_pages)(struct compress_ctx *cc); 24 int (*init_decompress_ctx)(struct decompress_io_ctx *dic); 25 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic); 26 int (*decompress_pages)(struct decompress_io_ctx *dic); 27 }; 28 29 static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index) 30 { 31 return index & (cc->cluster_size - 1); 32 } 33 34 static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index) 35 { 36 return index >> cc->log_cluster_size; 37 } 38 39 static pgoff_t start_idx_of_cluster(struct compress_ctx *cc) 40 { 41 return cc->cluster_idx << cc->log_cluster_size; 42 } 43 44 bool f2fs_is_compressed_page(struct page *page) 45 { 46 if (!PagePrivate(page)) 47 return false; 48 if (!page_private(page)) 49 return false; 50 if (IS_ATOMIC_WRITTEN_PAGE(page) || IS_DUMMY_WRITTEN_PAGE(page)) 51 return false; 52 /* 53 * page->private may be set with pid. 54 * pid_max is enough to check if it is traced. 55 */ 56 if (IS_IO_TRACED_PAGE(page)) 57 return false; 58 59 f2fs_bug_on(F2FS_M_SB(page->mapping), 60 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC); 61 return true; 62 } 63 64 static void f2fs_set_compressed_page(struct page *page, 65 struct inode *inode, pgoff_t index, void *data) 66 { 67 SetPagePrivate(page); 68 set_page_private(page, (unsigned long)data); 69 70 /* i_crypto_info and iv index */ 71 page->index = index; 72 page->mapping = inode->i_mapping; 73 } 74 75 static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock) 76 { 77 int i; 78 79 for (i = 0; i < len; i++) { 80 if (!cc->rpages[i]) 81 continue; 82 if (unlock) 83 unlock_page(cc->rpages[i]); 84 else 85 put_page(cc->rpages[i]); 86 } 87 } 88 89 static void f2fs_put_rpages(struct compress_ctx *cc) 90 { 91 f2fs_drop_rpages(cc, cc->cluster_size, false); 92 } 93 94 static void f2fs_unlock_rpages(struct compress_ctx *cc, int len) 95 { 96 f2fs_drop_rpages(cc, len, true); 97 } 98 99 static void f2fs_put_rpages_mapping(struct address_space *mapping, 100 pgoff_t start, int len) 101 { 102 int i; 103 104 for (i = 0; i < len; i++) { 105 struct page *page = find_get_page(mapping, start + i); 106 107 put_page(page); 108 put_page(page); 109 } 110 } 111 112 static void f2fs_put_rpages_wbc(struct compress_ctx *cc, 113 struct writeback_control *wbc, bool redirty, int unlock) 114 { 115 unsigned int i; 116 117 for (i = 0; i < cc->cluster_size; i++) { 118 if (!cc->rpages[i]) 119 continue; 120 if (redirty) 121 redirty_page_for_writepage(wbc, cc->rpages[i]); 122 f2fs_put_page(cc->rpages[i], unlock); 123 } 124 } 125 126 struct page *f2fs_compress_control_page(struct page *page) 127 { 128 return ((struct compress_io_ctx *)page_private(page))->rpages[0]; 129 } 130 131 int f2fs_init_compress_ctx(struct compress_ctx *cc) 132 { 133 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 134 135 if (cc->nr_rpages) 136 return 0; 137 138 cc->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) << 139 cc->log_cluster_size, GFP_NOFS); 140 return cc->rpages ? 0 : -ENOMEM; 141 } 142 143 void f2fs_destroy_compress_ctx(struct compress_ctx *cc) 144 { 145 kfree(cc->rpages); 146 cc->rpages = NULL; 147 cc->nr_rpages = 0; 148 cc->nr_cpages = 0; 149 cc->cluster_idx = NULL_CLUSTER; 150 } 151 152 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page) 153 { 154 unsigned int cluster_ofs; 155 156 if (!f2fs_cluster_can_merge_page(cc, page->index)) 157 f2fs_bug_on(F2FS_I_SB(cc->inode), 1); 158 159 cluster_ofs = offset_in_cluster(cc, page->index); 160 cc->rpages[cluster_ofs] = page; 161 cc->nr_rpages++; 162 cc->cluster_idx = cluster_idx(cc, page->index); 163 } 164 165 #ifdef CONFIG_F2FS_FS_LZO 166 static int lzo_init_compress_ctx(struct compress_ctx *cc) 167 { 168 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 169 LZO1X_MEM_COMPRESS, GFP_NOFS); 170 if (!cc->private) 171 return -ENOMEM; 172 173 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size); 174 return 0; 175 } 176 177 static void lzo_destroy_compress_ctx(struct compress_ctx *cc) 178 { 179 kvfree(cc->private); 180 cc->private = NULL; 181 } 182 183 static int lzo_compress_pages(struct compress_ctx *cc) 184 { 185 int ret; 186 187 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 188 &cc->clen, cc->private); 189 if (ret != LZO_E_OK) { 190 printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n", 191 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret); 192 return -EIO; 193 } 194 return 0; 195 } 196 197 static int lzo_decompress_pages(struct decompress_io_ctx *dic) 198 { 199 int ret; 200 201 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, 202 dic->rbuf, &dic->rlen); 203 if (ret != LZO_E_OK) { 204 printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n", 205 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret); 206 return -EIO; 207 } 208 209 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { 210 printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, " 211 "expected:%lu\n", KERN_ERR, 212 F2FS_I_SB(dic->inode)->sb->s_id, 213 dic->rlen, 214 PAGE_SIZE << dic->log_cluster_size); 215 return -EIO; 216 } 217 return 0; 218 } 219 220 static const struct f2fs_compress_ops f2fs_lzo_ops = { 221 .init_compress_ctx = lzo_init_compress_ctx, 222 .destroy_compress_ctx = lzo_destroy_compress_ctx, 223 .compress_pages = lzo_compress_pages, 224 .decompress_pages = lzo_decompress_pages, 225 }; 226 #endif 227 228 #ifdef CONFIG_F2FS_FS_LZ4 229 static int lz4_init_compress_ctx(struct compress_ctx *cc) 230 { 231 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 232 LZ4_MEM_COMPRESS, GFP_NOFS); 233 if (!cc->private) 234 return -ENOMEM; 235 236 /* 237 * we do not change cc->clen to LZ4_compressBound(inputsize) to 238 * adapt worst compress case, because lz4 compressor can handle 239 * output budget properly. 240 */ 241 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 242 return 0; 243 } 244 245 static void lz4_destroy_compress_ctx(struct compress_ctx *cc) 246 { 247 kvfree(cc->private); 248 cc->private = NULL; 249 } 250 251 static int lz4_compress_pages(struct compress_ctx *cc) 252 { 253 int len; 254 255 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, 256 cc->clen, cc->private); 257 if (!len) 258 return -EAGAIN; 259 260 cc->clen = len; 261 return 0; 262 } 263 264 static int lz4_decompress_pages(struct decompress_io_ctx *dic) 265 { 266 int ret; 267 268 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, 269 dic->clen, dic->rlen); 270 if (ret < 0) { 271 printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n", 272 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret); 273 return -EIO; 274 } 275 276 if (ret != PAGE_SIZE << dic->log_cluster_size) { 277 printk_ratelimited("%sF2FS-fs (%s): lz4 invalid rlen:%zu, " 278 "expected:%lu\n", KERN_ERR, 279 F2FS_I_SB(dic->inode)->sb->s_id, 280 dic->rlen, 281 PAGE_SIZE << dic->log_cluster_size); 282 return -EIO; 283 } 284 return 0; 285 } 286 287 static const struct f2fs_compress_ops f2fs_lz4_ops = { 288 .init_compress_ctx = lz4_init_compress_ctx, 289 .destroy_compress_ctx = lz4_destroy_compress_ctx, 290 .compress_pages = lz4_compress_pages, 291 .decompress_pages = lz4_decompress_pages, 292 }; 293 #endif 294 295 #ifdef CONFIG_F2FS_FS_ZSTD 296 #define F2FS_ZSTD_DEFAULT_CLEVEL 1 297 298 static int zstd_init_compress_ctx(struct compress_ctx *cc) 299 { 300 ZSTD_parameters params; 301 ZSTD_CStream *stream; 302 void *workspace; 303 unsigned int workspace_size; 304 305 params = ZSTD_getParams(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen, 0); 306 workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams); 307 308 workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 309 workspace_size, GFP_NOFS); 310 if (!workspace) 311 return -ENOMEM; 312 313 stream = ZSTD_initCStream(params, 0, workspace, workspace_size); 314 if (!stream) { 315 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initCStream failed\n", 316 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 317 __func__); 318 kvfree(workspace); 319 return -EIO; 320 } 321 322 cc->private = workspace; 323 cc->private2 = stream; 324 325 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 326 return 0; 327 } 328 329 static void zstd_destroy_compress_ctx(struct compress_ctx *cc) 330 { 331 kvfree(cc->private); 332 cc->private = NULL; 333 cc->private2 = NULL; 334 } 335 336 static int zstd_compress_pages(struct compress_ctx *cc) 337 { 338 ZSTD_CStream *stream = cc->private2; 339 ZSTD_inBuffer inbuf; 340 ZSTD_outBuffer outbuf; 341 int src_size = cc->rlen; 342 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE; 343 int ret; 344 345 inbuf.pos = 0; 346 inbuf.src = cc->rbuf; 347 inbuf.size = src_size; 348 349 outbuf.pos = 0; 350 outbuf.dst = cc->cbuf->cdata; 351 outbuf.size = dst_size; 352 353 ret = ZSTD_compressStream(stream, &outbuf, &inbuf); 354 if (ZSTD_isError(ret)) { 355 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n", 356 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 357 __func__, ZSTD_getErrorCode(ret)); 358 return -EIO; 359 } 360 361 ret = ZSTD_endStream(stream, &outbuf); 362 if (ZSTD_isError(ret)) { 363 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_endStream returned %d\n", 364 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 365 __func__, ZSTD_getErrorCode(ret)); 366 return -EIO; 367 } 368 369 /* 370 * there is compressed data remained in intermediate buffer due to 371 * no more space in cbuf.cdata 372 */ 373 if (ret) 374 return -EAGAIN; 375 376 cc->clen = outbuf.pos; 377 return 0; 378 } 379 380 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) 381 { 382 ZSTD_DStream *stream; 383 void *workspace; 384 unsigned int workspace_size; 385 386 workspace_size = ZSTD_DStreamWorkspaceBound(MAX_COMPRESS_WINDOW_SIZE); 387 388 workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode), 389 workspace_size, GFP_NOFS); 390 if (!workspace) 391 return -ENOMEM; 392 393 stream = ZSTD_initDStream(MAX_COMPRESS_WINDOW_SIZE, 394 workspace, workspace_size); 395 if (!stream) { 396 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initDStream failed\n", 397 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, 398 __func__); 399 kvfree(workspace); 400 return -EIO; 401 } 402 403 dic->private = workspace; 404 dic->private2 = stream; 405 406 return 0; 407 } 408 409 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) 410 { 411 kvfree(dic->private); 412 dic->private = NULL; 413 dic->private2 = NULL; 414 } 415 416 static int zstd_decompress_pages(struct decompress_io_ctx *dic) 417 { 418 ZSTD_DStream *stream = dic->private2; 419 ZSTD_inBuffer inbuf; 420 ZSTD_outBuffer outbuf; 421 int ret; 422 423 inbuf.pos = 0; 424 inbuf.src = dic->cbuf->cdata; 425 inbuf.size = dic->clen; 426 427 outbuf.pos = 0; 428 outbuf.dst = dic->rbuf; 429 outbuf.size = dic->rlen; 430 431 ret = ZSTD_decompressStream(stream, &outbuf, &inbuf); 432 if (ZSTD_isError(ret)) { 433 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n", 434 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, 435 __func__, ZSTD_getErrorCode(ret)); 436 return -EIO; 437 } 438 439 if (dic->rlen != outbuf.pos) { 440 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD invalid rlen:%zu, " 441 "expected:%lu\n", KERN_ERR, 442 F2FS_I_SB(dic->inode)->sb->s_id, 443 __func__, dic->rlen, 444 PAGE_SIZE << dic->log_cluster_size); 445 return -EIO; 446 } 447 448 return 0; 449 } 450 451 static const struct f2fs_compress_ops f2fs_zstd_ops = { 452 .init_compress_ctx = zstd_init_compress_ctx, 453 .destroy_compress_ctx = zstd_destroy_compress_ctx, 454 .compress_pages = zstd_compress_pages, 455 .init_decompress_ctx = zstd_init_decompress_ctx, 456 .destroy_decompress_ctx = zstd_destroy_decompress_ctx, 457 .decompress_pages = zstd_decompress_pages, 458 }; 459 #endif 460 461 #ifdef CONFIG_F2FS_FS_LZO 462 #ifdef CONFIG_F2FS_FS_LZORLE 463 static int lzorle_compress_pages(struct compress_ctx *cc) 464 { 465 int ret; 466 467 ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 468 &cc->clen, cc->private); 469 if (ret != LZO_E_OK) { 470 printk_ratelimited("%sF2FS-fs (%s): lzo-rle compress failed, ret:%d\n", 471 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret); 472 return -EIO; 473 } 474 return 0; 475 } 476 477 static const struct f2fs_compress_ops f2fs_lzorle_ops = { 478 .init_compress_ctx = lzo_init_compress_ctx, 479 .destroy_compress_ctx = lzo_destroy_compress_ctx, 480 .compress_pages = lzorle_compress_pages, 481 .decompress_pages = lzo_decompress_pages, 482 }; 483 #endif 484 #endif 485 486 static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = { 487 #ifdef CONFIG_F2FS_FS_LZO 488 &f2fs_lzo_ops, 489 #else 490 NULL, 491 #endif 492 #ifdef CONFIG_F2FS_FS_LZ4 493 &f2fs_lz4_ops, 494 #else 495 NULL, 496 #endif 497 #ifdef CONFIG_F2FS_FS_ZSTD 498 &f2fs_zstd_ops, 499 #else 500 NULL, 501 #endif 502 #if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE) 503 &f2fs_lzorle_ops, 504 #else 505 NULL, 506 #endif 507 }; 508 509 bool f2fs_is_compress_backend_ready(struct inode *inode) 510 { 511 if (!f2fs_compressed_file(inode)) 512 return true; 513 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm]; 514 } 515 516 static mempool_t *compress_page_pool; 517 static int num_compress_pages = 512; 518 module_param(num_compress_pages, uint, 0444); 519 MODULE_PARM_DESC(num_compress_pages, 520 "Number of intermediate compress pages to preallocate"); 521 522 int f2fs_init_compress_mempool(void) 523 { 524 compress_page_pool = mempool_create_page_pool(num_compress_pages, 0); 525 if (!compress_page_pool) 526 return -ENOMEM; 527 528 return 0; 529 } 530 531 void f2fs_destroy_compress_mempool(void) 532 { 533 mempool_destroy(compress_page_pool); 534 } 535 536 static struct page *f2fs_compress_alloc_page(void) 537 { 538 struct page *page; 539 540 page = mempool_alloc(compress_page_pool, GFP_NOFS); 541 lock_page(page); 542 543 return page; 544 } 545 546 static void f2fs_compress_free_page(struct page *page) 547 { 548 if (!page) 549 return; 550 set_page_private(page, (unsigned long)NULL); 551 ClearPagePrivate(page); 552 page->mapping = NULL; 553 unlock_page(page); 554 mempool_free(page, compress_page_pool); 555 } 556 557 static int f2fs_compress_pages(struct compress_ctx *cc) 558 { 559 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 560 struct f2fs_inode_info *fi = F2FS_I(cc->inode); 561 const struct f2fs_compress_ops *cops = 562 f2fs_cops[fi->i_compress_algorithm]; 563 unsigned int max_len, nr_cpages; 564 int i, ret; 565 566 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx, 567 cc->cluster_size, fi->i_compress_algorithm); 568 569 if (cops->init_compress_ctx) { 570 ret = cops->init_compress_ctx(cc); 571 if (ret) 572 goto out; 573 } 574 575 max_len = COMPRESS_HEADER_SIZE + cc->clen; 576 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE); 577 578 cc->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) * 579 cc->nr_cpages, GFP_NOFS); 580 if (!cc->cpages) { 581 ret = -ENOMEM; 582 goto destroy_compress_ctx; 583 } 584 585 for (i = 0; i < cc->nr_cpages; i++) { 586 cc->cpages[i] = f2fs_compress_alloc_page(); 587 if (!cc->cpages[i]) { 588 ret = -ENOMEM; 589 goto out_free_cpages; 590 } 591 } 592 593 cc->rbuf = vmap(cc->rpages, cc->cluster_size, VM_MAP, PAGE_KERNEL_RO); 594 if (!cc->rbuf) { 595 ret = -ENOMEM; 596 goto out_free_cpages; 597 } 598 599 cc->cbuf = vmap(cc->cpages, cc->nr_cpages, VM_MAP, PAGE_KERNEL); 600 if (!cc->cbuf) { 601 ret = -ENOMEM; 602 goto out_vunmap_rbuf; 603 } 604 605 ret = cops->compress_pages(cc); 606 if (ret) 607 goto out_vunmap_cbuf; 608 609 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE; 610 611 if (cc->clen > max_len) { 612 ret = -EAGAIN; 613 goto out_vunmap_cbuf; 614 } 615 616 cc->cbuf->clen = cpu_to_le32(cc->clen); 617 618 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++) 619 cc->cbuf->reserved[i] = cpu_to_le32(0); 620 621 nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE); 622 623 /* zero out any unused part of the last page */ 624 memset(&cc->cbuf->cdata[cc->clen], 0, 625 (nr_cpages * PAGE_SIZE) - (cc->clen + COMPRESS_HEADER_SIZE)); 626 627 vunmap(cc->cbuf); 628 vunmap(cc->rbuf); 629 630 for (i = nr_cpages; i < cc->nr_cpages; i++) { 631 f2fs_compress_free_page(cc->cpages[i]); 632 cc->cpages[i] = NULL; 633 } 634 635 if (cops->destroy_compress_ctx) 636 cops->destroy_compress_ctx(cc); 637 638 cc->nr_cpages = nr_cpages; 639 640 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 641 cc->clen, ret); 642 return 0; 643 644 out_vunmap_cbuf: 645 vunmap(cc->cbuf); 646 out_vunmap_rbuf: 647 vunmap(cc->rbuf); 648 out_free_cpages: 649 for (i = 0; i < cc->nr_cpages; i++) { 650 if (cc->cpages[i]) 651 f2fs_compress_free_page(cc->cpages[i]); 652 } 653 kfree(cc->cpages); 654 cc->cpages = NULL; 655 destroy_compress_ctx: 656 if (cops->destroy_compress_ctx) 657 cops->destroy_compress_ctx(cc); 658 out: 659 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 660 cc->clen, ret); 661 return ret; 662 } 663 664 void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity) 665 { 666 struct decompress_io_ctx *dic = 667 (struct decompress_io_ctx *)page_private(page); 668 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); 669 struct f2fs_inode_info *fi= F2FS_I(dic->inode); 670 const struct f2fs_compress_ops *cops = 671 f2fs_cops[fi->i_compress_algorithm]; 672 int ret; 673 int i; 674 675 dec_page_count(sbi, F2FS_RD_DATA); 676 677 if (bio->bi_status || PageError(page)) 678 dic->failed = true; 679 680 if (refcount_dec_not_one(&dic->ref)) 681 return; 682 683 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx, 684 dic->cluster_size, fi->i_compress_algorithm); 685 686 /* submit partial compressed pages */ 687 if (dic->failed) { 688 ret = -EIO; 689 goto out_free_dic; 690 } 691 692 dic->tpages = f2fs_kzalloc(sbi, sizeof(struct page *) * 693 dic->cluster_size, GFP_NOFS); 694 if (!dic->tpages) { 695 ret = -ENOMEM; 696 goto out_free_dic; 697 } 698 699 for (i = 0; i < dic->cluster_size; i++) { 700 if (dic->rpages[i]) { 701 dic->tpages[i] = dic->rpages[i]; 702 continue; 703 } 704 705 dic->tpages[i] = f2fs_compress_alloc_page(); 706 if (!dic->tpages[i]) { 707 ret = -ENOMEM; 708 goto out_free_dic; 709 } 710 } 711 712 if (cops->init_decompress_ctx) { 713 ret = cops->init_decompress_ctx(dic); 714 if (ret) 715 goto out_free_dic; 716 } 717 718 dic->rbuf = vmap(dic->tpages, dic->cluster_size, VM_MAP, PAGE_KERNEL); 719 if (!dic->rbuf) { 720 ret = -ENOMEM; 721 goto destroy_decompress_ctx; 722 } 723 724 dic->cbuf = vmap(dic->cpages, dic->nr_cpages, VM_MAP, PAGE_KERNEL_RO); 725 if (!dic->cbuf) { 726 ret = -ENOMEM; 727 goto out_vunmap_rbuf; 728 } 729 730 dic->clen = le32_to_cpu(dic->cbuf->clen); 731 dic->rlen = PAGE_SIZE << dic->log_cluster_size; 732 733 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) { 734 ret = -EFSCORRUPTED; 735 goto out_vunmap_cbuf; 736 } 737 738 ret = cops->decompress_pages(dic); 739 740 out_vunmap_cbuf: 741 vunmap(dic->cbuf); 742 out_vunmap_rbuf: 743 vunmap(dic->rbuf); 744 destroy_decompress_ctx: 745 if (cops->destroy_decompress_ctx) 746 cops->destroy_decompress_ctx(dic); 747 out_free_dic: 748 if (verity) 749 refcount_set(&dic->ref, dic->nr_cpages); 750 if (!verity) 751 f2fs_decompress_end_io(dic->rpages, dic->cluster_size, 752 ret, false); 753 754 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx, 755 dic->clen, ret); 756 if (!verity) 757 f2fs_free_dic(dic); 758 } 759 760 static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index) 761 { 762 if (cc->cluster_idx == NULL_CLUSTER) 763 return true; 764 return cc->cluster_idx == cluster_idx(cc, index); 765 } 766 767 bool f2fs_cluster_is_empty(struct compress_ctx *cc) 768 { 769 return cc->nr_rpages == 0; 770 } 771 772 static bool f2fs_cluster_is_full(struct compress_ctx *cc) 773 { 774 return cc->cluster_size == cc->nr_rpages; 775 } 776 777 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) 778 { 779 if (f2fs_cluster_is_empty(cc)) 780 return true; 781 return is_page_in_cluster(cc, index); 782 } 783 784 static bool __cluster_may_compress(struct compress_ctx *cc) 785 { 786 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 787 loff_t i_size = i_size_read(cc->inode); 788 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE); 789 int i; 790 791 for (i = 0; i < cc->cluster_size; i++) { 792 struct page *page = cc->rpages[i]; 793 794 f2fs_bug_on(sbi, !page); 795 796 if (unlikely(f2fs_cp_error(sbi))) 797 return false; 798 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 799 return false; 800 801 /* beyond EOF */ 802 if (page->index >= nr_pages) 803 return false; 804 } 805 return true; 806 } 807 808 static int __f2fs_cluster_blocks(struct compress_ctx *cc, bool compr) 809 { 810 struct dnode_of_data dn; 811 int ret; 812 813 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 814 ret = f2fs_get_dnode_of_data(&dn, start_idx_of_cluster(cc), 815 LOOKUP_NODE); 816 if (ret) { 817 if (ret == -ENOENT) 818 ret = 0; 819 goto fail; 820 } 821 822 if (dn.data_blkaddr == COMPRESS_ADDR) { 823 int i; 824 825 ret = 1; 826 for (i = 1; i < cc->cluster_size; i++) { 827 block_t blkaddr; 828 829 blkaddr = data_blkaddr(dn.inode, 830 dn.node_page, dn.ofs_in_node + i); 831 if (compr) { 832 if (__is_valid_data_blkaddr(blkaddr)) 833 ret++; 834 } else { 835 if (blkaddr != NULL_ADDR) 836 ret++; 837 } 838 } 839 } 840 fail: 841 f2fs_put_dnode(&dn); 842 return ret; 843 } 844 845 /* return # of compressed blocks in compressed cluster */ 846 static int f2fs_compressed_blocks(struct compress_ctx *cc) 847 { 848 return __f2fs_cluster_blocks(cc, true); 849 } 850 851 /* return # of valid blocks in compressed cluster */ 852 static int f2fs_cluster_blocks(struct compress_ctx *cc) 853 { 854 return __f2fs_cluster_blocks(cc, false); 855 } 856 857 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index) 858 { 859 struct compress_ctx cc = { 860 .inode = inode, 861 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 862 .cluster_size = F2FS_I(inode)->i_cluster_size, 863 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, 864 }; 865 866 return f2fs_cluster_blocks(&cc); 867 } 868 869 static bool cluster_may_compress(struct compress_ctx *cc) 870 { 871 if (!f2fs_compressed_file(cc->inode)) 872 return false; 873 if (f2fs_is_atomic_file(cc->inode)) 874 return false; 875 if (f2fs_is_mmap_file(cc->inode)) 876 return false; 877 if (!f2fs_cluster_is_full(cc)) 878 return false; 879 if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode)))) 880 return false; 881 return __cluster_may_compress(cc); 882 } 883 884 static void set_cluster_writeback(struct compress_ctx *cc) 885 { 886 int i; 887 888 for (i = 0; i < cc->cluster_size; i++) { 889 if (cc->rpages[i]) 890 set_page_writeback(cc->rpages[i]); 891 } 892 } 893 894 static void set_cluster_dirty(struct compress_ctx *cc) 895 { 896 int i; 897 898 for (i = 0; i < cc->cluster_size; i++) 899 if (cc->rpages[i]) 900 set_page_dirty(cc->rpages[i]); 901 } 902 903 static int prepare_compress_overwrite(struct compress_ctx *cc, 904 struct page **pagep, pgoff_t index, void **fsdata) 905 { 906 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 907 struct address_space *mapping = cc->inode->i_mapping; 908 struct page *page; 909 struct dnode_of_data dn; 910 sector_t last_block_in_bio; 911 unsigned fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT; 912 pgoff_t start_idx = start_idx_of_cluster(cc); 913 int i, ret; 914 bool prealloc; 915 916 retry: 917 ret = f2fs_cluster_blocks(cc); 918 if (ret <= 0) 919 return ret; 920 921 /* compressed case */ 922 prealloc = (ret < cc->cluster_size); 923 924 ret = f2fs_init_compress_ctx(cc); 925 if (ret) 926 return ret; 927 928 /* keep page reference to avoid page reclaim */ 929 for (i = 0; i < cc->cluster_size; i++) { 930 page = f2fs_pagecache_get_page(mapping, start_idx + i, 931 fgp_flag, GFP_NOFS); 932 if (!page) { 933 ret = -ENOMEM; 934 goto unlock_pages; 935 } 936 937 if (PageUptodate(page)) 938 unlock_page(page); 939 else 940 f2fs_compress_ctx_add_page(cc, page); 941 } 942 943 if (!f2fs_cluster_is_empty(cc)) { 944 struct bio *bio = NULL; 945 946 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size, 947 &last_block_in_bio, false, true); 948 f2fs_destroy_compress_ctx(cc); 949 if (ret) 950 goto release_pages; 951 if (bio) 952 f2fs_submit_bio(sbi, bio, DATA); 953 954 ret = f2fs_init_compress_ctx(cc); 955 if (ret) 956 goto release_pages; 957 } 958 959 for (i = 0; i < cc->cluster_size; i++) { 960 f2fs_bug_on(sbi, cc->rpages[i]); 961 962 page = find_lock_page(mapping, start_idx + i); 963 f2fs_bug_on(sbi, !page); 964 965 f2fs_wait_on_page_writeback(page, DATA, true, true); 966 967 f2fs_compress_ctx_add_page(cc, page); 968 f2fs_put_page(page, 0); 969 970 if (!PageUptodate(page)) { 971 f2fs_unlock_rpages(cc, i + 1); 972 f2fs_put_rpages_mapping(mapping, start_idx, 973 cc->cluster_size); 974 f2fs_destroy_compress_ctx(cc); 975 goto retry; 976 } 977 } 978 979 if (prealloc) { 980 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true); 981 982 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 983 984 for (i = cc->cluster_size - 1; i > 0; i--) { 985 ret = f2fs_get_block(&dn, start_idx + i); 986 if (ret) { 987 i = cc->cluster_size; 988 break; 989 } 990 991 if (dn.data_blkaddr != NEW_ADDR) 992 break; 993 } 994 995 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false); 996 } 997 998 if (likely(!ret)) { 999 *fsdata = cc->rpages; 1000 *pagep = cc->rpages[offset_in_cluster(cc, index)]; 1001 return cc->cluster_size; 1002 } 1003 1004 unlock_pages: 1005 f2fs_unlock_rpages(cc, i); 1006 release_pages: 1007 f2fs_put_rpages_mapping(mapping, start_idx, i); 1008 f2fs_destroy_compress_ctx(cc); 1009 return ret; 1010 } 1011 1012 int f2fs_prepare_compress_overwrite(struct inode *inode, 1013 struct page **pagep, pgoff_t index, void **fsdata) 1014 { 1015 struct compress_ctx cc = { 1016 .inode = inode, 1017 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1018 .cluster_size = F2FS_I(inode)->i_cluster_size, 1019 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, 1020 .rpages = NULL, 1021 .nr_rpages = 0, 1022 }; 1023 1024 return prepare_compress_overwrite(&cc, pagep, index, fsdata); 1025 } 1026 1027 bool f2fs_compress_write_end(struct inode *inode, void *fsdata, 1028 pgoff_t index, unsigned copied) 1029 1030 { 1031 struct compress_ctx cc = { 1032 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1033 .cluster_size = F2FS_I(inode)->i_cluster_size, 1034 .rpages = fsdata, 1035 }; 1036 bool first_index = (index == cc.rpages[0]->index); 1037 1038 if (copied) 1039 set_cluster_dirty(&cc); 1040 1041 f2fs_put_rpages_wbc(&cc, NULL, false, 1); 1042 f2fs_destroy_compress_ctx(&cc); 1043 1044 return first_index; 1045 } 1046 1047 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) 1048 { 1049 void *fsdata = NULL; 1050 struct page *pagep; 1051 int log_cluster_size = F2FS_I(inode)->i_log_cluster_size; 1052 pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) << 1053 log_cluster_size; 1054 int err; 1055 1056 err = f2fs_is_compressed_cluster(inode, start_idx); 1057 if (err < 0) 1058 return err; 1059 1060 /* truncate normal cluster */ 1061 if (!err) 1062 return f2fs_do_truncate_blocks(inode, from, lock); 1063 1064 /* truncate compressed cluster */ 1065 err = f2fs_prepare_compress_overwrite(inode, &pagep, 1066 start_idx, &fsdata); 1067 1068 /* should not be a normal cluster */ 1069 f2fs_bug_on(F2FS_I_SB(inode), err == 0); 1070 1071 if (err <= 0) 1072 return err; 1073 1074 if (err > 0) { 1075 struct page **rpages = fsdata; 1076 int cluster_size = F2FS_I(inode)->i_cluster_size; 1077 int i; 1078 1079 for (i = cluster_size - 1; i >= 0; i--) { 1080 loff_t start = rpages[i]->index << PAGE_SHIFT; 1081 1082 if (from <= start) { 1083 zero_user_segment(rpages[i], 0, PAGE_SIZE); 1084 } else { 1085 zero_user_segment(rpages[i], from - start, 1086 PAGE_SIZE); 1087 break; 1088 } 1089 } 1090 1091 f2fs_compress_write_end(inode, fsdata, start_idx, true); 1092 } 1093 return 0; 1094 } 1095 1096 static int f2fs_write_compressed_pages(struct compress_ctx *cc, 1097 int *submitted, 1098 struct writeback_control *wbc, 1099 enum iostat_type io_type) 1100 { 1101 struct inode *inode = cc->inode; 1102 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1103 struct f2fs_inode_info *fi = F2FS_I(inode); 1104 struct f2fs_io_info fio = { 1105 .sbi = sbi, 1106 .ino = cc->inode->i_ino, 1107 .type = DATA, 1108 .op = REQ_OP_WRITE, 1109 .op_flags = wbc_to_write_flags(wbc), 1110 .old_blkaddr = NEW_ADDR, 1111 .page = NULL, 1112 .encrypted_page = NULL, 1113 .compressed_page = NULL, 1114 .submitted = false, 1115 .io_type = io_type, 1116 .io_wbc = wbc, 1117 .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode), 1118 }; 1119 struct dnode_of_data dn; 1120 struct node_info ni; 1121 struct compress_io_ctx *cic; 1122 pgoff_t start_idx = start_idx_of_cluster(cc); 1123 unsigned int last_index = cc->cluster_size - 1; 1124 loff_t psize; 1125 int i, err; 1126 1127 if (IS_NOQUOTA(inode)) { 1128 /* 1129 * We need to wait for node_write to avoid block allocation during 1130 * checkpoint. This can only happen to quota writes which can cause 1131 * the below discard race condition. 1132 */ 1133 down_read(&sbi->node_write); 1134 } else if (!f2fs_trylock_op(sbi)) { 1135 return -EAGAIN; 1136 } 1137 1138 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 1139 1140 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 1141 if (err) 1142 goto out_unlock_op; 1143 1144 for (i = 0; i < cc->cluster_size; i++) { 1145 if (data_blkaddr(dn.inode, dn.node_page, 1146 dn.ofs_in_node + i) == NULL_ADDR) 1147 goto out_put_dnode; 1148 } 1149 1150 psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT; 1151 1152 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni); 1153 if (err) 1154 goto out_put_dnode; 1155 1156 fio.version = ni.version; 1157 1158 cic = f2fs_kzalloc(sbi, sizeof(struct compress_io_ctx), GFP_NOFS); 1159 if (!cic) 1160 goto out_put_dnode; 1161 1162 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1163 cic->inode = inode; 1164 refcount_set(&cic->ref, cc->nr_cpages); 1165 cic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) << 1166 cc->log_cluster_size, GFP_NOFS); 1167 if (!cic->rpages) 1168 goto out_put_cic; 1169 1170 cic->nr_rpages = cc->cluster_size; 1171 1172 for (i = 0; i < cc->nr_cpages; i++) { 1173 f2fs_set_compressed_page(cc->cpages[i], inode, 1174 cc->rpages[i + 1]->index, cic); 1175 fio.compressed_page = cc->cpages[i]; 1176 1177 fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_page, 1178 dn.ofs_in_node + i + 1); 1179 1180 /* wait for GCed page writeback via META_MAPPING */ 1181 f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); 1182 1183 if (fio.encrypted) { 1184 fio.page = cc->rpages[i + 1]; 1185 err = f2fs_encrypt_one_page(&fio); 1186 if (err) 1187 goto out_destroy_crypt; 1188 cc->cpages[i] = fio.encrypted_page; 1189 } 1190 } 1191 1192 set_cluster_writeback(cc); 1193 1194 for (i = 0; i < cc->cluster_size; i++) 1195 cic->rpages[i] = cc->rpages[i]; 1196 1197 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) { 1198 block_t blkaddr; 1199 1200 blkaddr = f2fs_data_blkaddr(&dn); 1201 fio.page = cc->rpages[i]; 1202 fio.old_blkaddr = blkaddr; 1203 1204 /* cluster header */ 1205 if (i == 0) { 1206 if (blkaddr == COMPRESS_ADDR) 1207 fio.compr_blocks++; 1208 if (__is_valid_data_blkaddr(blkaddr)) 1209 f2fs_invalidate_blocks(sbi, blkaddr); 1210 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR); 1211 goto unlock_continue; 1212 } 1213 1214 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr)) 1215 fio.compr_blocks++; 1216 1217 if (i > cc->nr_cpages) { 1218 if (__is_valid_data_blkaddr(blkaddr)) { 1219 f2fs_invalidate_blocks(sbi, blkaddr); 1220 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 1221 } 1222 goto unlock_continue; 1223 } 1224 1225 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR); 1226 1227 if (fio.encrypted) 1228 fio.encrypted_page = cc->cpages[i - 1]; 1229 else 1230 fio.compressed_page = cc->cpages[i - 1]; 1231 1232 cc->cpages[i - 1] = NULL; 1233 f2fs_outplace_write_data(&dn, &fio); 1234 (*submitted)++; 1235 unlock_continue: 1236 inode_dec_dirty_pages(cc->inode); 1237 unlock_page(fio.page); 1238 } 1239 1240 if (fio.compr_blocks) 1241 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false); 1242 f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true); 1243 1244 set_inode_flag(cc->inode, FI_APPEND_WRITE); 1245 if (cc->cluster_idx == 0) 1246 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN); 1247 1248 f2fs_put_dnode(&dn); 1249 if (IS_NOQUOTA(inode)) 1250 up_read(&sbi->node_write); 1251 else 1252 f2fs_unlock_op(sbi); 1253 1254 spin_lock(&fi->i_size_lock); 1255 if (fi->last_disk_size < psize) 1256 fi->last_disk_size = psize; 1257 spin_unlock(&fi->i_size_lock); 1258 1259 f2fs_put_rpages(cc); 1260 f2fs_destroy_compress_ctx(cc); 1261 return 0; 1262 1263 out_destroy_crypt: 1264 kfree(cic->rpages); 1265 1266 for (--i; i >= 0; i--) 1267 fscrypt_finalize_bounce_page(&cc->cpages[i]); 1268 for (i = 0; i < cc->nr_cpages; i++) { 1269 if (!cc->cpages[i]) 1270 continue; 1271 f2fs_put_page(cc->cpages[i], 1); 1272 } 1273 out_put_cic: 1274 kfree(cic); 1275 out_put_dnode: 1276 f2fs_put_dnode(&dn); 1277 out_unlock_op: 1278 if (IS_NOQUOTA(inode)) 1279 up_read(&sbi->node_write); 1280 else 1281 f2fs_unlock_op(sbi); 1282 return -EAGAIN; 1283 } 1284 1285 void f2fs_compress_write_end_io(struct bio *bio, struct page *page) 1286 { 1287 struct f2fs_sb_info *sbi = bio->bi_private; 1288 struct compress_io_ctx *cic = 1289 (struct compress_io_ctx *)page_private(page); 1290 int i; 1291 1292 if (unlikely(bio->bi_status)) 1293 mapping_set_error(cic->inode->i_mapping, -EIO); 1294 1295 f2fs_compress_free_page(page); 1296 1297 dec_page_count(sbi, F2FS_WB_DATA); 1298 1299 if (refcount_dec_not_one(&cic->ref)) 1300 return; 1301 1302 for (i = 0; i < cic->nr_rpages; i++) { 1303 WARN_ON(!cic->rpages[i]); 1304 clear_cold_data(cic->rpages[i]); 1305 end_page_writeback(cic->rpages[i]); 1306 } 1307 1308 kfree(cic->rpages); 1309 kfree(cic); 1310 } 1311 1312 static int f2fs_write_raw_pages(struct compress_ctx *cc, 1313 int *submitted, 1314 struct writeback_control *wbc, 1315 enum iostat_type io_type) 1316 { 1317 struct address_space *mapping = cc->inode->i_mapping; 1318 int _submitted, compr_blocks, ret; 1319 int i = -1, err = 0; 1320 1321 compr_blocks = f2fs_compressed_blocks(cc); 1322 if (compr_blocks < 0) { 1323 err = compr_blocks; 1324 goto out_err; 1325 } 1326 1327 for (i = 0; i < cc->cluster_size; i++) { 1328 if (!cc->rpages[i]) 1329 continue; 1330 retry_write: 1331 if (cc->rpages[i]->mapping != mapping) { 1332 unlock_page(cc->rpages[i]); 1333 continue; 1334 } 1335 1336 BUG_ON(!PageLocked(cc->rpages[i])); 1337 1338 ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted, 1339 NULL, NULL, wbc, io_type, 1340 compr_blocks); 1341 if (ret) { 1342 if (ret == AOP_WRITEPAGE_ACTIVATE) { 1343 unlock_page(cc->rpages[i]); 1344 ret = 0; 1345 } else if (ret == -EAGAIN) { 1346 /* 1347 * for quota file, just redirty left pages to 1348 * avoid deadlock caused by cluster update race 1349 * from foreground operation. 1350 */ 1351 if (IS_NOQUOTA(cc->inode)) { 1352 err = 0; 1353 goto out_err; 1354 } 1355 ret = 0; 1356 cond_resched(); 1357 congestion_wait(BLK_RW_ASYNC, 1358 DEFAULT_IO_TIMEOUT); 1359 lock_page(cc->rpages[i]); 1360 1361 if (!PageDirty(cc->rpages[i])) { 1362 unlock_page(cc->rpages[i]); 1363 continue; 1364 } 1365 1366 clear_page_dirty_for_io(cc->rpages[i]); 1367 goto retry_write; 1368 } 1369 err = ret; 1370 goto out_err; 1371 } 1372 1373 *submitted += _submitted; 1374 } 1375 return 0; 1376 out_err: 1377 for (++i; i < cc->cluster_size; i++) { 1378 if (!cc->rpages[i]) 1379 continue; 1380 redirty_page_for_writepage(wbc, cc->rpages[i]); 1381 unlock_page(cc->rpages[i]); 1382 } 1383 return err; 1384 } 1385 1386 int f2fs_write_multi_pages(struct compress_ctx *cc, 1387 int *submitted, 1388 struct writeback_control *wbc, 1389 enum iostat_type io_type) 1390 { 1391 struct f2fs_inode_info *fi = F2FS_I(cc->inode); 1392 const struct f2fs_compress_ops *cops = 1393 f2fs_cops[fi->i_compress_algorithm]; 1394 int err; 1395 1396 *submitted = 0; 1397 if (cluster_may_compress(cc)) { 1398 err = f2fs_compress_pages(cc); 1399 if (err == -EAGAIN) { 1400 goto write; 1401 } else if (err) { 1402 f2fs_put_rpages_wbc(cc, wbc, true, 1); 1403 goto destroy_out; 1404 } 1405 1406 err = f2fs_write_compressed_pages(cc, submitted, 1407 wbc, io_type); 1408 cops->destroy_compress_ctx(cc); 1409 kfree(cc->cpages); 1410 cc->cpages = NULL; 1411 if (!err) 1412 return 0; 1413 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN); 1414 } 1415 write: 1416 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted); 1417 1418 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type); 1419 f2fs_put_rpages_wbc(cc, wbc, false, 0); 1420 destroy_out: 1421 f2fs_destroy_compress_ctx(cc); 1422 return err; 1423 } 1424 1425 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) 1426 { 1427 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1428 struct decompress_io_ctx *dic; 1429 pgoff_t start_idx = start_idx_of_cluster(cc); 1430 int i; 1431 1432 dic = f2fs_kzalloc(sbi, sizeof(struct decompress_io_ctx), GFP_NOFS); 1433 if (!dic) 1434 return ERR_PTR(-ENOMEM); 1435 1436 dic->rpages = f2fs_kzalloc(sbi, sizeof(struct page *) << 1437 cc->log_cluster_size, GFP_NOFS); 1438 if (!dic->rpages) { 1439 kfree(dic); 1440 return ERR_PTR(-ENOMEM); 1441 } 1442 1443 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1444 dic->inode = cc->inode; 1445 refcount_set(&dic->ref, cc->nr_cpages); 1446 dic->cluster_idx = cc->cluster_idx; 1447 dic->cluster_size = cc->cluster_size; 1448 dic->log_cluster_size = cc->log_cluster_size; 1449 dic->nr_cpages = cc->nr_cpages; 1450 dic->failed = false; 1451 1452 for (i = 0; i < dic->cluster_size; i++) 1453 dic->rpages[i] = cc->rpages[i]; 1454 dic->nr_rpages = cc->cluster_size; 1455 1456 dic->cpages = f2fs_kzalloc(sbi, sizeof(struct page *) * 1457 dic->nr_cpages, GFP_NOFS); 1458 if (!dic->cpages) 1459 goto out_free; 1460 1461 for (i = 0; i < dic->nr_cpages; i++) { 1462 struct page *page; 1463 1464 page = f2fs_compress_alloc_page(); 1465 if (!page) 1466 goto out_free; 1467 1468 f2fs_set_compressed_page(page, cc->inode, 1469 start_idx + i + 1, dic); 1470 dic->cpages[i] = page; 1471 } 1472 1473 return dic; 1474 1475 out_free: 1476 f2fs_free_dic(dic); 1477 return ERR_PTR(-ENOMEM); 1478 } 1479 1480 void f2fs_free_dic(struct decompress_io_ctx *dic) 1481 { 1482 int i; 1483 1484 if (dic->tpages) { 1485 for (i = 0; i < dic->cluster_size; i++) { 1486 if (dic->rpages[i]) 1487 continue; 1488 if (!dic->tpages[i]) 1489 continue; 1490 f2fs_compress_free_page(dic->tpages[i]); 1491 } 1492 kfree(dic->tpages); 1493 } 1494 1495 if (dic->cpages) { 1496 for (i = 0; i < dic->nr_cpages; i++) { 1497 if (!dic->cpages[i]) 1498 continue; 1499 f2fs_compress_free_page(dic->cpages[i]); 1500 } 1501 kfree(dic->cpages); 1502 } 1503 1504 kfree(dic->rpages); 1505 kfree(dic); 1506 } 1507 1508 void f2fs_decompress_end_io(struct page **rpages, 1509 unsigned int cluster_size, bool err, bool verity) 1510 { 1511 int i; 1512 1513 for (i = 0; i < cluster_size; i++) { 1514 struct page *rpage = rpages[i]; 1515 1516 if (!rpage) 1517 continue; 1518 1519 if (err || PageError(rpage)) 1520 goto clear_uptodate; 1521 1522 if (!verity || fsverity_verify_page(rpage)) { 1523 SetPageUptodate(rpage); 1524 goto unlock; 1525 } 1526 clear_uptodate: 1527 ClearPageUptodate(rpage); 1528 ClearPageError(rpage); 1529 unlock: 1530 unlock_page(rpage); 1531 } 1532 } 1533