1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 * Copyright (C) 2022 Alibaba Cloud 6 */ 7 #include "zdata.h" 8 #include "compress.h" 9 #include <linux/prefetch.h> 10 11 #include <trace/events/erofs.h> 12 13 /* 14 * since pclustersize is variable for big pcluster feature, introduce slab 15 * pools implementation for different pcluster sizes. 16 */ 17 struct z_erofs_pcluster_slab { 18 struct kmem_cache *slab; 19 unsigned int maxpages; 20 char name[48]; 21 }; 22 23 #define _PCLP(n) { .maxpages = n } 24 25 static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { 26 _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), 27 _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES) 28 }; 29 30 struct z_erofs_bvec_iter { 31 struct page *bvpage; 32 struct z_erofs_bvset *bvset; 33 unsigned int nr, cur; 34 }; 35 36 static struct page *z_erofs_bvec_iter_end(struct z_erofs_bvec_iter *iter) 37 { 38 if (iter->bvpage) 39 kunmap_local(iter->bvset); 40 return iter->bvpage; 41 } 42 43 static struct page *z_erofs_bvset_flip(struct z_erofs_bvec_iter *iter) 44 { 45 unsigned long base = (unsigned long)((struct z_erofs_bvset *)0)->bvec; 46 /* have to access nextpage in advance, otherwise it will be unmapped */ 47 struct page *nextpage = iter->bvset->nextpage; 48 struct page *oldpage; 49 50 DBG_BUGON(!nextpage); 51 oldpage = z_erofs_bvec_iter_end(iter); 52 iter->bvpage = nextpage; 53 iter->bvset = kmap_local_page(nextpage); 54 iter->nr = (PAGE_SIZE - base) / sizeof(struct z_erofs_bvec); 55 iter->cur = 0; 56 return oldpage; 57 } 58 59 static void z_erofs_bvec_iter_begin(struct z_erofs_bvec_iter *iter, 60 struct z_erofs_bvset_inline *bvset, 61 unsigned int bootstrap_nr, 62 unsigned int cur) 63 { 64 *iter = (struct z_erofs_bvec_iter) { 65 .nr = bootstrap_nr, 66 .bvset = (struct z_erofs_bvset *)bvset, 67 }; 68 69 while (cur > iter->nr) { 70 cur -= iter->nr; 71 z_erofs_bvset_flip(iter); 72 } 73 iter->cur = cur; 74 } 75 76 static int z_erofs_bvec_enqueue(struct z_erofs_bvec_iter *iter, 77 struct z_erofs_bvec *bvec, 78 struct page **candidate_bvpage) 79 { 80 if (iter->cur == iter->nr) { 81 if (!*candidate_bvpage) 82 return -EAGAIN; 83 84 DBG_BUGON(iter->bvset->nextpage); 85 iter->bvset->nextpage = *candidate_bvpage; 86 z_erofs_bvset_flip(iter); 87 88 iter->bvset->nextpage = NULL; 89 *candidate_bvpage = NULL; 90 } 91 iter->bvset->bvec[iter->cur++] = *bvec; 92 return 0; 93 } 94 95 static void z_erofs_bvec_dequeue(struct z_erofs_bvec_iter *iter, 96 struct z_erofs_bvec *bvec, 97 struct page **old_bvpage) 98 { 99 if (iter->cur == iter->nr) 100 *old_bvpage = z_erofs_bvset_flip(iter); 101 else 102 *old_bvpage = NULL; 103 *bvec = iter->bvset->bvec[iter->cur++]; 104 } 105 106 static void z_erofs_destroy_pcluster_pool(void) 107 { 108 int i; 109 110 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 111 if (!pcluster_pool[i].slab) 112 continue; 113 kmem_cache_destroy(pcluster_pool[i].slab); 114 pcluster_pool[i].slab = NULL; 115 } 116 } 117 118 static int z_erofs_create_pcluster_pool(void) 119 { 120 struct z_erofs_pcluster_slab *pcs; 121 struct z_erofs_pcluster *a; 122 unsigned int size; 123 124 for (pcs = pcluster_pool; 125 pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { 126 size = struct_size(a, compressed_bvecs, pcs->maxpages); 127 128 sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages); 129 pcs->slab = kmem_cache_create(pcs->name, size, 0, 130 SLAB_RECLAIM_ACCOUNT, NULL); 131 if (pcs->slab) 132 continue; 133 134 z_erofs_destroy_pcluster_pool(); 135 return -ENOMEM; 136 } 137 return 0; 138 } 139 140 static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages) 141 { 142 int i; 143 144 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 145 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; 146 struct z_erofs_pcluster *pcl; 147 148 if (nrpages > pcs->maxpages) 149 continue; 150 151 pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS); 152 if (!pcl) 153 return ERR_PTR(-ENOMEM); 154 pcl->pclusterpages = nrpages; 155 return pcl; 156 } 157 return ERR_PTR(-EINVAL); 158 } 159 160 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) 161 { 162 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 163 int i; 164 165 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 166 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; 167 168 if (pclusterpages > pcs->maxpages) 169 continue; 170 171 kmem_cache_free(pcs->slab, pcl); 172 return; 173 } 174 DBG_BUGON(1); 175 } 176 177 /* how to allocate cached pages for a pcluster */ 178 enum z_erofs_cache_alloctype { 179 DONTALLOC, /* don't allocate any cached pages */ 180 /* 181 * try to use cached I/O if page allocation succeeds or fallback 182 * to in-place I/O instead to avoid any direct reclaim. 183 */ 184 TRYALLOC, 185 }; 186 187 /* 188 * tagged pointer with 1-bit tag for all compressed pages 189 * tag 0 - the page is just found with an extra page reference 190 */ 191 typedef tagptr1_t compressed_page_t; 192 193 #define tag_compressed_page_justfound(page) \ 194 tagptr_fold(compressed_page_t, page, 1) 195 196 static struct workqueue_struct *z_erofs_workqueue __read_mostly; 197 198 void z_erofs_exit_zip_subsystem(void) 199 { 200 destroy_workqueue(z_erofs_workqueue); 201 z_erofs_destroy_pcluster_pool(); 202 } 203 204 static inline int z_erofs_init_workqueue(void) 205 { 206 const unsigned int onlinecpus = num_possible_cpus(); 207 208 /* 209 * no need to spawn too many threads, limiting threads could minimum 210 * scheduling overhead, perhaps per-CPU threads should be better? 211 */ 212 z_erofs_workqueue = alloc_workqueue("erofs_unzipd", 213 WQ_UNBOUND | WQ_HIGHPRI, 214 onlinecpus + onlinecpus / 4); 215 return z_erofs_workqueue ? 0 : -ENOMEM; 216 } 217 218 int __init z_erofs_init_zip_subsystem(void) 219 { 220 int err = z_erofs_create_pcluster_pool(); 221 222 if (err) 223 return err; 224 err = z_erofs_init_workqueue(); 225 if (err) 226 z_erofs_destroy_pcluster_pool(); 227 return err; 228 } 229 230 enum z_erofs_pclustermode { 231 Z_EROFS_PCLUSTER_INFLIGHT, 232 /* 233 * The current pclusters was the tail of an exist chain, in addition 234 * that the previous processed chained pclusters are all decided to 235 * be hooked up to it. 236 * A new chain will be created for the remaining pclusters which are 237 * not processed yet, so different from Z_EROFS_PCLUSTER_FOLLOWED, 238 * the next pcluster cannot reuse the whole page safely for inplace I/O 239 * in the following scenario: 240 * ________________________________________________________________ 241 * | tail (partial) page | head (partial) page | 242 * | (belongs to the next pcl) | (belongs to the current pcl) | 243 * |_______PCLUSTER_FOLLOWED______|________PCLUSTER_HOOKED__________| 244 */ 245 Z_EROFS_PCLUSTER_HOOKED, 246 /* 247 * a weak form of Z_EROFS_PCLUSTER_FOLLOWED, the difference is that it 248 * could be dispatched into bypass queue later due to uptodated managed 249 * pages. All related online pages cannot be reused for inplace I/O (or 250 * bvpage) since it can be directly decoded without I/O submission. 251 */ 252 Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE, 253 /* 254 * The current collection has been linked with the owned chain, and 255 * could also be linked with the remaining collections, which means 256 * if the processing page is the tail page of the collection, thus 257 * the current collection can safely use the whole page (since 258 * the previous collection is under control) for in-place I/O, as 259 * illustrated below: 260 * ________________________________________________________________ 261 * | tail (partial) page | head (partial) page | 262 * | (of the current cl) | (of the previous collection) | 263 * | PCLUSTER_FOLLOWED or | | 264 * |_____PCLUSTER_HOOKED__|___________PCLUSTER_FOLLOWED____________| 265 * 266 * [ (*) the above page can be used as inplace I/O. ] 267 */ 268 Z_EROFS_PCLUSTER_FOLLOWED, 269 }; 270 271 struct z_erofs_decompress_frontend { 272 struct inode *const inode; 273 struct erofs_map_blocks map; 274 struct z_erofs_bvec_iter biter; 275 276 struct page *candidate_bvpage; 277 struct z_erofs_pcluster *pcl, *tailpcl; 278 z_erofs_next_pcluster_t owned_head; 279 enum z_erofs_pclustermode mode; 280 281 bool readahead; 282 /* used for applying cache strategy on the fly */ 283 bool backmost; 284 erofs_off_t headoffset; 285 286 /* a pointer used to pick up inplace I/O pages */ 287 unsigned int icur; 288 }; 289 290 #define DECOMPRESS_FRONTEND_INIT(__i) { \ 291 .inode = __i, .owned_head = Z_EROFS_PCLUSTER_TAIL, \ 292 .mode = Z_EROFS_PCLUSTER_FOLLOWED, .backmost = true } 293 294 static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe, 295 enum z_erofs_cache_alloctype type, 296 struct page **pagepool) 297 { 298 struct address_space *mc = MNGD_MAPPING(EROFS_I_SB(fe->inode)); 299 struct z_erofs_pcluster *pcl = fe->pcl; 300 bool standalone = true; 301 /* 302 * optimistic allocation without direct reclaim since inplace I/O 303 * can be used if low memory otherwise. 304 */ 305 gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) | 306 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 307 unsigned int i; 308 309 if (fe->mode < Z_EROFS_PCLUSTER_FOLLOWED) 310 return; 311 312 for (i = 0; i < pcl->pclusterpages; ++i) { 313 struct page *page; 314 compressed_page_t t; 315 struct page *newpage = NULL; 316 317 /* the compressed page was loaded before */ 318 if (READ_ONCE(pcl->compressed_bvecs[i].page)) 319 continue; 320 321 page = find_get_page(mc, pcl->obj.index + i); 322 323 if (page) { 324 t = tag_compressed_page_justfound(page); 325 } else { 326 /* I/O is needed, no possible to decompress directly */ 327 standalone = false; 328 switch (type) { 329 case TRYALLOC: 330 newpage = erofs_allocpage(pagepool, gfp); 331 if (!newpage) 332 continue; 333 set_page_private(newpage, 334 Z_EROFS_PREALLOCATED_PAGE); 335 t = tag_compressed_page_justfound(newpage); 336 break; 337 default: /* DONTALLOC */ 338 continue; 339 } 340 } 341 342 if (!cmpxchg_relaxed(&pcl->compressed_bvecs[i].page, NULL, 343 tagptr_cast_ptr(t))) 344 continue; 345 346 if (page) 347 put_page(page); 348 else if (newpage) 349 erofs_pagepool_add(pagepool, newpage); 350 } 351 352 /* 353 * don't do inplace I/O if all compressed pages are available in 354 * managed cache since it can be moved to the bypass queue instead. 355 */ 356 if (standalone) 357 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; 358 } 359 360 /* called by erofs_shrinker to get rid of all compressed_pages */ 361 int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, 362 struct erofs_workgroup *grp) 363 { 364 struct z_erofs_pcluster *const pcl = 365 container_of(grp, struct z_erofs_pcluster, obj); 366 int i; 367 368 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); 369 /* 370 * refcount of workgroup is now freezed as 1, 371 * therefore no need to worry about available decompression users. 372 */ 373 for (i = 0; i < pcl->pclusterpages; ++i) { 374 struct page *page = pcl->compressed_bvecs[i].page; 375 376 if (!page) 377 continue; 378 379 /* block other users from reclaiming or migrating the page */ 380 if (!trylock_page(page)) 381 return -EBUSY; 382 383 if (!erofs_page_is_managed(sbi, page)) 384 continue; 385 386 /* barrier is implied in the following 'unlock_page' */ 387 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); 388 detach_page_private(page); 389 unlock_page(page); 390 } 391 return 0; 392 } 393 394 int erofs_try_to_free_cached_page(struct page *page) 395 { 396 struct z_erofs_pcluster *const pcl = (void *)page_private(page); 397 int ret, i; 398 399 if (!erofs_workgroup_try_to_freeze(&pcl->obj, 1)) 400 return 0; 401 402 ret = 0; 403 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); 404 for (i = 0; i < pcl->pclusterpages; ++i) { 405 if (pcl->compressed_bvecs[i].page == page) { 406 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); 407 ret = 1; 408 break; 409 } 410 } 411 erofs_workgroup_unfreeze(&pcl->obj, 1); 412 if (ret) 413 detach_page_private(page); 414 return ret; 415 } 416 417 static bool z_erofs_try_inplace_io(struct z_erofs_decompress_frontend *fe, 418 struct z_erofs_bvec *bvec) 419 { 420 struct z_erofs_pcluster *const pcl = fe->pcl; 421 422 while (fe->icur > 0) { 423 if (!cmpxchg(&pcl->compressed_bvecs[--fe->icur].page, 424 NULL, bvec->page)) { 425 pcl->compressed_bvecs[fe->icur] = *bvec; 426 return true; 427 } 428 } 429 return false; 430 } 431 432 /* callers must be with pcluster lock held */ 433 static int z_erofs_attach_page(struct z_erofs_decompress_frontend *fe, 434 struct z_erofs_bvec *bvec, bool exclusive) 435 { 436 int ret; 437 438 if (exclusive) { 439 /* give priority for inplaceio to use file pages first */ 440 if (z_erofs_try_inplace_io(fe, bvec)) 441 return 0; 442 /* otherwise, check if it can be used as a bvpage */ 443 if (fe->mode >= Z_EROFS_PCLUSTER_FOLLOWED && 444 !fe->candidate_bvpage) 445 fe->candidate_bvpage = bvec->page; 446 } 447 ret = z_erofs_bvec_enqueue(&fe->biter, bvec, &fe->candidate_bvpage); 448 fe->pcl->vcnt += (ret >= 0); 449 return ret; 450 } 451 452 static void z_erofs_try_to_claim_pcluster(struct z_erofs_decompress_frontend *f) 453 { 454 struct z_erofs_pcluster *pcl = f->pcl; 455 z_erofs_next_pcluster_t *owned_head = &f->owned_head; 456 457 /* type 1, nil pcluster (this pcluster doesn't belong to any chain.) */ 458 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL, 459 *owned_head) == Z_EROFS_PCLUSTER_NIL) { 460 *owned_head = &pcl->next; 461 /* so we can attach this pcluster to our submission chain. */ 462 f->mode = Z_EROFS_PCLUSTER_FOLLOWED; 463 return; 464 } 465 466 /* 467 * type 2, link to the end of an existing open chain, be careful 468 * that its submission is controlled by the original attached chain. 469 */ 470 if (*owned_head != &pcl->next && pcl != f->tailpcl && 471 cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, 472 *owned_head) == Z_EROFS_PCLUSTER_TAIL) { 473 *owned_head = Z_EROFS_PCLUSTER_TAIL; 474 f->mode = Z_EROFS_PCLUSTER_HOOKED; 475 f->tailpcl = NULL; 476 return; 477 } 478 /* type 3, it belongs to a chain, but it isn't the end of the chain */ 479 f->mode = Z_EROFS_PCLUSTER_INFLIGHT; 480 } 481 482 static int z_erofs_register_pcluster(struct z_erofs_decompress_frontend *fe) 483 { 484 struct erofs_map_blocks *map = &fe->map; 485 bool ztailpacking = map->m_flags & EROFS_MAP_META; 486 struct z_erofs_pcluster *pcl; 487 struct erofs_workgroup *grp; 488 int err; 489 490 if (!(map->m_flags & EROFS_MAP_ENCODED)) { 491 DBG_BUGON(1); 492 return -EFSCORRUPTED; 493 } 494 495 /* no available pcluster, let's allocate one */ 496 pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 : 497 map->m_plen >> PAGE_SHIFT); 498 if (IS_ERR(pcl)) 499 return PTR_ERR(pcl); 500 501 atomic_set(&pcl->obj.refcount, 1); 502 pcl->algorithmformat = map->m_algorithmformat; 503 pcl->length = 0; 504 pcl->partial = true; 505 506 /* new pclusters should be claimed as type 1, primary and followed */ 507 pcl->next = fe->owned_head; 508 pcl->pageofs_out = map->m_la & ~PAGE_MASK; 509 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED; 510 511 /* 512 * lock all primary followed works before visible to others 513 * and mutex_trylock *never* fails for a new pcluster. 514 */ 515 mutex_init(&pcl->lock); 516 DBG_BUGON(!mutex_trylock(&pcl->lock)); 517 518 if (ztailpacking) { 519 pcl->obj.index = 0; /* which indicates ztailpacking */ 520 pcl->pageofs_in = erofs_blkoff(map->m_pa); 521 pcl->tailpacking_size = map->m_plen; 522 } else { 523 pcl->obj.index = map->m_pa >> PAGE_SHIFT; 524 525 grp = erofs_insert_workgroup(fe->inode->i_sb, &pcl->obj); 526 if (IS_ERR(grp)) { 527 err = PTR_ERR(grp); 528 goto err_out; 529 } 530 531 if (grp != &pcl->obj) { 532 fe->pcl = container_of(grp, 533 struct z_erofs_pcluster, obj); 534 err = -EEXIST; 535 goto err_out; 536 } 537 } 538 /* used to check tail merging loop due to corrupted images */ 539 if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL) 540 fe->tailpcl = pcl; 541 fe->owned_head = &pcl->next; 542 fe->pcl = pcl; 543 return 0; 544 545 err_out: 546 mutex_unlock(&pcl->lock); 547 z_erofs_free_pcluster(pcl); 548 return err; 549 } 550 551 static int z_erofs_collector_begin(struct z_erofs_decompress_frontend *fe) 552 { 553 struct erofs_map_blocks *map = &fe->map; 554 struct erofs_workgroup *grp = NULL; 555 int ret; 556 557 DBG_BUGON(fe->pcl); 558 559 /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */ 560 DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_NIL); 561 DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 562 563 if (!(map->m_flags & EROFS_MAP_META)) { 564 grp = erofs_find_workgroup(fe->inode->i_sb, 565 map->m_pa >> PAGE_SHIFT); 566 } else if ((map->m_pa & ~PAGE_MASK) + map->m_plen > PAGE_SIZE) { 567 DBG_BUGON(1); 568 return -EFSCORRUPTED; 569 } 570 571 if (grp) { 572 fe->pcl = container_of(grp, struct z_erofs_pcluster, obj); 573 ret = -EEXIST; 574 } else { 575 ret = z_erofs_register_pcluster(fe); 576 } 577 578 if (ret == -EEXIST) { 579 mutex_lock(&fe->pcl->lock); 580 /* used to check tail merging loop due to corrupted images */ 581 if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL) 582 fe->tailpcl = fe->pcl; 583 584 z_erofs_try_to_claim_pcluster(fe); 585 } else if (ret) { 586 return ret; 587 } 588 z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset, 589 Z_EROFS_INLINE_BVECS, fe->pcl->vcnt); 590 /* since file-backed online pages are traversed in reverse order */ 591 fe->icur = z_erofs_pclusterpages(fe->pcl); 592 return 0; 593 } 594 595 /* 596 * keep in mind that no referenced pclusters will be freed 597 * only after a RCU grace period. 598 */ 599 static void z_erofs_rcu_callback(struct rcu_head *head) 600 { 601 z_erofs_free_pcluster(container_of(head, 602 struct z_erofs_pcluster, rcu)); 603 } 604 605 void erofs_workgroup_free_rcu(struct erofs_workgroup *grp) 606 { 607 struct z_erofs_pcluster *const pcl = 608 container_of(grp, struct z_erofs_pcluster, obj); 609 610 call_rcu(&pcl->rcu, z_erofs_rcu_callback); 611 } 612 613 static bool z_erofs_collector_end(struct z_erofs_decompress_frontend *fe) 614 { 615 struct z_erofs_pcluster *pcl = fe->pcl; 616 617 if (!pcl) 618 return false; 619 620 z_erofs_bvec_iter_end(&fe->biter); 621 mutex_unlock(&pcl->lock); 622 623 if (fe->candidate_bvpage) { 624 DBG_BUGON(z_erofs_is_shortlived_page(fe->candidate_bvpage)); 625 fe->candidate_bvpage = NULL; 626 } 627 628 /* 629 * if all pending pages are added, don't hold its reference 630 * any longer if the pcluster isn't hosted by ourselves. 631 */ 632 if (fe->mode < Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE) 633 erofs_workgroup_put(&pcl->obj); 634 635 fe->pcl = NULL; 636 return true; 637 } 638 639 static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe, 640 unsigned int cachestrategy, 641 erofs_off_t la) 642 { 643 if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED) 644 return false; 645 646 if (fe->backmost) 647 return true; 648 649 return cachestrategy >= EROFS_ZIP_CACHE_READAROUND && 650 la < fe->headoffset; 651 } 652 653 static int z_erofs_read_fragment(struct inode *inode, erofs_off_t pos, 654 struct page *page, unsigned int pageofs, 655 unsigned int len) 656 { 657 struct inode *packed_inode = EROFS_I_SB(inode)->packed_inode; 658 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 659 u8 *src, *dst; 660 unsigned int i, cnt; 661 662 pos += EROFS_I(inode)->z_fragmentoff; 663 for (i = 0; i < len; i += cnt) { 664 cnt = min_t(unsigned int, len - i, 665 EROFS_BLKSIZ - erofs_blkoff(pos)); 666 src = erofs_bread(&buf, packed_inode, 667 erofs_blknr(pos), EROFS_KMAP); 668 if (IS_ERR(src)) { 669 erofs_put_metabuf(&buf); 670 return PTR_ERR(src); 671 } 672 673 dst = kmap_local_page(page); 674 memcpy(dst + pageofs + i, src + erofs_blkoff(pos), cnt); 675 kunmap_local(dst); 676 pos += cnt; 677 } 678 erofs_put_metabuf(&buf); 679 return 0; 680 } 681 682 static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, 683 struct page *page, struct page **pagepool) 684 { 685 struct inode *const inode = fe->inode; 686 struct erofs_sb_info *const sbi = EROFS_I_SB(inode); 687 struct erofs_map_blocks *const map = &fe->map; 688 const loff_t offset = page_offset(page); 689 bool tight = true, exclusive; 690 691 enum z_erofs_cache_alloctype cache_strategy; 692 unsigned int cur, end, spiltted; 693 int err = 0; 694 695 /* register locked file pages as online pages in pack */ 696 z_erofs_onlinepage_init(page); 697 698 spiltted = 0; 699 end = PAGE_SIZE; 700 repeat: 701 cur = end - 1; 702 703 if (offset + cur < map->m_la || 704 offset + cur >= map->m_la + map->m_llen) { 705 erofs_dbg("out-of-range map @ pos %llu", offset + cur); 706 707 if (z_erofs_collector_end(fe)) 708 fe->backmost = false; 709 map->m_la = offset + cur; 710 map->m_llen = 0; 711 err = z_erofs_map_blocks_iter(inode, map, 0); 712 if (err) 713 goto out; 714 } else { 715 if (fe->pcl) 716 goto hitted; 717 /* didn't get a valid pcluster previously (very rare) */ 718 } 719 720 if (!(map->m_flags & EROFS_MAP_MAPPED) || 721 map->m_flags & EROFS_MAP_FRAGMENT) 722 goto hitted; 723 724 err = z_erofs_collector_begin(fe); 725 if (err) 726 goto out; 727 728 if (z_erofs_is_inline_pcluster(fe->pcl)) { 729 void *mp; 730 731 mp = erofs_read_metabuf(&fe->map.buf, inode->i_sb, 732 erofs_blknr(map->m_pa), EROFS_NO_KMAP); 733 if (IS_ERR(mp)) { 734 err = PTR_ERR(mp); 735 erofs_err(inode->i_sb, 736 "failed to get inline page, err %d", err); 737 goto out; 738 } 739 get_page(fe->map.buf.page); 740 WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, 741 fe->map.buf.page); 742 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; 743 } else { 744 /* bind cache first when cached decompression is preferred */ 745 if (should_alloc_managed_pages(fe, sbi->opt.cache_strategy, 746 map->m_la)) 747 cache_strategy = TRYALLOC; 748 else 749 cache_strategy = DONTALLOC; 750 751 z_erofs_bind_cache(fe, cache_strategy, pagepool); 752 } 753 hitted: 754 /* 755 * Ensure the current partial page belongs to this submit chain rather 756 * than other concurrent submit chains or the noio(bypass) chain since 757 * those chains are handled asynchronously thus the page cannot be used 758 * for inplace I/O or bvpage (should be processed in a strict order.) 759 */ 760 tight &= (fe->mode >= Z_EROFS_PCLUSTER_HOOKED && 761 fe->mode != Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE); 762 763 cur = end - min_t(unsigned int, offset + end - map->m_la, end); 764 if (!(map->m_flags & EROFS_MAP_MAPPED)) { 765 zero_user_segment(page, cur, end); 766 goto next_part; 767 } 768 if (map->m_flags & EROFS_MAP_FRAGMENT) { 769 unsigned int pageofs, skip, len; 770 771 if (offset > map->m_la) { 772 pageofs = 0; 773 skip = offset - map->m_la; 774 } else { 775 pageofs = map->m_la & ~PAGE_MASK; 776 skip = 0; 777 } 778 len = min_t(unsigned int, map->m_llen - skip, end - cur); 779 err = z_erofs_read_fragment(inode, skip, page, pageofs, len); 780 if (err) 781 goto out; 782 ++spiltted; 783 tight = false; 784 goto next_part; 785 } 786 787 exclusive = (!cur && (!spiltted || tight)); 788 if (cur) 789 tight &= (fe->mode >= Z_EROFS_PCLUSTER_FOLLOWED); 790 791 retry: 792 err = z_erofs_attach_page(fe, &((struct z_erofs_bvec) { 793 .page = page, 794 .offset = offset - map->m_la, 795 .end = end, 796 }), exclusive); 797 /* should allocate an additional short-lived page for bvset */ 798 if (err == -EAGAIN && !fe->candidate_bvpage) { 799 fe->candidate_bvpage = alloc_page(GFP_NOFS | __GFP_NOFAIL); 800 set_page_private(fe->candidate_bvpage, 801 Z_EROFS_SHORTLIVED_PAGE); 802 goto retry; 803 } 804 805 if (err) { 806 DBG_BUGON(err == -EAGAIN && fe->candidate_bvpage); 807 goto out; 808 } 809 810 z_erofs_onlinepage_split(page); 811 /* bump up the number of spiltted parts of a page */ 812 ++spiltted; 813 if (fe->pcl->pageofs_out != (map->m_la & ~PAGE_MASK)) 814 fe->pcl->multibases = true; 815 816 if ((map->m_flags & EROFS_MAP_FULL_MAPPED) && 817 fe->pcl->length == map->m_llen) 818 fe->pcl->partial = false; 819 if (fe->pcl->length < offset + end - map->m_la) { 820 fe->pcl->length = offset + end - map->m_la; 821 fe->pcl->pageofs_out = map->m_la & ~PAGE_MASK; 822 } 823 next_part: 824 /* shorten the remaining extent to update progress */ 825 map->m_llen = offset + cur - map->m_la; 826 map->m_flags &= ~EROFS_MAP_FULL_MAPPED; 827 828 end = cur; 829 if (end > 0) 830 goto repeat; 831 832 out: 833 if (err) 834 z_erofs_page_mark_eio(page); 835 z_erofs_onlinepage_endio(page); 836 837 erofs_dbg("%s, finish page: %pK spiltted: %u map->m_llen %llu", 838 __func__, page, spiltted, map->m_llen); 839 return err; 840 } 841 842 static bool z_erofs_get_sync_decompress_policy(struct erofs_sb_info *sbi, 843 unsigned int readahead_pages) 844 { 845 /* auto: enable for read_folio, disable for readahead */ 846 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) && 847 !readahead_pages) 848 return true; 849 850 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_FORCE_ON) && 851 (readahead_pages <= sbi->opt.max_sync_decompress_pages)) 852 return true; 853 854 return false; 855 } 856 857 static bool z_erofs_page_is_invalidated(struct page *page) 858 { 859 return !page->mapping && !z_erofs_is_shortlived_page(page); 860 } 861 862 struct z_erofs_decompress_backend { 863 struct page *onstack_pages[Z_EROFS_ONSTACK_PAGES]; 864 struct super_block *sb; 865 struct z_erofs_pcluster *pcl; 866 867 /* pages with the longest decompressed length for deduplication */ 868 struct page **decompressed_pages; 869 /* pages to keep the compressed data */ 870 struct page **compressed_pages; 871 872 struct list_head decompressed_secondary_bvecs; 873 struct page **pagepool; 874 unsigned int onstack_used, nr_pages; 875 }; 876 877 struct z_erofs_bvec_item { 878 struct z_erofs_bvec bvec; 879 struct list_head list; 880 }; 881 882 static void z_erofs_do_decompressed_bvec(struct z_erofs_decompress_backend *be, 883 struct z_erofs_bvec *bvec) 884 { 885 struct z_erofs_bvec_item *item; 886 887 if (!((bvec->offset + be->pcl->pageofs_out) & ~PAGE_MASK)) { 888 unsigned int pgnr; 889 struct page *oldpage; 890 891 pgnr = (bvec->offset + be->pcl->pageofs_out) >> PAGE_SHIFT; 892 DBG_BUGON(pgnr >= be->nr_pages); 893 oldpage = be->decompressed_pages[pgnr]; 894 be->decompressed_pages[pgnr] = bvec->page; 895 896 if (!oldpage) 897 return; 898 } 899 900 /* (cold path) one pcluster is requested multiple times */ 901 item = kmalloc(sizeof(*item), GFP_KERNEL | __GFP_NOFAIL); 902 item->bvec = *bvec; 903 list_add(&item->list, &be->decompressed_secondary_bvecs); 904 } 905 906 static void z_erofs_fill_other_copies(struct z_erofs_decompress_backend *be, 907 int err) 908 { 909 unsigned int off0 = be->pcl->pageofs_out; 910 struct list_head *p, *n; 911 912 list_for_each_safe(p, n, &be->decompressed_secondary_bvecs) { 913 struct z_erofs_bvec_item *bvi; 914 unsigned int end, cur; 915 void *dst, *src; 916 917 bvi = container_of(p, struct z_erofs_bvec_item, list); 918 cur = bvi->bvec.offset < 0 ? -bvi->bvec.offset : 0; 919 end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset, 920 bvi->bvec.end); 921 dst = kmap_local_page(bvi->bvec.page); 922 while (cur < end) { 923 unsigned int pgnr, scur, len; 924 925 pgnr = (bvi->bvec.offset + cur + off0) >> PAGE_SHIFT; 926 DBG_BUGON(pgnr >= be->nr_pages); 927 928 scur = bvi->bvec.offset + cur - 929 ((pgnr << PAGE_SHIFT) - off0); 930 len = min_t(unsigned int, end - cur, PAGE_SIZE - scur); 931 if (!be->decompressed_pages[pgnr]) { 932 err = -EFSCORRUPTED; 933 cur += len; 934 continue; 935 } 936 src = kmap_local_page(be->decompressed_pages[pgnr]); 937 memcpy(dst + cur, src + scur, len); 938 kunmap_local(src); 939 cur += len; 940 } 941 kunmap_local(dst); 942 if (err) 943 z_erofs_page_mark_eio(bvi->bvec.page); 944 z_erofs_onlinepage_endio(bvi->bvec.page); 945 list_del(p); 946 kfree(bvi); 947 } 948 } 949 950 static void z_erofs_parse_out_bvecs(struct z_erofs_decompress_backend *be) 951 { 952 struct z_erofs_pcluster *pcl = be->pcl; 953 struct z_erofs_bvec_iter biter; 954 struct page *old_bvpage; 955 int i; 956 957 z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0); 958 for (i = 0; i < pcl->vcnt; ++i) { 959 struct z_erofs_bvec bvec; 960 961 z_erofs_bvec_dequeue(&biter, &bvec, &old_bvpage); 962 963 if (old_bvpage) 964 z_erofs_put_shortlivedpage(be->pagepool, old_bvpage); 965 966 DBG_BUGON(z_erofs_page_is_invalidated(bvec.page)); 967 z_erofs_do_decompressed_bvec(be, &bvec); 968 } 969 970 old_bvpage = z_erofs_bvec_iter_end(&biter); 971 if (old_bvpage) 972 z_erofs_put_shortlivedpage(be->pagepool, old_bvpage); 973 } 974 975 static int z_erofs_parse_in_bvecs(struct z_erofs_decompress_backend *be, 976 bool *overlapped) 977 { 978 struct z_erofs_pcluster *pcl = be->pcl; 979 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 980 int i, err = 0; 981 982 *overlapped = false; 983 for (i = 0; i < pclusterpages; ++i) { 984 struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i]; 985 struct page *page = bvec->page; 986 987 /* compressed pages ought to be present before decompressing */ 988 if (!page) { 989 DBG_BUGON(1); 990 continue; 991 } 992 be->compressed_pages[i] = page; 993 994 if (z_erofs_is_inline_pcluster(pcl)) { 995 if (!PageUptodate(page)) 996 err = -EIO; 997 continue; 998 } 999 1000 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1001 if (!z_erofs_is_shortlived_page(page)) { 1002 if (erofs_page_is_managed(EROFS_SB(be->sb), page)) { 1003 if (!PageUptodate(page)) 1004 err = -EIO; 1005 continue; 1006 } 1007 z_erofs_do_decompressed_bvec(be, bvec); 1008 *overlapped = true; 1009 } 1010 } 1011 1012 if (err) 1013 return err; 1014 return 0; 1015 } 1016 1017 static int z_erofs_decompress_pcluster(struct z_erofs_decompress_backend *be, 1018 int err) 1019 { 1020 struct erofs_sb_info *const sbi = EROFS_SB(be->sb); 1021 struct z_erofs_pcluster *pcl = be->pcl; 1022 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 1023 unsigned int i, inputsize; 1024 int err2; 1025 struct page *page; 1026 bool overlapped; 1027 1028 mutex_lock(&pcl->lock); 1029 be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT; 1030 1031 /* allocate (de)compressed page arrays if cannot be kept on stack */ 1032 be->decompressed_pages = NULL; 1033 be->compressed_pages = NULL; 1034 be->onstack_used = 0; 1035 if (be->nr_pages <= Z_EROFS_ONSTACK_PAGES) { 1036 be->decompressed_pages = be->onstack_pages; 1037 be->onstack_used = be->nr_pages; 1038 memset(be->decompressed_pages, 0, 1039 sizeof(struct page *) * be->nr_pages); 1040 } 1041 1042 if (pclusterpages + be->onstack_used <= Z_EROFS_ONSTACK_PAGES) 1043 be->compressed_pages = be->onstack_pages + be->onstack_used; 1044 1045 if (!be->decompressed_pages) 1046 be->decompressed_pages = 1047 kvcalloc(be->nr_pages, sizeof(struct page *), 1048 GFP_KERNEL | __GFP_NOFAIL); 1049 if (!be->compressed_pages) 1050 be->compressed_pages = 1051 kvcalloc(pclusterpages, sizeof(struct page *), 1052 GFP_KERNEL | __GFP_NOFAIL); 1053 1054 z_erofs_parse_out_bvecs(be); 1055 err2 = z_erofs_parse_in_bvecs(be, &overlapped); 1056 if (err2) 1057 err = err2; 1058 if (err) 1059 goto out; 1060 1061 if (z_erofs_is_inline_pcluster(pcl)) 1062 inputsize = pcl->tailpacking_size; 1063 else 1064 inputsize = pclusterpages * PAGE_SIZE; 1065 1066 err = z_erofs_decompress(&(struct z_erofs_decompress_req) { 1067 .sb = be->sb, 1068 .in = be->compressed_pages, 1069 .out = be->decompressed_pages, 1070 .pageofs_in = pcl->pageofs_in, 1071 .pageofs_out = pcl->pageofs_out, 1072 .inputsize = inputsize, 1073 .outputsize = pcl->length, 1074 .alg = pcl->algorithmformat, 1075 .inplace_io = overlapped, 1076 .partial_decoding = pcl->partial, 1077 .fillgaps = pcl->multibases, 1078 }, be->pagepool); 1079 1080 out: 1081 /* must handle all compressed pages before actual file pages */ 1082 if (z_erofs_is_inline_pcluster(pcl)) { 1083 page = pcl->compressed_bvecs[0].page; 1084 WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL); 1085 put_page(page); 1086 } else { 1087 for (i = 0; i < pclusterpages; ++i) { 1088 page = pcl->compressed_bvecs[i].page; 1089 1090 if (erofs_page_is_managed(sbi, page)) 1091 continue; 1092 1093 /* recycle all individual short-lived pages */ 1094 (void)z_erofs_put_shortlivedpage(be->pagepool, page); 1095 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); 1096 } 1097 } 1098 if (be->compressed_pages < be->onstack_pages || 1099 be->compressed_pages >= be->onstack_pages + Z_EROFS_ONSTACK_PAGES) 1100 kvfree(be->compressed_pages); 1101 z_erofs_fill_other_copies(be, err); 1102 1103 for (i = 0; i < be->nr_pages; ++i) { 1104 page = be->decompressed_pages[i]; 1105 if (!page) 1106 continue; 1107 1108 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1109 1110 /* recycle all individual short-lived pages */ 1111 if (z_erofs_put_shortlivedpage(be->pagepool, page)) 1112 continue; 1113 if (err) 1114 z_erofs_page_mark_eio(page); 1115 z_erofs_onlinepage_endio(page); 1116 } 1117 1118 if (be->decompressed_pages != be->onstack_pages) 1119 kvfree(be->decompressed_pages); 1120 1121 pcl->length = 0; 1122 pcl->partial = true; 1123 pcl->multibases = false; 1124 pcl->bvset.nextpage = NULL; 1125 pcl->vcnt = 0; 1126 1127 /* pcluster lock MUST be taken before the following line */ 1128 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL); 1129 mutex_unlock(&pcl->lock); 1130 return err; 1131 } 1132 1133 static void z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, 1134 struct page **pagepool) 1135 { 1136 struct z_erofs_decompress_backend be = { 1137 .sb = io->sb, 1138 .pagepool = pagepool, 1139 .decompressed_secondary_bvecs = 1140 LIST_HEAD_INIT(be.decompressed_secondary_bvecs), 1141 }; 1142 z_erofs_next_pcluster_t owned = io->head; 1143 1144 while (owned != Z_EROFS_PCLUSTER_TAIL_CLOSED) { 1145 /* impossible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */ 1146 DBG_BUGON(owned == Z_EROFS_PCLUSTER_TAIL); 1147 /* impossible that 'owned' equals Z_EROFS_PCLUSTER_NIL */ 1148 DBG_BUGON(owned == Z_EROFS_PCLUSTER_NIL); 1149 1150 be.pcl = container_of(owned, struct z_erofs_pcluster, next); 1151 owned = READ_ONCE(be.pcl->next); 1152 1153 z_erofs_decompress_pcluster(&be, io->eio ? -EIO : 0); 1154 erofs_workgroup_put(&be.pcl->obj); 1155 } 1156 } 1157 1158 static void z_erofs_decompressqueue_work(struct work_struct *work) 1159 { 1160 struct z_erofs_decompressqueue *bgq = 1161 container_of(work, struct z_erofs_decompressqueue, u.work); 1162 struct page *pagepool = NULL; 1163 1164 DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1165 z_erofs_decompress_queue(bgq, &pagepool); 1166 1167 erofs_release_pages(&pagepool); 1168 kvfree(bgq); 1169 } 1170 1171 static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, 1172 bool sync, int bios) 1173 { 1174 struct erofs_sb_info *const sbi = EROFS_SB(io->sb); 1175 1176 /* wake up the caller thread for sync decompression */ 1177 if (sync) { 1178 if (!atomic_add_return(bios, &io->pending_bios)) 1179 complete(&io->u.done); 1180 return; 1181 } 1182 1183 if (atomic_add_return(bios, &io->pending_bios)) 1184 return; 1185 /* Use workqueue and sync decompression for atomic contexts only */ 1186 if (in_atomic() || irqs_disabled()) { 1187 queue_work(z_erofs_workqueue, &io->u.work); 1188 /* enable sync decompression for readahead */ 1189 if (sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) 1190 sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; 1191 return; 1192 } 1193 z_erofs_decompressqueue_work(&io->u.work); 1194 } 1195 1196 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, 1197 unsigned int nr, 1198 struct page **pagepool, 1199 struct address_space *mc) 1200 { 1201 const pgoff_t index = pcl->obj.index; 1202 gfp_t gfp = mapping_gfp_mask(mc); 1203 bool tocache = false; 1204 1205 struct address_space *mapping; 1206 struct page *oldpage, *page; 1207 1208 compressed_page_t t; 1209 int justfound; 1210 1211 repeat: 1212 page = READ_ONCE(pcl->compressed_bvecs[nr].page); 1213 oldpage = page; 1214 1215 if (!page) 1216 goto out_allocpage; 1217 1218 /* process the target tagged pointer */ 1219 t = tagptr_init(compressed_page_t, page); 1220 justfound = tagptr_unfold_tags(t); 1221 page = tagptr_unfold_ptr(t); 1222 1223 /* 1224 * preallocated cached pages, which is used to avoid direct reclaim 1225 * otherwise, it will go inplace I/O path instead. 1226 */ 1227 if (page->private == Z_EROFS_PREALLOCATED_PAGE) { 1228 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); 1229 set_page_private(page, 0); 1230 tocache = true; 1231 goto out_tocache; 1232 } 1233 mapping = READ_ONCE(page->mapping); 1234 1235 /* 1236 * file-backed online pages in plcuster are all locked steady, 1237 * therefore it is impossible for `mapping' to be NULL. 1238 */ 1239 if (mapping && mapping != mc) 1240 /* ought to be unmanaged pages */ 1241 goto out; 1242 1243 /* directly return for shortlived page as well */ 1244 if (z_erofs_is_shortlived_page(page)) 1245 goto out; 1246 1247 lock_page(page); 1248 1249 /* only true if page reclaim goes wrong, should never happen */ 1250 DBG_BUGON(justfound && PagePrivate(page)); 1251 1252 /* the page is still in manage cache */ 1253 if (page->mapping == mc) { 1254 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); 1255 1256 if (!PagePrivate(page)) { 1257 /* 1258 * impossible to be !PagePrivate(page) for 1259 * the current restriction as well if 1260 * the page is already in compressed_bvecs[]. 1261 */ 1262 DBG_BUGON(!justfound); 1263 1264 justfound = 0; 1265 set_page_private(page, (unsigned long)pcl); 1266 SetPagePrivate(page); 1267 } 1268 1269 /* no need to submit io if it is already up-to-date */ 1270 if (PageUptodate(page)) { 1271 unlock_page(page); 1272 page = NULL; 1273 } 1274 goto out; 1275 } 1276 1277 /* 1278 * the managed page has been truncated, it's unsafe to 1279 * reuse this one, let's allocate a new cache-managed page. 1280 */ 1281 DBG_BUGON(page->mapping); 1282 DBG_BUGON(!justfound); 1283 1284 tocache = true; 1285 unlock_page(page); 1286 put_page(page); 1287 out_allocpage: 1288 page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL); 1289 if (oldpage != cmpxchg(&pcl->compressed_bvecs[nr].page, 1290 oldpage, page)) { 1291 erofs_pagepool_add(pagepool, page); 1292 cond_resched(); 1293 goto repeat; 1294 } 1295 out_tocache: 1296 if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) { 1297 /* turn into temporary page if fails (1 ref) */ 1298 set_page_private(page, Z_EROFS_SHORTLIVED_PAGE); 1299 goto out; 1300 } 1301 attach_page_private(page, pcl); 1302 /* drop a refcount added by allocpage (then we have 2 refs here) */ 1303 put_page(page); 1304 1305 out: /* the only exit (for tracing and debugging) */ 1306 return page; 1307 } 1308 1309 static struct z_erofs_decompressqueue * 1310 jobqueue_init(struct super_block *sb, 1311 struct z_erofs_decompressqueue *fgq, bool *fg) 1312 { 1313 struct z_erofs_decompressqueue *q; 1314 1315 if (fg && !*fg) { 1316 q = kvzalloc(sizeof(*q), GFP_KERNEL | __GFP_NOWARN); 1317 if (!q) { 1318 *fg = true; 1319 goto fg_out; 1320 } 1321 INIT_WORK(&q->u.work, z_erofs_decompressqueue_work); 1322 } else { 1323 fg_out: 1324 q = fgq; 1325 init_completion(&fgq->u.done); 1326 atomic_set(&fgq->pending_bios, 0); 1327 q->eio = false; 1328 } 1329 q->sb = sb; 1330 q->head = Z_EROFS_PCLUSTER_TAIL_CLOSED; 1331 return q; 1332 } 1333 1334 /* define decompression jobqueue types */ 1335 enum { 1336 JQ_BYPASS, 1337 JQ_SUBMIT, 1338 NR_JOBQUEUES, 1339 }; 1340 1341 static void *jobqueueset_init(struct super_block *sb, 1342 struct z_erofs_decompressqueue *q[], 1343 struct z_erofs_decompressqueue *fgq, bool *fg) 1344 { 1345 /* 1346 * if managed cache is enabled, bypass jobqueue is needed, 1347 * no need to read from device for all pclusters in this queue. 1348 */ 1349 q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL); 1350 q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, fg); 1351 1352 return tagptr_cast_ptr(tagptr_fold(tagptr1_t, q[JQ_SUBMIT], *fg)); 1353 } 1354 1355 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, 1356 z_erofs_next_pcluster_t qtail[], 1357 z_erofs_next_pcluster_t owned_head) 1358 { 1359 z_erofs_next_pcluster_t *const submit_qtail = qtail[JQ_SUBMIT]; 1360 z_erofs_next_pcluster_t *const bypass_qtail = qtail[JQ_BYPASS]; 1361 1362 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1363 if (owned_head == Z_EROFS_PCLUSTER_TAIL) 1364 owned_head = Z_EROFS_PCLUSTER_TAIL_CLOSED; 1365 1366 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED); 1367 1368 WRITE_ONCE(*submit_qtail, owned_head); 1369 WRITE_ONCE(*bypass_qtail, &pcl->next); 1370 1371 qtail[JQ_BYPASS] = &pcl->next; 1372 } 1373 1374 static void z_erofs_decompressqueue_endio(struct bio *bio) 1375 { 1376 tagptr1_t t = tagptr_init(tagptr1_t, bio->bi_private); 1377 struct z_erofs_decompressqueue *q = tagptr_unfold_ptr(t); 1378 blk_status_t err = bio->bi_status; 1379 struct bio_vec *bvec; 1380 struct bvec_iter_all iter_all; 1381 1382 bio_for_each_segment_all(bvec, bio, iter_all) { 1383 struct page *page = bvec->bv_page; 1384 1385 DBG_BUGON(PageUptodate(page)); 1386 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1387 1388 if (erofs_page_is_managed(EROFS_SB(q->sb), page)) { 1389 if (!err) 1390 SetPageUptodate(page); 1391 unlock_page(page); 1392 } 1393 } 1394 if (err) 1395 q->eio = true; 1396 z_erofs_decompress_kickoff(q, tagptr_unfold_tags(t), -1); 1397 bio_put(bio); 1398 } 1399 1400 static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, 1401 struct page **pagepool, 1402 struct z_erofs_decompressqueue *fgq, 1403 bool *force_fg) 1404 { 1405 struct super_block *sb = f->inode->i_sb; 1406 struct address_space *mc = MNGD_MAPPING(EROFS_SB(sb)); 1407 z_erofs_next_pcluster_t qtail[NR_JOBQUEUES]; 1408 struct z_erofs_decompressqueue *q[NR_JOBQUEUES]; 1409 void *bi_private; 1410 z_erofs_next_pcluster_t owned_head = f->owned_head; 1411 /* bio is NULL initially, so no need to initialize last_{index,bdev} */ 1412 pgoff_t last_index; 1413 struct block_device *last_bdev; 1414 unsigned int nr_bios = 0; 1415 struct bio *bio = NULL; 1416 1417 bi_private = jobqueueset_init(sb, q, fgq, force_fg); 1418 qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; 1419 qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head; 1420 1421 /* by default, all need io submission */ 1422 q[JQ_SUBMIT]->head = owned_head; 1423 1424 do { 1425 struct erofs_map_dev mdev; 1426 struct z_erofs_pcluster *pcl; 1427 pgoff_t cur, end; 1428 unsigned int i = 0; 1429 bool bypass = true; 1430 1431 /* no possible 'owned_head' equals the following */ 1432 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1433 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_NIL); 1434 1435 pcl = container_of(owned_head, struct z_erofs_pcluster, next); 1436 1437 /* close the main owned chain at first */ 1438 owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, 1439 Z_EROFS_PCLUSTER_TAIL_CLOSED); 1440 if (z_erofs_is_inline_pcluster(pcl)) { 1441 move_to_bypass_jobqueue(pcl, qtail, owned_head); 1442 continue; 1443 } 1444 1445 /* no device id here, thus it will always succeed */ 1446 mdev = (struct erofs_map_dev) { 1447 .m_pa = blknr_to_addr(pcl->obj.index), 1448 }; 1449 (void)erofs_map_dev(sb, &mdev); 1450 1451 cur = erofs_blknr(mdev.m_pa); 1452 end = cur + pcl->pclusterpages; 1453 1454 do { 1455 struct page *page; 1456 1457 page = pickup_page_for_submission(pcl, i++, pagepool, 1458 mc); 1459 if (!page) 1460 continue; 1461 1462 if (bio && (cur != last_index + 1 || 1463 last_bdev != mdev.m_bdev)) { 1464 submit_bio_retry: 1465 submit_bio(bio); 1466 bio = NULL; 1467 } 1468 1469 if (!bio) { 1470 bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS, 1471 REQ_OP_READ, GFP_NOIO); 1472 bio->bi_end_io = z_erofs_decompressqueue_endio; 1473 1474 last_bdev = mdev.m_bdev; 1475 bio->bi_iter.bi_sector = (sector_t)cur << 1476 LOG_SECTORS_PER_BLOCK; 1477 bio->bi_private = bi_private; 1478 if (f->readahead) 1479 bio->bi_opf |= REQ_RAHEAD; 1480 ++nr_bios; 1481 } 1482 1483 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) 1484 goto submit_bio_retry; 1485 1486 last_index = cur; 1487 bypass = false; 1488 } while (++cur < end); 1489 1490 if (!bypass) 1491 qtail[JQ_SUBMIT] = &pcl->next; 1492 else 1493 move_to_bypass_jobqueue(pcl, qtail, owned_head); 1494 } while (owned_head != Z_EROFS_PCLUSTER_TAIL); 1495 1496 if (bio) 1497 submit_bio(bio); 1498 1499 /* 1500 * although background is preferred, no one is pending for submission. 1501 * don't issue workqueue for decompression but drop it directly instead. 1502 */ 1503 if (!*force_fg && !nr_bios) { 1504 kvfree(q[JQ_SUBMIT]); 1505 return; 1506 } 1507 z_erofs_decompress_kickoff(q[JQ_SUBMIT], *force_fg, nr_bios); 1508 } 1509 1510 static void z_erofs_runqueue(struct z_erofs_decompress_frontend *f, 1511 struct page **pagepool, bool force_fg) 1512 { 1513 struct z_erofs_decompressqueue io[NR_JOBQUEUES]; 1514 1515 if (f->owned_head == Z_EROFS_PCLUSTER_TAIL) 1516 return; 1517 z_erofs_submit_queue(f, pagepool, io, &force_fg); 1518 1519 /* handle bypass queue (no i/o pclusters) immediately */ 1520 z_erofs_decompress_queue(&io[JQ_BYPASS], pagepool); 1521 1522 if (!force_fg) 1523 return; 1524 1525 /* wait until all bios are completed */ 1526 wait_for_completion_io(&io[JQ_SUBMIT].u.done); 1527 1528 /* handle synchronous decompress queue in the caller context */ 1529 z_erofs_decompress_queue(&io[JQ_SUBMIT], pagepool); 1530 } 1531 1532 /* 1533 * Since partial uptodate is still unimplemented for now, we have to use 1534 * approximate readmore strategies as a start. 1535 */ 1536 static void z_erofs_pcluster_readmore(struct z_erofs_decompress_frontend *f, 1537 struct readahead_control *rac, 1538 erofs_off_t end, 1539 struct page **pagepool, 1540 bool backmost) 1541 { 1542 struct inode *inode = f->inode; 1543 struct erofs_map_blocks *map = &f->map; 1544 erofs_off_t cur; 1545 int err; 1546 1547 if (backmost) { 1548 map->m_la = end; 1549 err = z_erofs_map_blocks_iter(inode, map, 1550 EROFS_GET_BLOCKS_READMORE); 1551 if (err) 1552 return; 1553 1554 /* expend ra for the trailing edge if readahead */ 1555 if (rac) { 1556 loff_t newstart = readahead_pos(rac); 1557 1558 cur = round_up(map->m_la + map->m_llen, PAGE_SIZE); 1559 readahead_expand(rac, newstart, cur - newstart); 1560 return; 1561 } 1562 end = round_up(end, PAGE_SIZE); 1563 } else { 1564 end = round_up(map->m_la, PAGE_SIZE); 1565 1566 if (!map->m_llen) 1567 return; 1568 } 1569 1570 cur = map->m_la + map->m_llen - 1; 1571 while (cur >= end) { 1572 pgoff_t index = cur >> PAGE_SHIFT; 1573 struct page *page; 1574 1575 page = erofs_grab_cache_page_nowait(inode->i_mapping, index); 1576 if (page) { 1577 if (PageUptodate(page)) { 1578 unlock_page(page); 1579 } else { 1580 err = z_erofs_do_read_page(f, page, pagepool); 1581 if (err) 1582 erofs_err(inode->i_sb, 1583 "readmore error at page %lu @ nid %llu", 1584 index, EROFS_I(inode)->nid); 1585 } 1586 put_page(page); 1587 } 1588 1589 if (cur < PAGE_SIZE) 1590 break; 1591 cur = (index << PAGE_SHIFT) - 1; 1592 } 1593 } 1594 1595 static int z_erofs_read_folio(struct file *file, struct folio *folio) 1596 { 1597 struct page *page = &folio->page; 1598 struct inode *const inode = page->mapping->host; 1599 struct erofs_sb_info *const sbi = EROFS_I_SB(inode); 1600 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); 1601 struct page *pagepool = NULL; 1602 int err; 1603 1604 trace_erofs_readpage(page, false); 1605 f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT; 1606 1607 z_erofs_pcluster_readmore(&f, NULL, f.headoffset + PAGE_SIZE - 1, 1608 &pagepool, true); 1609 err = z_erofs_do_read_page(&f, page, &pagepool); 1610 z_erofs_pcluster_readmore(&f, NULL, 0, &pagepool, false); 1611 1612 (void)z_erofs_collector_end(&f); 1613 1614 /* if some compressed cluster ready, need submit them anyway */ 1615 z_erofs_runqueue(&f, &pagepool, 1616 z_erofs_get_sync_decompress_policy(sbi, 0)); 1617 1618 if (err) 1619 erofs_err(inode->i_sb, "failed to read, err [%d]", err); 1620 1621 erofs_put_metabuf(&f.map.buf); 1622 erofs_release_pages(&pagepool); 1623 return err; 1624 } 1625 1626 static void z_erofs_readahead(struct readahead_control *rac) 1627 { 1628 struct inode *const inode = rac->mapping->host; 1629 struct erofs_sb_info *const sbi = EROFS_I_SB(inode); 1630 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); 1631 struct page *pagepool = NULL, *head = NULL, *page; 1632 unsigned int nr_pages; 1633 1634 f.readahead = true; 1635 f.headoffset = readahead_pos(rac); 1636 1637 z_erofs_pcluster_readmore(&f, rac, f.headoffset + 1638 readahead_length(rac) - 1, &pagepool, true); 1639 nr_pages = readahead_count(rac); 1640 trace_erofs_readpages(inode, readahead_index(rac), nr_pages, false); 1641 1642 while ((page = readahead_page(rac))) { 1643 set_page_private(page, (unsigned long)head); 1644 head = page; 1645 } 1646 1647 while (head) { 1648 struct page *page = head; 1649 int err; 1650 1651 /* traversal in reverse order */ 1652 head = (void *)page_private(page); 1653 1654 err = z_erofs_do_read_page(&f, page, &pagepool); 1655 if (err) 1656 erofs_err(inode->i_sb, 1657 "readahead error at page %lu @ nid %llu", 1658 page->index, EROFS_I(inode)->nid); 1659 put_page(page); 1660 } 1661 z_erofs_pcluster_readmore(&f, rac, 0, &pagepool, false); 1662 (void)z_erofs_collector_end(&f); 1663 1664 z_erofs_runqueue(&f, &pagepool, 1665 z_erofs_get_sync_decompress_policy(sbi, nr_pages)); 1666 erofs_put_metabuf(&f.map.buf); 1667 erofs_release_pages(&pagepool); 1668 } 1669 1670 const struct address_space_operations z_erofs_aops = { 1671 .read_folio = z_erofs_read_folio, 1672 .readahead = z_erofs_readahead, 1673 }; 1674