1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 */ 6 #include "zdata.h" 7 #include "compress.h" 8 #include <linux/prefetch.h> 9 10 #include <trace/events/erofs.h> 11 12 /* 13 * since pclustersize is variable for big pcluster feature, introduce slab 14 * pools implementation for different pcluster sizes. 15 */ 16 struct z_erofs_pcluster_slab { 17 struct kmem_cache *slab; 18 unsigned int maxpages; 19 char name[48]; 20 }; 21 22 #define _PCLP(n) { .maxpages = n } 23 24 static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { 25 _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), 26 _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES) 27 }; 28 29 static void z_erofs_destroy_pcluster_pool(void) 30 { 31 int i; 32 33 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 34 if (!pcluster_pool[i].slab) 35 continue; 36 kmem_cache_destroy(pcluster_pool[i].slab); 37 pcluster_pool[i].slab = NULL; 38 } 39 } 40 41 static int z_erofs_create_pcluster_pool(void) 42 { 43 struct z_erofs_pcluster_slab *pcs; 44 struct z_erofs_pcluster *a; 45 unsigned int size; 46 47 for (pcs = pcluster_pool; 48 pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { 49 size = struct_size(a, compressed_pages, pcs->maxpages); 50 51 sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages); 52 pcs->slab = kmem_cache_create(pcs->name, size, 0, 53 SLAB_RECLAIM_ACCOUNT, NULL); 54 if (pcs->slab) 55 continue; 56 57 z_erofs_destroy_pcluster_pool(); 58 return -ENOMEM; 59 } 60 return 0; 61 } 62 63 static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages) 64 { 65 int i; 66 67 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 68 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; 69 struct z_erofs_pcluster *pcl; 70 71 if (nrpages > pcs->maxpages) 72 continue; 73 74 pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS); 75 if (!pcl) 76 return ERR_PTR(-ENOMEM); 77 pcl->pclusterpages = nrpages; 78 return pcl; 79 } 80 return ERR_PTR(-EINVAL); 81 } 82 83 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) 84 { 85 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 86 int i; 87 88 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 89 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; 90 91 if (pclusterpages > pcs->maxpages) 92 continue; 93 94 kmem_cache_free(pcs->slab, pcl); 95 return; 96 } 97 DBG_BUGON(1); 98 } 99 100 /* how to allocate cached pages for a pcluster */ 101 enum z_erofs_cache_alloctype { 102 DONTALLOC, /* don't allocate any cached pages */ 103 /* 104 * try to use cached I/O if page allocation succeeds or fallback 105 * to in-place I/O instead to avoid any direct reclaim. 106 */ 107 TRYALLOC, 108 }; 109 110 /* 111 * tagged pointer with 1-bit tag for all compressed pages 112 * tag 0 - the page is just found with an extra page reference 113 */ 114 typedef tagptr1_t compressed_page_t; 115 116 #define tag_compressed_page_justfound(page) \ 117 tagptr_fold(compressed_page_t, page, 1) 118 119 static struct workqueue_struct *z_erofs_workqueue __read_mostly; 120 121 void z_erofs_exit_zip_subsystem(void) 122 { 123 destroy_workqueue(z_erofs_workqueue); 124 z_erofs_destroy_pcluster_pool(); 125 } 126 127 static inline int z_erofs_init_workqueue(void) 128 { 129 const unsigned int onlinecpus = num_possible_cpus(); 130 131 /* 132 * no need to spawn too many threads, limiting threads could minimum 133 * scheduling overhead, perhaps per-CPU threads should be better? 134 */ 135 z_erofs_workqueue = alloc_workqueue("erofs_unzipd", 136 WQ_UNBOUND | WQ_HIGHPRI, 137 onlinecpus + onlinecpus / 4); 138 return z_erofs_workqueue ? 0 : -ENOMEM; 139 } 140 141 int __init z_erofs_init_zip_subsystem(void) 142 { 143 int err = z_erofs_create_pcluster_pool(); 144 145 if (err) 146 return err; 147 err = z_erofs_init_workqueue(); 148 if (err) 149 z_erofs_destroy_pcluster_pool(); 150 return err; 151 } 152 153 enum z_erofs_collectmode { 154 COLLECT_SECONDARY, 155 COLLECT_PRIMARY, 156 /* 157 * The current collection was the tail of an exist chain, in addition 158 * that the previous processed chained collections are all decided to 159 * be hooked up to it. 160 * A new chain will be created for the remaining collections which are 161 * not processed yet, therefore different from COLLECT_PRIMARY_FOLLOWED, 162 * the next collection cannot reuse the whole page safely in 163 * the following scenario: 164 * ________________________________________________________________ 165 * | tail (partial) page | head (partial) page | 166 * | (belongs to the next cl) | (belongs to the current cl) | 167 * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________| 168 */ 169 COLLECT_PRIMARY_HOOKED, 170 /* 171 * a weak form of COLLECT_PRIMARY_FOLLOWED, the difference is that it 172 * could be dispatched into bypass queue later due to uptodated managed 173 * pages. All related online pages cannot be reused for inplace I/O (or 174 * pagevec) since it can be directly decoded without I/O submission. 175 */ 176 COLLECT_PRIMARY_FOLLOWED_NOINPLACE, 177 /* 178 * The current collection has been linked with the owned chain, and 179 * could also be linked with the remaining collections, which means 180 * if the processing page is the tail page of the collection, thus 181 * the current collection can safely use the whole page (since 182 * the previous collection is under control) for in-place I/O, as 183 * illustrated below: 184 * ________________________________________________________________ 185 * | tail (partial) page | head (partial) page | 186 * | (of the current cl) | (of the previous collection) | 187 * | PRIMARY_FOLLOWED or | | 188 * |_____PRIMARY_HOOKED___|____________PRIMARY_FOLLOWED____________| 189 * 190 * [ (*) the above page can be used as inplace I/O. ] 191 */ 192 COLLECT_PRIMARY_FOLLOWED, 193 }; 194 195 struct z_erofs_decompress_frontend { 196 struct inode *const inode; 197 struct erofs_map_blocks map; 198 199 struct z_erofs_pagevec_ctor vector; 200 201 struct z_erofs_pcluster *pcl, *tailpcl; 202 struct z_erofs_collection *cl; 203 /* a pointer used to pick up inplace I/O pages */ 204 struct page **icpage_ptr; 205 z_erofs_next_pcluster_t owned_head; 206 207 enum z_erofs_collectmode mode; 208 209 bool readahead; 210 /* used for applying cache strategy on the fly */ 211 bool backmost; 212 erofs_off_t headoffset; 213 }; 214 215 #define DECOMPRESS_FRONTEND_INIT(__i) { \ 216 .inode = __i, .owned_head = Z_EROFS_PCLUSTER_TAIL, \ 217 .mode = COLLECT_PRIMARY_FOLLOWED } 218 219 static struct page *z_pagemap_global[Z_EROFS_VMAP_GLOBAL_PAGES]; 220 static DEFINE_MUTEX(z_pagemap_global_lock); 221 222 static void preload_compressed_pages(struct z_erofs_decompress_frontend *fe, 223 struct address_space *mc, 224 enum z_erofs_cache_alloctype type, 225 struct page **pagepool) 226 { 227 struct z_erofs_pcluster *pcl = fe->pcl; 228 bool standalone = true; 229 gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) | 230 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 231 struct page **pages; 232 pgoff_t index; 233 234 if (fe->mode < COLLECT_PRIMARY_FOLLOWED) 235 return; 236 237 pages = pcl->compressed_pages; 238 index = pcl->obj.index; 239 for (; index < pcl->obj.index + pcl->pclusterpages; ++index, ++pages) { 240 struct page *page; 241 compressed_page_t t; 242 struct page *newpage = NULL; 243 244 /* the compressed page was loaded before */ 245 if (READ_ONCE(*pages)) 246 continue; 247 248 page = find_get_page(mc, index); 249 250 if (page) { 251 t = tag_compressed_page_justfound(page); 252 } else { 253 /* I/O is needed, no possible to decompress directly */ 254 standalone = false; 255 switch (type) { 256 case TRYALLOC: 257 newpage = erofs_allocpage(pagepool, gfp); 258 if (!newpage) 259 continue; 260 set_page_private(newpage, 261 Z_EROFS_PREALLOCATED_PAGE); 262 t = tag_compressed_page_justfound(newpage); 263 break; 264 default: /* DONTALLOC */ 265 continue; 266 } 267 } 268 269 if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t))) 270 continue; 271 272 if (page) 273 put_page(page); 274 else if (newpage) 275 erofs_pagepool_add(pagepool, newpage); 276 } 277 278 /* 279 * don't do inplace I/O if all compressed pages are available in 280 * managed cache since it can be moved to the bypass queue instead. 281 */ 282 if (standalone) 283 fe->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE; 284 } 285 286 /* called by erofs_shrinker to get rid of all compressed_pages */ 287 int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, 288 struct erofs_workgroup *grp) 289 { 290 struct z_erofs_pcluster *const pcl = 291 container_of(grp, struct z_erofs_pcluster, obj); 292 int i; 293 294 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); 295 /* 296 * refcount of workgroup is now freezed as 1, 297 * therefore no need to worry about available decompression users. 298 */ 299 for (i = 0; i < pcl->pclusterpages; ++i) { 300 struct page *page = pcl->compressed_pages[i]; 301 302 if (!page) 303 continue; 304 305 /* block other users from reclaiming or migrating the page */ 306 if (!trylock_page(page)) 307 return -EBUSY; 308 309 if (!erofs_page_is_managed(sbi, page)) 310 continue; 311 312 /* barrier is implied in the following 'unlock_page' */ 313 WRITE_ONCE(pcl->compressed_pages[i], NULL); 314 detach_page_private(page); 315 unlock_page(page); 316 } 317 return 0; 318 } 319 320 int erofs_try_to_free_cached_page(struct page *page) 321 { 322 struct z_erofs_pcluster *const pcl = (void *)page_private(page); 323 int ret = 0; /* 0 - busy */ 324 325 if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) { 326 unsigned int i; 327 328 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); 329 for (i = 0; i < pcl->pclusterpages; ++i) { 330 if (pcl->compressed_pages[i] == page) { 331 WRITE_ONCE(pcl->compressed_pages[i], NULL); 332 ret = 1; 333 break; 334 } 335 } 336 erofs_workgroup_unfreeze(&pcl->obj, 1); 337 338 if (ret) 339 detach_page_private(page); 340 } 341 return ret; 342 } 343 344 /* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */ 345 static bool z_erofs_try_inplace_io(struct z_erofs_decompress_frontend *fe, 346 struct page *page) 347 { 348 struct z_erofs_pcluster *const pcl = fe->pcl; 349 350 while (fe->icpage_ptr > pcl->compressed_pages) 351 if (!cmpxchg(--fe->icpage_ptr, NULL, page)) 352 return true; 353 return false; 354 } 355 356 /* callers must be with collection lock held */ 357 static int z_erofs_attach_page(struct z_erofs_decompress_frontend *fe, 358 struct page *page, enum z_erofs_page_type type, 359 bool pvec_safereuse) 360 { 361 int ret; 362 363 /* give priority for inplaceio */ 364 if (fe->mode >= COLLECT_PRIMARY && 365 type == Z_EROFS_PAGE_TYPE_EXCLUSIVE && 366 z_erofs_try_inplace_io(fe, page)) 367 return 0; 368 369 ret = z_erofs_pagevec_enqueue(&fe->vector, page, type, 370 pvec_safereuse); 371 fe->cl->vcnt += (unsigned int)ret; 372 return ret ? 0 : -EAGAIN; 373 } 374 375 static void z_erofs_try_to_claim_pcluster(struct z_erofs_decompress_frontend *f) 376 { 377 struct z_erofs_pcluster *pcl = f->pcl; 378 z_erofs_next_pcluster_t *owned_head = &f->owned_head; 379 380 /* type 1, nil pcluster (this pcluster doesn't belong to any chain.) */ 381 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL, 382 *owned_head) == Z_EROFS_PCLUSTER_NIL) { 383 *owned_head = &pcl->next; 384 /* so we can attach this pcluster to our submission chain. */ 385 f->mode = COLLECT_PRIMARY_FOLLOWED; 386 return; 387 } 388 389 /* 390 * type 2, link to the end of an existing open chain, be careful 391 * that its submission is controlled by the original attached chain. 392 */ 393 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, 394 *owned_head) == Z_EROFS_PCLUSTER_TAIL) { 395 *owned_head = Z_EROFS_PCLUSTER_TAIL; 396 f->mode = COLLECT_PRIMARY_HOOKED; 397 f->tailpcl = NULL; 398 return; 399 } 400 /* type 3, it belongs to a chain, but it isn't the end of the chain */ 401 f->mode = COLLECT_PRIMARY; 402 } 403 404 static int z_erofs_lookup_collection(struct z_erofs_decompress_frontend *fe, 405 struct inode *inode, 406 struct erofs_map_blocks *map) 407 { 408 struct z_erofs_pcluster *pcl = fe->pcl; 409 struct z_erofs_collection *cl; 410 unsigned int length; 411 412 /* to avoid unexpected loop formed by corrupted images */ 413 if (fe->owned_head == &pcl->next || pcl == fe->tailpcl) { 414 DBG_BUGON(1); 415 return -EFSCORRUPTED; 416 } 417 418 cl = z_erofs_primarycollection(pcl); 419 if (cl->pageofs != (map->m_la & ~PAGE_MASK)) { 420 DBG_BUGON(1); 421 return -EFSCORRUPTED; 422 } 423 424 length = READ_ONCE(pcl->length); 425 if (length & Z_EROFS_PCLUSTER_FULL_LENGTH) { 426 if ((map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) > length) { 427 DBG_BUGON(1); 428 return -EFSCORRUPTED; 429 } 430 } else { 431 unsigned int llen = map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT; 432 433 if (map->m_flags & EROFS_MAP_FULL_MAPPED) 434 llen |= Z_EROFS_PCLUSTER_FULL_LENGTH; 435 436 while (llen > length && 437 length != cmpxchg_relaxed(&pcl->length, length, llen)) { 438 cpu_relax(); 439 length = READ_ONCE(pcl->length); 440 } 441 } 442 mutex_lock(&cl->lock); 443 /* used to check tail merging loop due to corrupted images */ 444 if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL) 445 fe->tailpcl = pcl; 446 447 z_erofs_try_to_claim_pcluster(fe); 448 fe->cl = cl; 449 return 0; 450 } 451 452 static int z_erofs_register_collection(struct z_erofs_decompress_frontend *fe, 453 struct inode *inode, 454 struct erofs_map_blocks *map) 455 { 456 bool ztailpacking = map->m_flags & EROFS_MAP_META; 457 struct z_erofs_pcluster *pcl; 458 struct z_erofs_collection *cl; 459 struct erofs_workgroup *grp; 460 int err; 461 462 if (!(map->m_flags & EROFS_MAP_ENCODED)) { 463 DBG_BUGON(1); 464 return -EFSCORRUPTED; 465 } 466 467 /* no available pcluster, let's allocate one */ 468 pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 : 469 map->m_plen >> PAGE_SHIFT); 470 if (IS_ERR(pcl)) 471 return PTR_ERR(pcl); 472 473 atomic_set(&pcl->obj.refcount, 1); 474 pcl->algorithmformat = map->m_algorithmformat; 475 pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) | 476 (map->m_flags & EROFS_MAP_FULL_MAPPED ? 477 Z_EROFS_PCLUSTER_FULL_LENGTH : 0); 478 479 /* new pclusters should be claimed as type 1, primary and followed */ 480 pcl->next = fe->owned_head; 481 fe->mode = COLLECT_PRIMARY_FOLLOWED; 482 483 cl = z_erofs_primarycollection(pcl); 484 cl->pageofs = map->m_la & ~PAGE_MASK; 485 486 /* 487 * lock all primary followed works before visible to others 488 * and mutex_trylock *never* fails for a new pcluster. 489 */ 490 mutex_init(&cl->lock); 491 DBG_BUGON(!mutex_trylock(&cl->lock)); 492 493 if (ztailpacking) { 494 pcl->obj.index = 0; /* which indicates ztailpacking */ 495 pcl->pageofs_in = erofs_blkoff(map->m_pa); 496 pcl->tailpacking_size = map->m_plen; 497 } else { 498 pcl->obj.index = map->m_pa >> PAGE_SHIFT; 499 500 grp = erofs_insert_workgroup(inode->i_sb, &pcl->obj); 501 if (IS_ERR(grp)) { 502 err = PTR_ERR(grp); 503 goto err_out; 504 } 505 506 if (grp != &pcl->obj) { 507 fe->pcl = container_of(grp, 508 struct z_erofs_pcluster, obj); 509 err = -EEXIST; 510 goto err_out; 511 } 512 } 513 /* used to check tail merging loop due to corrupted images */ 514 if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL) 515 fe->tailpcl = pcl; 516 fe->owned_head = &pcl->next; 517 fe->pcl = pcl; 518 fe->cl = cl; 519 return 0; 520 521 err_out: 522 mutex_unlock(&cl->lock); 523 z_erofs_free_pcluster(pcl); 524 return err; 525 } 526 527 static int z_erofs_collector_begin(struct z_erofs_decompress_frontend *fe, 528 struct inode *inode, 529 struct erofs_map_blocks *map) 530 { 531 struct erofs_workgroup *grp; 532 int ret; 533 534 DBG_BUGON(fe->cl); 535 536 /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous collection */ 537 DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_NIL); 538 DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 539 540 if (map->m_flags & EROFS_MAP_META) { 541 if ((map->m_pa & ~PAGE_MASK) + map->m_plen > PAGE_SIZE) { 542 DBG_BUGON(1); 543 return -EFSCORRUPTED; 544 } 545 goto tailpacking; 546 } 547 548 grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT); 549 if (grp) { 550 fe->pcl = container_of(grp, struct z_erofs_pcluster, obj); 551 } else { 552 tailpacking: 553 ret = z_erofs_register_collection(fe, inode, map); 554 if (!ret) 555 goto out; 556 if (ret != -EEXIST) 557 return ret; 558 } 559 560 ret = z_erofs_lookup_collection(fe, inode, map); 561 if (ret) { 562 erofs_workgroup_put(&fe->pcl->obj); 563 return ret; 564 } 565 566 out: 567 z_erofs_pagevec_ctor_init(&fe->vector, Z_EROFS_NR_INLINE_PAGEVECS, 568 fe->cl->pagevec, fe->cl->vcnt); 569 /* since file-backed online pages are traversed in reverse order */ 570 fe->icpage_ptr = fe->pcl->compressed_pages + 571 z_erofs_pclusterpages(fe->pcl); 572 return 0; 573 } 574 575 /* 576 * keep in mind that no referenced pclusters will be freed 577 * only after a RCU grace period. 578 */ 579 static void z_erofs_rcu_callback(struct rcu_head *head) 580 { 581 struct z_erofs_collection *const cl = 582 container_of(head, struct z_erofs_collection, rcu); 583 584 z_erofs_free_pcluster(container_of(cl, struct z_erofs_pcluster, 585 primary_collection)); 586 } 587 588 void erofs_workgroup_free_rcu(struct erofs_workgroup *grp) 589 { 590 struct z_erofs_pcluster *const pcl = 591 container_of(grp, struct z_erofs_pcluster, obj); 592 struct z_erofs_collection *const cl = z_erofs_primarycollection(pcl); 593 594 call_rcu(&cl->rcu, z_erofs_rcu_callback); 595 } 596 597 static void z_erofs_collection_put(struct z_erofs_collection *cl) 598 { 599 struct z_erofs_pcluster *const pcl = 600 container_of(cl, struct z_erofs_pcluster, primary_collection); 601 602 erofs_workgroup_put(&pcl->obj); 603 } 604 605 static bool z_erofs_collector_end(struct z_erofs_decompress_frontend *fe) 606 { 607 struct z_erofs_collection *cl = fe->cl; 608 609 if (!cl) 610 return false; 611 612 z_erofs_pagevec_ctor_exit(&fe->vector, false); 613 mutex_unlock(&cl->lock); 614 615 /* 616 * if all pending pages are added, don't hold its reference 617 * any longer if the pcluster isn't hosted by ourselves. 618 */ 619 if (fe->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE) 620 z_erofs_collection_put(cl); 621 622 fe->cl = NULL; 623 return true; 624 } 625 626 static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe, 627 unsigned int cachestrategy, 628 erofs_off_t la) 629 { 630 if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED) 631 return false; 632 633 if (fe->backmost) 634 return true; 635 636 return cachestrategy >= EROFS_ZIP_CACHE_READAROUND && 637 la < fe->headoffset; 638 } 639 640 static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, 641 struct page *page, struct page **pagepool) 642 { 643 struct inode *const inode = fe->inode; 644 struct erofs_sb_info *const sbi = EROFS_I_SB(inode); 645 struct erofs_map_blocks *const map = &fe->map; 646 const loff_t offset = page_offset(page); 647 bool tight = true; 648 649 enum z_erofs_cache_alloctype cache_strategy; 650 enum z_erofs_page_type page_type; 651 unsigned int cur, end, spiltted, index; 652 int err = 0; 653 654 /* register locked file pages as online pages in pack */ 655 z_erofs_onlinepage_init(page); 656 657 spiltted = 0; 658 end = PAGE_SIZE; 659 repeat: 660 cur = end - 1; 661 662 /* lucky, within the range of the current map_blocks */ 663 if (offset + cur >= map->m_la && 664 offset + cur < map->m_la + map->m_llen) { 665 /* didn't get a valid collection previously (very rare) */ 666 if (!fe->cl) 667 goto restart_now; 668 goto hitted; 669 } 670 671 /* go ahead the next map_blocks */ 672 erofs_dbg("%s: [out-of-range] pos %llu", __func__, offset + cur); 673 674 if (z_erofs_collector_end(fe)) 675 fe->backmost = false; 676 677 map->m_la = offset + cur; 678 map->m_llen = 0; 679 err = z_erofs_map_blocks_iter(inode, map, 0); 680 if (err) 681 goto err_out; 682 683 restart_now: 684 if (!(map->m_flags & EROFS_MAP_MAPPED)) 685 goto hitted; 686 687 err = z_erofs_collector_begin(fe, inode, map); 688 if (err) 689 goto err_out; 690 691 if (z_erofs_is_inline_pcluster(fe->pcl)) { 692 void *mp; 693 694 mp = erofs_read_metabuf(&fe->map.buf, inode->i_sb, 695 erofs_blknr(map->m_pa), EROFS_NO_KMAP); 696 if (IS_ERR(mp)) { 697 err = PTR_ERR(mp); 698 erofs_err(inode->i_sb, 699 "failed to get inline page, err %d", err); 700 goto err_out; 701 } 702 get_page(fe->map.buf.page); 703 WRITE_ONCE(fe->pcl->compressed_pages[0], fe->map.buf.page); 704 fe->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE; 705 } else { 706 /* preload all compressed pages (can change mode if needed) */ 707 if (should_alloc_managed_pages(fe, sbi->opt.cache_strategy, 708 map->m_la)) 709 cache_strategy = TRYALLOC; 710 else 711 cache_strategy = DONTALLOC; 712 713 preload_compressed_pages(fe, MNGD_MAPPING(sbi), 714 cache_strategy, pagepool); 715 } 716 717 hitted: 718 /* 719 * Ensure the current partial page belongs to this submit chain rather 720 * than other concurrent submit chains or the noio(bypass) chain since 721 * those chains are handled asynchronously thus the page cannot be used 722 * for inplace I/O or pagevec (should be processed in strict order.) 723 */ 724 tight &= (fe->mode >= COLLECT_PRIMARY_HOOKED && 725 fe->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE); 726 727 cur = end - min_t(unsigned int, offset + end - map->m_la, end); 728 if (!(map->m_flags & EROFS_MAP_MAPPED)) { 729 zero_user_segment(page, cur, end); 730 goto next_part; 731 } 732 733 /* let's derive page type */ 734 page_type = cur ? Z_EROFS_VLE_PAGE_TYPE_HEAD : 735 (!spiltted ? Z_EROFS_PAGE_TYPE_EXCLUSIVE : 736 (tight ? Z_EROFS_PAGE_TYPE_EXCLUSIVE : 737 Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED)); 738 739 if (cur) 740 tight &= (fe->mode >= COLLECT_PRIMARY_FOLLOWED); 741 742 retry: 743 err = z_erofs_attach_page(fe, page, page_type, 744 fe->mode >= COLLECT_PRIMARY_FOLLOWED); 745 /* should allocate an additional short-lived page for pagevec */ 746 if (err == -EAGAIN) { 747 struct page *const newpage = 748 alloc_page(GFP_NOFS | __GFP_NOFAIL); 749 750 set_page_private(newpage, Z_EROFS_SHORTLIVED_PAGE); 751 err = z_erofs_attach_page(fe, newpage, 752 Z_EROFS_PAGE_TYPE_EXCLUSIVE, true); 753 if (!err) 754 goto retry; 755 } 756 757 if (err) 758 goto err_out; 759 760 index = page->index - (map->m_la >> PAGE_SHIFT); 761 762 z_erofs_onlinepage_fixup(page, index, true); 763 764 /* bump up the number of spiltted parts of a page */ 765 ++spiltted; 766 /* also update nr_pages */ 767 fe->cl->nr_pages = max_t(pgoff_t, fe->cl->nr_pages, index + 1); 768 next_part: 769 /* can be used for verification */ 770 map->m_llen = offset + cur - map->m_la; 771 772 end = cur; 773 if (end > 0) 774 goto repeat; 775 776 out: 777 z_erofs_onlinepage_endio(page); 778 779 erofs_dbg("%s, finish page: %pK spiltted: %u map->m_llen %llu", 780 __func__, page, spiltted, map->m_llen); 781 return err; 782 783 /* if some error occurred while processing this page */ 784 err_out: 785 SetPageError(page); 786 goto out; 787 } 788 789 static bool z_erofs_get_sync_decompress_policy(struct erofs_sb_info *sbi, 790 unsigned int readahead_pages) 791 { 792 /* auto: enable for readpage, disable for readahead */ 793 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) && 794 !readahead_pages) 795 return true; 796 797 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_FORCE_ON) && 798 (readahead_pages <= sbi->opt.max_sync_decompress_pages)) 799 return true; 800 801 return false; 802 } 803 804 static bool z_erofs_page_is_invalidated(struct page *page) 805 { 806 return !page->mapping && !z_erofs_is_shortlived_page(page); 807 } 808 809 static int z_erofs_decompress_pcluster(struct super_block *sb, 810 struct z_erofs_pcluster *pcl, 811 struct page **pagepool) 812 { 813 struct erofs_sb_info *const sbi = EROFS_SB(sb); 814 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 815 struct z_erofs_pagevec_ctor ctor; 816 unsigned int i, inputsize, outputsize, llen, nr_pages; 817 struct page *pages_onstack[Z_EROFS_VMAP_ONSTACK_PAGES]; 818 struct page **pages, **compressed_pages, *page; 819 820 enum z_erofs_page_type page_type; 821 bool overlapped, partial; 822 struct z_erofs_collection *cl; 823 int err; 824 825 might_sleep(); 826 cl = z_erofs_primarycollection(pcl); 827 DBG_BUGON(!READ_ONCE(cl->nr_pages)); 828 829 mutex_lock(&cl->lock); 830 nr_pages = cl->nr_pages; 831 832 if (nr_pages <= Z_EROFS_VMAP_ONSTACK_PAGES) { 833 pages = pages_onstack; 834 } else if (nr_pages <= Z_EROFS_VMAP_GLOBAL_PAGES && 835 mutex_trylock(&z_pagemap_global_lock)) { 836 pages = z_pagemap_global; 837 } else { 838 gfp_t gfp_flags = GFP_KERNEL; 839 840 if (nr_pages > Z_EROFS_VMAP_GLOBAL_PAGES) 841 gfp_flags |= __GFP_NOFAIL; 842 843 pages = kvmalloc_array(nr_pages, sizeof(struct page *), 844 gfp_flags); 845 846 /* fallback to global pagemap for the lowmem scenario */ 847 if (!pages) { 848 mutex_lock(&z_pagemap_global_lock); 849 pages = z_pagemap_global; 850 } 851 } 852 853 for (i = 0; i < nr_pages; ++i) 854 pages[i] = NULL; 855 856 err = 0; 857 z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_NR_INLINE_PAGEVECS, 858 cl->pagevec, 0); 859 860 for (i = 0; i < cl->vcnt; ++i) { 861 unsigned int pagenr; 862 863 page = z_erofs_pagevec_dequeue(&ctor, &page_type); 864 865 /* all pages in pagevec ought to be valid */ 866 DBG_BUGON(!page); 867 DBG_BUGON(z_erofs_page_is_invalidated(page)); 868 869 if (z_erofs_put_shortlivedpage(pagepool, page)) 870 continue; 871 872 if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD) 873 pagenr = 0; 874 else 875 pagenr = z_erofs_onlinepage_index(page); 876 877 DBG_BUGON(pagenr >= nr_pages); 878 879 /* 880 * currently EROFS doesn't support multiref(dedup), 881 * so here erroring out one multiref page. 882 */ 883 if (pages[pagenr]) { 884 DBG_BUGON(1); 885 SetPageError(pages[pagenr]); 886 z_erofs_onlinepage_endio(pages[pagenr]); 887 err = -EFSCORRUPTED; 888 } 889 pages[pagenr] = page; 890 } 891 z_erofs_pagevec_ctor_exit(&ctor, true); 892 893 overlapped = false; 894 compressed_pages = pcl->compressed_pages; 895 896 for (i = 0; i < pclusterpages; ++i) { 897 unsigned int pagenr; 898 899 page = compressed_pages[i]; 900 /* all compressed pages ought to be valid */ 901 DBG_BUGON(!page); 902 903 if (z_erofs_is_inline_pcluster(pcl)) { 904 if (!PageUptodate(page)) 905 err = -EIO; 906 continue; 907 } 908 909 DBG_BUGON(z_erofs_page_is_invalidated(page)); 910 if (!z_erofs_is_shortlived_page(page)) { 911 if (erofs_page_is_managed(sbi, page)) { 912 if (!PageUptodate(page)) 913 err = -EIO; 914 continue; 915 } 916 917 /* 918 * only if non-head page can be selected 919 * for inplace decompression 920 */ 921 pagenr = z_erofs_onlinepage_index(page); 922 923 DBG_BUGON(pagenr >= nr_pages); 924 if (pages[pagenr]) { 925 DBG_BUGON(1); 926 SetPageError(pages[pagenr]); 927 z_erofs_onlinepage_endio(pages[pagenr]); 928 err = -EFSCORRUPTED; 929 } 930 pages[pagenr] = page; 931 932 overlapped = true; 933 } 934 935 /* PG_error needs checking for all non-managed pages */ 936 if (PageError(page)) { 937 DBG_BUGON(PageUptodate(page)); 938 err = -EIO; 939 } 940 } 941 942 if (err) 943 goto out; 944 945 llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT; 946 if (nr_pages << PAGE_SHIFT >= cl->pageofs + llen) { 947 outputsize = llen; 948 partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH); 949 } else { 950 outputsize = (nr_pages << PAGE_SHIFT) - cl->pageofs; 951 partial = true; 952 } 953 954 if (z_erofs_is_inline_pcluster(pcl)) 955 inputsize = pcl->tailpacking_size; 956 else 957 inputsize = pclusterpages * PAGE_SIZE; 958 959 err = z_erofs_decompress(&(struct z_erofs_decompress_req) { 960 .sb = sb, 961 .in = compressed_pages, 962 .out = pages, 963 .pageofs_in = pcl->pageofs_in, 964 .pageofs_out = cl->pageofs, 965 .inputsize = inputsize, 966 .outputsize = outputsize, 967 .alg = pcl->algorithmformat, 968 .inplace_io = overlapped, 969 .partial_decoding = partial 970 }, pagepool); 971 972 out: 973 /* must handle all compressed pages before actual file pages */ 974 if (z_erofs_is_inline_pcluster(pcl)) { 975 page = compressed_pages[0]; 976 WRITE_ONCE(compressed_pages[0], NULL); 977 put_page(page); 978 } else { 979 for (i = 0; i < pclusterpages; ++i) { 980 page = compressed_pages[i]; 981 982 if (erofs_page_is_managed(sbi, page)) 983 continue; 984 985 /* recycle all individual short-lived pages */ 986 (void)z_erofs_put_shortlivedpage(pagepool, page); 987 WRITE_ONCE(compressed_pages[i], NULL); 988 } 989 } 990 991 for (i = 0; i < nr_pages; ++i) { 992 page = pages[i]; 993 if (!page) 994 continue; 995 996 DBG_BUGON(z_erofs_page_is_invalidated(page)); 997 998 /* recycle all individual short-lived pages */ 999 if (z_erofs_put_shortlivedpage(pagepool, page)) 1000 continue; 1001 1002 if (err < 0) 1003 SetPageError(page); 1004 1005 z_erofs_onlinepage_endio(page); 1006 } 1007 1008 if (pages == z_pagemap_global) 1009 mutex_unlock(&z_pagemap_global_lock); 1010 else if (pages != pages_onstack) 1011 kvfree(pages); 1012 1013 cl->nr_pages = 0; 1014 cl->vcnt = 0; 1015 1016 /* all cl locks MUST be taken before the following line */ 1017 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL); 1018 1019 /* all cl locks SHOULD be released right now */ 1020 mutex_unlock(&cl->lock); 1021 1022 z_erofs_collection_put(cl); 1023 return err; 1024 } 1025 1026 static void z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, 1027 struct page **pagepool) 1028 { 1029 z_erofs_next_pcluster_t owned = io->head; 1030 1031 while (owned != Z_EROFS_PCLUSTER_TAIL_CLOSED) { 1032 struct z_erofs_pcluster *pcl; 1033 1034 /* no possible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */ 1035 DBG_BUGON(owned == Z_EROFS_PCLUSTER_TAIL); 1036 1037 /* no possible that 'owned' equals NULL */ 1038 DBG_BUGON(owned == Z_EROFS_PCLUSTER_NIL); 1039 1040 pcl = container_of(owned, struct z_erofs_pcluster, next); 1041 owned = READ_ONCE(pcl->next); 1042 1043 z_erofs_decompress_pcluster(io->sb, pcl, pagepool); 1044 } 1045 } 1046 1047 static void z_erofs_decompressqueue_work(struct work_struct *work) 1048 { 1049 struct z_erofs_decompressqueue *bgq = 1050 container_of(work, struct z_erofs_decompressqueue, u.work); 1051 struct page *pagepool = NULL; 1052 1053 DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1054 z_erofs_decompress_queue(bgq, &pagepool); 1055 1056 erofs_release_pages(&pagepool); 1057 kvfree(bgq); 1058 } 1059 1060 static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, 1061 bool sync, int bios) 1062 { 1063 struct erofs_sb_info *const sbi = EROFS_SB(io->sb); 1064 1065 /* wake up the caller thread for sync decompression */ 1066 if (sync) { 1067 unsigned long flags; 1068 1069 spin_lock_irqsave(&io->u.wait.lock, flags); 1070 if (!atomic_add_return(bios, &io->pending_bios)) 1071 wake_up_locked(&io->u.wait); 1072 spin_unlock_irqrestore(&io->u.wait.lock, flags); 1073 return; 1074 } 1075 1076 if (atomic_add_return(bios, &io->pending_bios)) 1077 return; 1078 /* Use workqueue and sync decompression for atomic contexts only */ 1079 if (in_atomic() || irqs_disabled()) { 1080 queue_work(z_erofs_workqueue, &io->u.work); 1081 /* enable sync decompression for readahead */ 1082 if (sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) 1083 sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; 1084 return; 1085 } 1086 z_erofs_decompressqueue_work(&io->u.work); 1087 } 1088 1089 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, 1090 unsigned int nr, 1091 struct page **pagepool, 1092 struct address_space *mc, 1093 gfp_t gfp) 1094 { 1095 const pgoff_t index = pcl->obj.index; 1096 bool tocache = false; 1097 1098 struct address_space *mapping; 1099 struct page *oldpage, *page; 1100 1101 compressed_page_t t; 1102 int justfound; 1103 1104 repeat: 1105 page = READ_ONCE(pcl->compressed_pages[nr]); 1106 oldpage = page; 1107 1108 if (!page) 1109 goto out_allocpage; 1110 1111 /* process the target tagged pointer */ 1112 t = tagptr_init(compressed_page_t, page); 1113 justfound = tagptr_unfold_tags(t); 1114 page = tagptr_unfold_ptr(t); 1115 1116 /* 1117 * preallocated cached pages, which is used to avoid direct reclaim 1118 * otherwise, it will go inplace I/O path instead. 1119 */ 1120 if (page->private == Z_EROFS_PREALLOCATED_PAGE) { 1121 WRITE_ONCE(pcl->compressed_pages[nr], page); 1122 set_page_private(page, 0); 1123 tocache = true; 1124 goto out_tocache; 1125 } 1126 mapping = READ_ONCE(page->mapping); 1127 1128 /* 1129 * file-backed online pages in plcuster are all locked steady, 1130 * therefore it is impossible for `mapping' to be NULL. 1131 */ 1132 if (mapping && mapping != mc) 1133 /* ought to be unmanaged pages */ 1134 goto out; 1135 1136 /* directly return for shortlived page as well */ 1137 if (z_erofs_is_shortlived_page(page)) 1138 goto out; 1139 1140 lock_page(page); 1141 1142 /* only true if page reclaim goes wrong, should never happen */ 1143 DBG_BUGON(justfound && PagePrivate(page)); 1144 1145 /* the page is still in manage cache */ 1146 if (page->mapping == mc) { 1147 WRITE_ONCE(pcl->compressed_pages[nr], page); 1148 1149 ClearPageError(page); 1150 if (!PagePrivate(page)) { 1151 /* 1152 * impossible to be !PagePrivate(page) for 1153 * the current restriction as well if 1154 * the page is already in compressed_pages[]. 1155 */ 1156 DBG_BUGON(!justfound); 1157 1158 justfound = 0; 1159 set_page_private(page, (unsigned long)pcl); 1160 SetPagePrivate(page); 1161 } 1162 1163 /* no need to submit io if it is already up-to-date */ 1164 if (PageUptodate(page)) { 1165 unlock_page(page); 1166 page = NULL; 1167 } 1168 goto out; 1169 } 1170 1171 /* 1172 * the managed page has been truncated, it's unsafe to 1173 * reuse this one, let's allocate a new cache-managed page. 1174 */ 1175 DBG_BUGON(page->mapping); 1176 DBG_BUGON(!justfound); 1177 1178 tocache = true; 1179 unlock_page(page); 1180 put_page(page); 1181 out_allocpage: 1182 page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL); 1183 if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) { 1184 erofs_pagepool_add(pagepool, page); 1185 cond_resched(); 1186 goto repeat; 1187 } 1188 out_tocache: 1189 if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) { 1190 /* turn into temporary page if fails (1 ref) */ 1191 set_page_private(page, Z_EROFS_SHORTLIVED_PAGE); 1192 goto out; 1193 } 1194 attach_page_private(page, pcl); 1195 /* drop a refcount added by allocpage (then we have 2 refs here) */ 1196 put_page(page); 1197 1198 out: /* the only exit (for tracing and debugging) */ 1199 return page; 1200 } 1201 1202 static struct z_erofs_decompressqueue * 1203 jobqueue_init(struct super_block *sb, 1204 struct z_erofs_decompressqueue *fgq, bool *fg) 1205 { 1206 struct z_erofs_decompressqueue *q; 1207 1208 if (fg && !*fg) { 1209 q = kvzalloc(sizeof(*q), GFP_KERNEL | __GFP_NOWARN); 1210 if (!q) { 1211 *fg = true; 1212 goto fg_out; 1213 } 1214 INIT_WORK(&q->u.work, z_erofs_decompressqueue_work); 1215 } else { 1216 fg_out: 1217 q = fgq; 1218 init_waitqueue_head(&fgq->u.wait); 1219 atomic_set(&fgq->pending_bios, 0); 1220 } 1221 q->sb = sb; 1222 q->head = Z_EROFS_PCLUSTER_TAIL_CLOSED; 1223 return q; 1224 } 1225 1226 /* define decompression jobqueue types */ 1227 enum { 1228 JQ_BYPASS, 1229 JQ_SUBMIT, 1230 NR_JOBQUEUES, 1231 }; 1232 1233 static void *jobqueueset_init(struct super_block *sb, 1234 struct z_erofs_decompressqueue *q[], 1235 struct z_erofs_decompressqueue *fgq, bool *fg) 1236 { 1237 /* 1238 * if managed cache is enabled, bypass jobqueue is needed, 1239 * no need to read from device for all pclusters in this queue. 1240 */ 1241 q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL); 1242 q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, fg); 1243 1244 return tagptr_cast_ptr(tagptr_fold(tagptr1_t, q[JQ_SUBMIT], *fg)); 1245 } 1246 1247 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, 1248 z_erofs_next_pcluster_t qtail[], 1249 z_erofs_next_pcluster_t owned_head) 1250 { 1251 z_erofs_next_pcluster_t *const submit_qtail = qtail[JQ_SUBMIT]; 1252 z_erofs_next_pcluster_t *const bypass_qtail = qtail[JQ_BYPASS]; 1253 1254 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1255 if (owned_head == Z_EROFS_PCLUSTER_TAIL) 1256 owned_head = Z_EROFS_PCLUSTER_TAIL_CLOSED; 1257 1258 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED); 1259 1260 WRITE_ONCE(*submit_qtail, owned_head); 1261 WRITE_ONCE(*bypass_qtail, &pcl->next); 1262 1263 qtail[JQ_BYPASS] = &pcl->next; 1264 } 1265 1266 static void z_erofs_decompressqueue_endio(struct bio *bio) 1267 { 1268 tagptr1_t t = tagptr_init(tagptr1_t, bio->bi_private); 1269 struct z_erofs_decompressqueue *q = tagptr_unfold_ptr(t); 1270 blk_status_t err = bio->bi_status; 1271 struct bio_vec *bvec; 1272 struct bvec_iter_all iter_all; 1273 1274 bio_for_each_segment_all(bvec, bio, iter_all) { 1275 struct page *page = bvec->bv_page; 1276 1277 DBG_BUGON(PageUptodate(page)); 1278 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1279 1280 if (err) 1281 SetPageError(page); 1282 1283 if (erofs_page_is_managed(EROFS_SB(q->sb), page)) { 1284 if (!err) 1285 SetPageUptodate(page); 1286 unlock_page(page); 1287 } 1288 } 1289 z_erofs_decompress_kickoff(q, tagptr_unfold_tags(t), -1); 1290 bio_put(bio); 1291 } 1292 1293 static void z_erofs_submit_queue(struct super_block *sb, 1294 struct z_erofs_decompress_frontend *f, 1295 struct page **pagepool, 1296 struct z_erofs_decompressqueue *fgq, 1297 bool *force_fg) 1298 { 1299 struct erofs_sb_info *const sbi = EROFS_SB(sb); 1300 z_erofs_next_pcluster_t qtail[NR_JOBQUEUES]; 1301 struct z_erofs_decompressqueue *q[NR_JOBQUEUES]; 1302 void *bi_private; 1303 z_erofs_next_pcluster_t owned_head = f->owned_head; 1304 /* bio is NULL initially, so no need to initialize last_{index,bdev} */ 1305 pgoff_t last_index; 1306 struct block_device *last_bdev; 1307 unsigned int nr_bios = 0; 1308 struct bio *bio = NULL; 1309 1310 bi_private = jobqueueset_init(sb, q, fgq, force_fg); 1311 qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; 1312 qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head; 1313 1314 /* by default, all need io submission */ 1315 q[JQ_SUBMIT]->head = owned_head; 1316 1317 do { 1318 struct erofs_map_dev mdev; 1319 struct z_erofs_pcluster *pcl; 1320 pgoff_t cur, end; 1321 unsigned int i = 0; 1322 bool bypass = true; 1323 1324 /* no possible 'owned_head' equals the following */ 1325 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1326 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_NIL); 1327 1328 pcl = container_of(owned_head, struct z_erofs_pcluster, next); 1329 1330 /* close the main owned chain at first */ 1331 owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, 1332 Z_EROFS_PCLUSTER_TAIL_CLOSED); 1333 if (z_erofs_is_inline_pcluster(pcl)) { 1334 move_to_bypass_jobqueue(pcl, qtail, owned_head); 1335 continue; 1336 } 1337 1338 /* no device id here, thus it will always succeed */ 1339 mdev = (struct erofs_map_dev) { 1340 .m_pa = blknr_to_addr(pcl->obj.index), 1341 }; 1342 (void)erofs_map_dev(sb, &mdev); 1343 1344 cur = erofs_blknr(mdev.m_pa); 1345 end = cur + pcl->pclusterpages; 1346 1347 do { 1348 struct page *page; 1349 1350 page = pickup_page_for_submission(pcl, i++, pagepool, 1351 MNGD_MAPPING(sbi), 1352 GFP_NOFS); 1353 if (!page) 1354 continue; 1355 1356 if (bio && (cur != last_index + 1 || 1357 last_bdev != mdev.m_bdev)) { 1358 submit_bio_retry: 1359 submit_bio(bio); 1360 bio = NULL; 1361 } 1362 1363 if (!bio) { 1364 bio = bio_alloc(GFP_NOIO, BIO_MAX_VECS); 1365 bio->bi_end_io = z_erofs_decompressqueue_endio; 1366 1367 bio_set_dev(bio, mdev.m_bdev); 1368 last_bdev = mdev.m_bdev; 1369 bio->bi_iter.bi_sector = (sector_t)cur << 1370 LOG_SECTORS_PER_BLOCK; 1371 bio->bi_private = bi_private; 1372 bio->bi_opf = REQ_OP_READ; 1373 if (f->readahead) 1374 bio->bi_opf |= REQ_RAHEAD; 1375 ++nr_bios; 1376 } 1377 1378 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) 1379 goto submit_bio_retry; 1380 1381 last_index = cur; 1382 bypass = false; 1383 } while (++cur < end); 1384 1385 if (!bypass) 1386 qtail[JQ_SUBMIT] = &pcl->next; 1387 else 1388 move_to_bypass_jobqueue(pcl, qtail, owned_head); 1389 } while (owned_head != Z_EROFS_PCLUSTER_TAIL); 1390 1391 if (bio) 1392 submit_bio(bio); 1393 1394 /* 1395 * although background is preferred, no one is pending for submission. 1396 * don't issue workqueue for decompression but drop it directly instead. 1397 */ 1398 if (!*force_fg && !nr_bios) { 1399 kvfree(q[JQ_SUBMIT]); 1400 return; 1401 } 1402 z_erofs_decompress_kickoff(q[JQ_SUBMIT], *force_fg, nr_bios); 1403 } 1404 1405 static void z_erofs_runqueue(struct super_block *sb, 1406 struct z_erofs_decompress_frontend *f, 1407 struct page **pagepool, bool force_fg) 1408 { 1409 struct z_erofs_decompressqueue io[NR_JOBQUEUES]; 1410 1411 if (f->owned_head == Z_EROFS_PCLUSTER_TAIL) 1412 return; 1413 z_erofs_submit_queue(sb, f, pagepool, io, &force_fg); 1414 1415 /* handle bypass queue (no i/o pclusters) immediately */ 1416 z_erofs_decompress_queue(&io[JQ_BYPASS], pagepool); 1417 1418 if (!force_fg) 1419 return; 1420 1421 /* wait until all bios are completed */ 1422 io_wait_event(io[JQ_SUBMIT].u.wait, 1423 !atomic_read(&io[JQ_SUBMIT].pending_bios)); 1424 1425 /* handle synchronous decompress queue in the caller context */ 1426 z_erofs_decompress_queue(&io[JQ_SUBMIT], pagepool); 1427 } 1428 1429 /* 1430 * Since partial uptodate is still unimplemented for now, we have to use 1431 * approximate readmore strategies as a start. 1432 */ 1433 static void z_erofs_pcluster_readmore(struct z_erofs_decompress_frontend *f, 1434 struct readahead_control *rac, 1435 erofs_off_t end, 1436 struct page **pagepool, 1437 bool backmost) 1438 { 1439 struct inode *inode = f->inode; 1440 struct erofs_map_blocks *map = &f->map; 1441 erofs_off_t cur; 1442 int err; 1443 1444 if (backmost) { 1445 map->m_la = end; 1446 err = z_erofs_map_blocks_iter(inode, map, 1447 EROFS_GET_BLOCKS_READMORE); 1448 if (err) 1449 return; 1450 1451 /* expend ra for the trailing edge if readahead */ 1452 if (rac) { 1453 loff_t newstart = readahead_pos(rac); 1454 1455 cur = round_up(map->m_la + map->m_llen, PAGE_SIZE); 1456 readahead_expand(rac, newstart, cur - newstart); 1457 return; 1458 } 1459 end = round_up(end, PAGE_SIZE); 1460 } else { 1461 end = round_up(map->m_la, PAGE_SIZE); 1462 1463 if (!map->m_llen) 1464 return; 1465 } 1466 1467 cur = map->m_la + map->m_llen - 1; 1468 while (cur >= end) { 1469 pgoff_t index = cur >> PAGE_SHIFT; 1470 struct page *page; 1471 1472 page = erofs_grab_cache_page_nowait(inode->i_mapping, index); 1473 if (!page) 1474 goto skip; 1475 1476 if (PageUptodate(page)) { 1477 unlock_page(page); 1478 put_page(page); 1479 goto skip; 1480 } 1481 1482 err = z_erofs_do_read_page(f, page, pagepool); 1483 if (err) 1484 erofs_err(inode->i_sb, 1485 "readmore error at page %lu @ nid %llu", 1486 index, EROFS_I(inode)->nid); 1487 put_page(page); 1488 skip: 1489 if (cur < PAGE_SIZE) 1490 break; 1491 cur = (index << PAGE_SHIFT) - 1; 1492 } 1493 } 1494 1495 static int z_erofs_readpage(struct file *file, struct page *page) 1496 { 1497 struct inode *const inode = page->mapping->host; 1498 struct erofs_sb_info *const sbi = EROFS_I_SB(inode); 1499 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); 1500 struct page *pagepool = NULL; 1501 int err; 1502 1503 trace_erofs_readpage(page, false); 1504 f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT; 1505 1506 z_erofs_pcluster_readmore(&f, NULL, f.headoffset + PAGE_SIZE - 1, 1507 &pagepool, true); 1508 err = z_erofs_do_read_page(&f, page, &pagepool); 1509 z_erofs_pcluster_readmore(&f, NULL, 0, &pagepool, false); 1510 1511 (void)z_erofs_collector_end(&f); 1512 1513 /* if some compressed cluster ready, need submit them anyway */ 1514 z_erofs_runqueue(inode->i_sb, &f, &pagepool, 1515 z_erofs_get_sync_decompress_policy(sbi, 0)); 1516 1517 if (err) 1518 erofs_err(inode->i_sb, "failed to read, err [%d]", err); 1519 1520 erofs_put_metabuf(&f.map.buf); 1521 erofs_release_pages(&pagepool); 1522 return err; 1523 } 1524 1525 static void z_erofs_readahead(struct readahead_control *rac) 1526 { 1527 struct inode *const inode = rac->mapping->host; 1528 struct erofs_sb_info *const sbi = EROFS_I_SB(inode); 1529 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); 1530 struct page *pagepool = NULL, *head = NULL, *page; 1531 unsigned int nr_pages; 1532 1533 f.readahead = true; 1534 f.headoffset = readahead_pos(rac); 1535 1536 z_erofs_pcluster_readmore(&f, rac, f.headoffset + 1537 readahead_length(rac) - 1, &pagepool, true); 1538 nr_pages = readahead_count(rac); 1539 trace_erofs_readpages(inode, readahead_index(rac), nr_pages, false); 1540 1541 while ((page = readahead_page(rac))) { 1542 set_page_private(page, (unsigned long)head); 1543 head = page; 1544 } 1545 1546 while (head) { 1547 struct page *page = head; 1548 int err; 1549 1550 /* traversal in reverse order */ 1551 head = (void *)page_private(page); 1552 1553 err = z_erofs_do_read_page(&f, page, &pagepool); 1554 if (err) 1555 erofs_err(inode->i_sb, 1556 "readahead error at page %lu @ nid %llu", 1557 page->index, EROFS_I(inode)->nid); 1558 put_page(page); 1559 } 1560 z_erofs_pcluster_readmore(&f, rac, 0, &pagepool, false); 1561 (void)z_erofs_collector_end(&f); 1562 1563 z_erofs_runqueue(inode->i_sb, &f, &pagepool, 1564 z_erofs_get_sync_decompress_policy(sbi, nr_pages)); 1565 erofs_put_metabuf(&f.map.buf); 1566 erofs_release_pages(&pagepool); 1567 } 1568 1569 const struct address_space_operations z_erofs_aops = { 1570 .readpage = z_erofs_readpage, 1571 .readahead = z_erofs_readahead, 1572 }; 1573