1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 * Copyright (C) 2022 Alibaba Cloud 6 */ 7 #include "compress.h" 8 #include <linux/psi.h> 9 #include <linux/cpuhotplug.h> 10 #include <trace/events/erofs.h> 11 12 #define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE) 13 #define Z_EROFS_INLINE_BVECS 2 14 15 /* 16 * let's leave a type here in case of introducing 17 * another tagged pointer later. 18 */ 19 typedef void *z_erofs_next_pcluster_t; 20 21 struct z_erofs_bvec { 22 struct page *page; 23 int offset; 24 unsigned int end; 25 }; 26 27 #define __Z_EROFS_BVSET(name, total) \ 28 struct name { \ 29 /* point to the next page which contains the following bvecs */ \ 30 struct page *nextpage; \ 31 struct z_erofs_bvec bvec[total]; \ 32 } 33 __Z_EROFS_BVSET(z_erofs_bvset,); 34 __Z_EROFS_BVSET(z_erofs_bvset_inline, Z_EROFS_INLINE_BVECS); 35 36 /* 37 * Structure fields follow one of the following exclusion rules. 38 * 39 * I: Modifiable by initialization/destruction paths and read-only 40 * for everyone else; 41 * 42 * L: Field should be protected by the pcluster lock; 43 * 44 * A: Field should be accessed / updated in atomic for parallelized code. 45 */ 46 struct z_erofs_pcluster { 47 struct erofs_workgroup obj; 48 struct mutex lock; 49 50 /* A: point to next chained pcluster or TAILs */ 51 z_erofs_next_pcluster_t next; 52 53 /* L: the maximum decompression size of this round */ 54 unsigned int length; 55 56 /* L: total number of bvecs */ 57 unsigned int vcnt; 58 59 /* I: page offset of start position of decompression */ 60 unsigned short pageofs_out; 61 62 /* I: page offset of inline compressed data */ 63 unsigned short pageofs_in; 64 65 union { 66 /* L: inline a certain number of bvec for bootstrap */ 67 struct z_erofs_bvset_inline bvset; 68 69 /* I: can be used to free the pcluster by RCU. */ 70 struct rcu_head rcu; 71 }; 72 73 union { 74 /* I: physical cluster size in pages */ 75 unsigned short pclusterpages; 76 77 /* I: tailpacking inline compressed size */ 78 unsigned short tailpacking_size; 79 }; 80 81 /* I: compression algorithm format */ 82 unsigned char algorithmformat; 83 84 /* L: whether partial decompression or not */ 85 bool partial; 86 87 /* L: indicate several pageofs_outs or not */ 88 bool multibases; 89 90 /* A: compressed bvecs (can be cached or inplaced pages) */ 91 struct z_erofs_bvec compressed_bvecs[]; 92 }; 93 94 /* let's avoid the valid 32-bit kernel addresses */ 95 96 /* the chained workgroup has't submitted io (still open) */ 97 #define Z_EROFS_PCLUSTER_TAIL ((void *)0x5F0ECAFE) 98 /* the chained workgroup has already submitted io */ 99 #define Z_EROFS_PCLUSTER_TAIL_CLOSED ((void *)0x5F0EDEAD) 100 101 #define Z_EROFS_PCLUSTER_NIL (NULL) 102 103 struct z_erofs_decompressqueue { 104 struct super_block *sb; 105 atomic_t pending_bios; 106 z_erofs_next_pcluster_t head; 107 108 union { 109 struct completion done; 110 struct work_struct work; 111 struct kthread_work kthread_work; 112 } u; 113 bool eio, sync; 114 }; 115 116 static inline bool z_erofs_is_inline_pcluster(struct z_erofs_pcluster *pcl) 117 { 118 return !pcl->obj.index; 119 } 120 121 static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl) 122 { 123 if (z_erofs_is_inline_pcluster(pcl)) 124 return 1; 125 return pcl->pclusterpages; 126 } 127 128 /* 129 * bit 30: I/O error occurred on this page 130 * bit 0 - 29: remaining parts to complete this page 131 */ 132 #define Z_EROFS_PAGE_EIO (1 << 30) 133 134 static inline void z_erofs_onlinepage_init(struct page *page) 135 { 136 union { 137 atomic_t o; 138 unsigned long v; 139 } u = { .o = ATOMIC_INIT(1) }; 140 141 set_page_private(page, u.v); 142 smp_wmb(); 143 SetPagePrivate(page); 144 } 145 146 static inline void z_erofs_onlinepage_split(struct page *page) 147 { 148 atomic_inc((atomic_t *)&page->private); 149 } 150 151 static inline void z_erofs_page_mark_eio(struct page *page) 152 { 153 int orig; 154 155 do { 156 orig = atomic_read((atomic_t *)&page->private); 157 } while (atomic_cmpxchg((atomic_t *)&page->private, orig, 158 orig | Z_EROFS_PAGE_EIO) != orig); 159 } 160 161 static inline void z_erofs_onlinepage_endio(struct page *page) 162 { 163 unsigned int v; 164 165 DBG_BUGON(!PagePrivate(page)); 166 v = atomic_dec_return((atomic_t *)&page->private); 167 if (!(v & ~Z_EROFS_PAGE_EIO)) { 168 set_page_private(page, 0); 169 ClearPagePrivate(page); 170 if (!(v & Z_EROFS_PAGE_EIO)) 171 SetPageUptodate(page); 172 unlock_page(page); 173 } 174 } 175 176 #define Z_EROFS_ONSTACK_PAGES 32 177 178 /* 179 * since pclustersize is variable for big pcluster feature, introduce slab 180 * pools implementation for different pcluster sizes. 181 */ 182 struct z_erofs_pcluster_slab { 183 struct kmem_cache *slab; 184 unsigned int maxpages; 185 char name[48]; 186 }; 187 188 #define _PCLP(n) { .maxpages = n } 189 190 static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { 191 _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), 192 _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES) 193 }; 194 195 struct z_erofs_bvec_iter { 196 struct page *bvpage; 197 struct z_erofs_bvset *bvset; 198 unsigned int nr, cur; 199 }; 200 201 static struct page *z_erofs_bvec_iter_end(struct z_erofs_bvec_iter *iter) 202 { 203 if (iter->bvpage) 204 kunmap_local(iter->bvset); 205 return iter->bvpage; 206 } 207 208 static struct page *z_erofs_bvset_flip(struct z_erofs_bvec_iter *iter) 209 { 210 unsigned long base = (unsigned long)((struct z_erofs_bvset *)0)->bvec; 211 /* have to access nextpage in advance, otherwise it will be unmapped */ 212 struct page *nextpage = iter->bvset->nextpage; 213 struct page *oldpage; 214 215 DBG_BUGON(!nextpage); 216 oldpage = z_erofs_bvec_iter_end(iter); 217 iter->bvpage = nextpage; 218 iter->bvset = kmap_local_page(nextpage); 219 iter->nr = (PAGE_SIZE - base) / sizeof(struct z_erofs_bvec); 220 iter->cur = 0; 221 return oldpage; 222 } 223 224 static void z_erofs_bvec_iter_begin(struct z_erofs_bvec_iter *iter, 225 struct z_erofs_bvset_inline *bvset, 226 unsigned int bootstrap_nr, 227 unsigned int cur) 228 { 229 *iter = (struct z_erofs_bvec_iter) { 230 .nr = bootstrap_nr, 231 .bvset = (struct z_erofs_bvset *)bvset, 232 }; 233 234 while (cur > iter->nr) { 235 cur -= iter->nr; 236 z_erofs_bvset_flip(iter); 237 } 238 iter->cur = cur; 239 } 240 241 static int z_erofs_bvec_enqueue(struct z_erofs_bvec_iter *iter, 242 struct z_erofs_bvec *bvec, 243 struct page **candidate_bvpage, 244 struct page **pagepool) 245 { 246 if (iter->cur >= iter->nr) { 247 struct page *nextpage = *candidate_bvpage; 248 249 if (!nextpage) { 250 nextpage = erofs_allocpage(pagepool, GFP_NOFS); 251 if (!nextpage) 252 return -ENOMEM; 253 set_page_private(nextpage, Z_EROFS_SHORTLIVED_PAGE); 254 } 255 DBG_BUGON(iter->bvset->nextpage); 256 iter->bvset->nextpage = nextpage; 257 z_erofs_bvset_flip(iter); 258 259 iter->bvset->nextpage = NULL; 260 *candidate_bvpage = NULL; 261 } 262 iter->bvset->bvec[iter->cur++] = *bvec; 263 return 0; 264 } 265 266 static void z_erofs_bvec_dequeue(struct z_erofs_bvec_iter *iter, 267 struct z_erofs_bvec *bvec, 268 struct page **old_bvpage) 269 { 270 if (iter->cur == iter->nr) 271 *old_bvpage = z_erofs_bvset_flip(iter); 272 else 273 *old_bvpage = NULL; 274 *bvec = iter->bvset->bvec[iter->cur++]; 275 } 276 277 static void z_erofs_destroy_pcluster_pool(void) 278 { 279 int i; 280 281 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 282 if (!pcluster_pool[i].slab) 283 continue; 284 kmem_cache_destroy(pcluster_pool[i].slab); 285 pcluster_pool[i].slab = NULL; 286 } 287 } 288 289 static int z_erofs_create_pcluster_pool(void) 290 { 291 struct z_erofs_pcluster_slab *pcs; 292 struct z_erofs_pcluster *a; 293 unsigned int size; 294 295 for (pcs = pcluster_pool; 296 pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { 297 size = struct_size(a, compressed_bvecs, pcs->maxpages); 298 299 sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages); 300 pcs->slab = kmem_cache_create(pcs->name, size, 0, 301 SLAB_RECLAIM_ACCOUNT, NULL); 302 if (pcs->slab) 303 continue; 304 305 z_erofs_destroy_pcluster_pool(); 306 return -ENOMEM; 307 } 308 return 0; 309 } 310 311 static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int nrpages) 312 { 313 int i; 314 315 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 316 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; 317 struct z_erofs_pcluster *pcl; 318 319 if (nrpages > pcs->maxpages) 320 continue; 321 322 pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS); 323 if (!pcl) 324 return ERR_PTR(-ENOMEM); 325 pcl->pclusterpages = nrpages; 326 return pcl; 327 } 328 return ERR_PTR(-EINVAL); 329 } 330 331 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) 332 { 333 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 334 int i; 335 336 for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { 337 struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; 338 339 if (pclusterpages > pcs->maxpages) 340 continue; 341 342 kmem_cache_free(pcs->slab, pcl); 343 return; 344 } 345 DBG_BUGON(1); 346 } 347 348 static struct workqueue_struct *z_erofs_workqueue __read_mostly; 349 350 #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD 351 static struct kthread_worker __rcu **z_erofs_pcpu_workers; 352 353 static void erofs_destroy_percpu_workers(void) 354 { 355 struct kthread_worker *worker; 356 unsigned int cpu; 357 358 for_each_possible_cpu(cpu) { 359 worker = rcu_dereference_protected( 360 z_erofs_pcpu_workers[cpu], 1); 361 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL); 362 if (worker) 363 kthread_destroy_worker(worker); 364 } 365 kfree(z_erofs_pcpu_workers); 366 } 367 368 static struct kthread_worker *erofs_init_percpu_worker(int cpu) 369 { 370 struct kthread_worker *worker = 371 kthread_create_worker_on_cpu(cpu, 0, "erofs_worker/%u", cpu); 372 373 if (IS_ERR(worker)) 374 return worker; 375 if (IS_ENABLED(CONFIG_EROFS_FS_PCPU_KTHREAD_HIPRI)) 376 sched_set_fifo_low(worker->task); 377 return worker; 378 } 379 380 static int erofs_init_percpu_workers(void) 381 { 382 struct kthread_worker *worker; 383 unsigned int cpu; 384 385 z_erofs_pcpu_workers = kcalloc(num_possible_cpus(), 386 sizeof(struct kthread_worker *), GFP_ATOMIC); 387 if (!z_erofs_pcpu_workers) 388 return -ENOMEM; 389 390 for_each_online_cpu(cpu) { /* could miss cpu{off,on}line? */ 391 worker = erofs_init_percpu_worker(cpu); 392 if (!IS_ERR(worker)) 393 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker); 394 } 395 return 0; 396 } 397 #else 398 static inline void erofs_destroy_percpu_workers(void) {} 399 static inline int erofs_init_percpu_workers(void) { return 0; } 400 #endif 401 402 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_EROFS_FS_PCPU_KTHREAD) 403 static DEFINE_SPINLOCK(z_erofs_pcpu_worker_lock); 404 static enum cpuhp_state erofs_cpuhp_state; 405 406 static int erofs_cpu_online(unsigned int cpu) 407 { 408 struct kthread_worker *worker, *old; 409 410 worker = erofs_init_percpu_worker(cpu); 411 if (IS_ERR(worker)) 412 return PTR_ERR(worker); 413 414 spin_lock(&z_erofs_pcpu_worker_lock); 415 old = rcu_dereference_protected(z_erofs_pcpu_workers[cpu], 416 lockdep_is_held(&z_erofs_pcpu_worker_lock)); 417 if (!old) 418 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker); 419 spin_unlock(&z_erofs_pcpu_worker_lock); 420 if (old) 421 kthread_destroy_worker(worker); 422 return 0; 423 } 424 425 static int erofs_cpu_offline(unsigned int cpu) 426 { 427 struct kthread_worker *worker; 428 429 spin_lock(&z_erofs_pcpu_worker_lock); 430 worker = rcu_dereference_protected(z_erofs_pcpu_workers[cpu], 431 lockdep_is_held(&z_erofs_pcpu_worker_lock)); 432 rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL); 433 spin_unlock(&z_erofs_pcpu_worker_lock); 434 435 synchronize_rcu(); 436 if (worker) 437 kthread_destroy_worker(worker); 438 return 0; 439 } 440 441 static int erofs_cpu_hotplug_init(void) 442 { 443 int state; 444 445 state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 446 "fs/erofs:online", erofs_cpu_online, erofs_cpu_offline); 447 if (state < 0) 448 return state; 449 450 erofs_cpuhp_state = state; 451 return 0; 452 } 453 454 static void erofs_cpu_hotplug_destroy(void) 455 { 456 if (erofs_cpuhp_state) 457 cpuhp_remove_state_nocalls(erofs_cpuhp_state); 458 } 459 #else /* !CONFIG_HOTPLUG_CPU || !CONFIG_EROFS_FS_PCPU_KTHREAD */ 460 static inline int erofs_cpu_hotplug_init(void) { return 0; } 461 static inline void erofs_cpu_hotplug_destroy(void) {} 462 #endif 463 464 void z_erofs_exit_zip_subsystem(void) 465 { 466 erofs_cpu_hotplug_destroy(); 467 erofs_destroy_percpu_workers(); 468 destroy_workqueue(z_erofs_workqueue); 469 z_erofs_destroy_pcluster_pool(); 470 } 471 472 int __init z_erofs_init_zip_subsystem(void) 473 { 474 int err = z_erofs_create_pcluster_pool(); 475 476 if (err) 477 goto out_error_pcluster_pool; 478 479 z_erofs_workqueue = alloc_workqueue("erofs_worker", 480 WQ_UNBOUND | WQ_HIGHPRI, num_possible_cpus()); 481 if (!z_erofs_workqueue) { 482 err = -ENOMEM; 483 goto out_error_workqueue_init; 484 } 485 486 err = erofs_init_percpu_workers(); 487 if (err) 488 goto out_error_pcpu_worker; 489 490 err = erofs_cpu_hotplug_init(); 491 if (err < 0) 492 goto out_error_cpuhp_init; 493 return err; 494 495 out_error_cpuhp_init: 496 erofs_destroy_percpu_workers(); 497 out_error_pcpu_worker: 498 destroy_workqueue(z_erofs_workqueue); 499 out_error_workqueue_init: 500 z_erofs_destroy_pcluster_pool(); 501 out_error_pcluster_pool: 502 return err; 503 } 504 505 enum z_erofs_pclustermode { 506 Z_EROFS_PCLUSTER_INFLIGHT, 507 /* 508 * The current pclusters was the tail of an exist chain, in addition 509 * that the previous processed chained pclusters are all decided to 510 * be hooked up to it. 511 * A new chain will be created for the remaining pclusters which are 512 * not processed yet, so different from Z_EROFS_PCLUSTER_FOLLOWED, 513 * the next pcluster cannot reuse the whole page safely for inplace I/O 514 * in the following scenario: 515 * ________________________________________________________________ 516 * | tail (partial) page | head (partial) page | 517 * | (belongs to the next pcl) | (belongs to the current pcl) | 518 * |_______PCLUSTER_FOLLOWED______|________PCLUSTER_HOOKED__________| 519 */ 520 Z_EROFS_PCLUSTER_HOOKED, 521 /* 522 * a weak form of Z_EROFS_PCLUSTER_FOLLOWED, the difference is that it 523 * could be dispatched into bypass queue later due to uptodated managed 524 * pages. All related online pages cannot be reused for inplace I/O (or 525 * bvpage) since it can be directly decoded without I/O submission. 526 */ 527 Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE, 528 /* 529 * The current collection has been linked with the owned chain, and 530 * could also be linked with the remaining collections, which means 531 * if the processing page is the tail page of the collection, thus 532 * the current collection can safely use the whole page (since 533 * the previous collection is under control) for in-place I/O, as 534 * illustrated below: 535 * ________________________________________________________________ 536 * | tail (partial) page | head (partial) page | 537 * | (of the current cl) | (of the previous collection) | 538 * | PCLUSTER_FOLLOWED or | | 539 * |_____PCLUSTER_HOOKED__|___________PCLUSTER_FOLLOWED____________| 540 * 541 * [ (*) the above page can be used as inplace I/O. ] 542 */ 543 Z_EROFS_PCLUSTER_FOLLOWED, 544 }; 545 546 struct z_erofs_decompress_frontend { 547 struct inode *const inode; 548 struct erofs_map_blocks map; 549 struct z_erofs_bvec_iter biter; 550 551 struct page *pagepool; 552 struct page *candidate_bvpage; 553 struct z_erofs_pcluster *pcl, *tailpcl; 554 z_erofs_next_pcluster_t owned_head; 555 enum z_erofs_pclustermode mode; 556 557 /* used for applying cache strategy on the fly */ 558 bool backmost; 559 erofs_off_t headoffset; 560 561 /* a pointer used to pick up inplace I/O pages */ 562 unsigned int icur; 563 }; 564 565 #define DECOMPRESS_FRONTEND_INIT(__i) { \ 566 .inode = __i, .owned_head = Z_EROFS_PCLUSTER_TAIL, \ 567 .mode = Z_EROFS_PCLUSTER_FOLLOWED, .backmost = true } 568 569 static bool z_erofs_should_alloc_cache(struct z_erofs_decompress_frontend *fe) 570 { 571 unsigned int cachestrategy = EROFS_I_SB(fe->inode)->opt.cache_strategy; 572 573 if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED) 574 return false; 575 576 if (fe->backmost) 577 return true; 578 579 if (cachestrategy >= EROFS_ZIP_CACHE_READAROUND && 580 fe->map.m_la < fe->headoffset) 581 return true; 582 583 return false; 584 } 585 586 static void z_erofs_bind_cache(struct z_erofs_decompress_frontend *fe) 587 { 588 struct address_space *mc = MNGD_MAPPING(EROFS_I_SB(fe->inode)); 589 struct z_erofs_pcluster *pcl = fe->pcl; 590 bool shouldalloc = z_erofs_should_alloc_cache(fe); 591 bool standalone = true; 592 /* 593 * optimistic allocation without direct reclaim since inplace I/O 594 * can be used if low memory otherwise. 595 */ 596 gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) | 597 __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; 598 unsigned int i; 599 600 if (fe->mode < Z_EROFS_PCLUSTER_FOLLOWED) 601 return; 602 603 for (i = 0; i < pcl->pclusterpages; ++i) { 604 struct page *page; 605 void *t; /* mark pages just found for debugging */ 606 struct page *newpage = NULL; 607 608 /* the compressed page was loaded before */ 609 if (READ_ONCE(pcl->compressed_bvecs[i].page)) 610 continue; 611 612 page = find_get_page(mc, pcl->obj.index + i); 613 614 if (page) { 615 t = (void *)((unsigned long)page | 1); 616 } else { 617 /* I/O is needed, no possible to decompress directly */ 618 standalone = false; 619 if (!shouldalloc) 620 continue; 621 622 /* 623 * try to use cached I/O if page allocation 624 * succeeds or fallback to in-place I/O instead 625 * to avoid any direct reclaim. 626 */ 627 newpage = erofs_allocpage(&fe->pagepool, gfp); 628 if (!newpage) 629 continue; 630 set_page_private(newpage, Z_EROFS_PREALLOCATED_PAGE); 631 t = (void *)((unsigned long)newpage | 1); 632 } 633 634 if (!cmpxchg_relaxed(&pcl->compressed_bvecs[i].page, NULL, t)) 635 continue; 636 637 if (page) 638 put_page(page); 639 else if (newpage) 640 erofs_pagepool_add(&fe->pagepool, newpage); 641 } 642 643 /* 644 * don't do inplace I/O if all compressed pages are available in 645 * managed cache since it can be moved to the bypass queue instead. 646 */ 647 if (standalone) 648 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; 649 } 650 651 /* called by erofs_shrinker to get rid of all compressed_pages */ 652 int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi, 653 struct erofs_workgroup *grp) 654 { 655 struct z_erofs_pcluster *const pcl = 656 container_of(grp, struct z_erofs_pcluster, obj); 657 int i; 658 659 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); 660 /* 661 * refcount of workgroup is now freezed as 1, 662 * therefore no need to worry about available decompression users. 663 */ 664 for (i = 0; i < pcl->pclusterpages; ++i) { 665 struct page *page = pcl->compressed_bvecs[i].page; 666 667 if (!page) 668 continue; 669 670 /* block other users from reclaiming or migrating the page */ 671 if (!trylock_page(page)) 672 return -EBUSY; 673 674 if (!erofs_page_is_managed(sbi, page)) 675 continue; 676 677 /* barrier is implied in the following 'unlock_page' */ 678 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); 679 detach_page_private(page); 680 unlock_page(page); 681 } 682 return 0; 683 } 684 685 int erofs_try_to_free_cached_page(struct page *page) 686 { 687 struct z_erofs_pcluster *const pcl = (void *)page_private(page); 688 int ret, i; 689 690 if (!erofs_workgroup_try_to_freeze(&pcl->obj, 1)) 691 return 0; 692 693 ret = 0; 694 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); 695 for (i = 0; i < pcl->pclusterpages; ++i) { 696 if (pcl->compressed_bvecs[i].page == page) { 697 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); 698 ret = 1; 699 break; 700 } 701 } 702 erofs_workgroup_unfreeze(&pcl->obj, 1); 703 if (ret) 704 detach_page_private(page); 705 return ret; 706 } 707 708 static bool z_erofs_try_inplace_io(struct z_erofs_decompress_frontend *fe, 709 struct z_erofs_bvec *bvec) 710 { 711 struct z_erofs_pcluster *const pcl = fe->pcl; 712 713 while (fe->icur > 0) { 714 if (!cmpxchg(&pcl->compressed_bvecs[--fe->icur].page, 715 NULL, bvec->page)) { 716 pcl->compressed_bvecs[fe->icur] = *bvec; 717 return true; 718 } 719 } 720 return false; 721 } 722 723 /* callers must be with pcluster lock held */ 724 static int z_erofs_attach_page(struct z_erofs_decompress_frontend *fe, 725 struct z_erofs_bvec *bvec, bool exclusive) 726 { 727 int ret; 728 729 if (exclusive) { 730 /* give priority for inplaceio to use file pages first */ 731 if (z_erofs_try_inplace_io(fe, bvec)) 732 return 0; 733 /* otherwise, check if it can be used as a bvpage */ 734 if (fe->mode >= Z_EROFS_PCLUSTER_FOLLOWED && 735 !fe->candidate_bvpage) 736 fe->candidate_bvpage = bvec->page; 737 } 738 ret = z_erofs_bvec_enqueue(&fe->biter, bvec, &fe->candidate_bvpage, 739 &fe->pagepool); 740 fe->pcl->vcnt += (ret >= 0); 741 return ret; 742 } 743 744 static void z_erofs_try_to_claim_pcluster(struct z_erofs_decompress_frontend *f) 745 { 746 struct z_erofs_pcluster *pcl = f->pcl; 747 z_erofs_next_pcluster_t *owned_head = &f->owned_head; 748 749 /* type 1, nil pcluster (this pcluster doesn't belong to any chain.) */ 750 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL, 751 *owned_head) == Z_EROFS_PCLUSTER_NIL) { 752 *owned_head = &pcl->next; 753 /* so we can attach this pcluster to our submission chain. */ 754 f->mode = Z_EROFS_PCLUSTER_FOLLOWED; 755 return; 756 } 757 758 /* 759 * type 2, link to the end of an existing open chain, be careful 760 * that its submission is controlled by the original attached chain. 761 */ 762 if (*owned_head != &pcl->next && pcl != f->tailpcl && 763 cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, 764 *owned_head) == Z_EROFS_PCLUSTER_TAIL) { 765 *owned_head = Z_EROFS_PCLUSTER_TAIL; 766 f->mode = Z_EROFS_PCLUSTER_HOOKED; 767 f->tailpcl = NULL; 768 return; 769 } 770 /* type 3, it belongs to a chain, but it isn't the end of the chain */ 771 f->mode = Z_EROFS_PCLUSTER_INFLIGHT; 772 } 773 774 static int z_erofs_register_pcluster(struct z_erofs_decompress_frontend *fe) 775 { 776 struct erofs_map_blocks *map = &fe->map; 777 bool ztailpacking = map->m_flags & EROFS_MAP_META; 778 struct z_erofs_pcluster *pcl; 779 struct erofs_workgroup *grp; 780 int err; 781 782 if (!(map->m_flags & EROFS_MAP_ENCODED) || 783 (!ztailpacking && !(map->m_pa >> PAGE_SHIFT))) { 784 DBG_BUGON(1); 785 return -EFSCORRUPTED; 786 } 787 788 /* no available pcluster, let's allocate one */ 789 pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 : 790 map->m_plen >> PAGE_SHIFT); 791 if (IS_ERR(pcl)) 792 return PTR_ERR(pcl); 793 794 atomic_set(&pcl->obj.refcount, 1); 795 pcl->algorithmformat = map->m_algorithmformat; 796 pcl->length = 0; 797 pcl->partial = true; 798 799 /* new pclusters should be claimed as type 1, primary and followed */ 800 pcl->next = fe->owned_head; 801 pcl->pageofs_out = map->m_la & ~PAGE_MASK; 802 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED; 803 804 /* 805 * lock all primary followed works before visible to others 806 * and mutex_trylock *never* fails for a new pcluster. 807 */ 808 mutex_init(&pcl->lock); 809 DBG_BUGON(!mutex_trylock(&pcl->lock)); 810 811 if (ztailpacking) { 812 pcl->obj.index = 0; /* which indicates ztailpacking */ 813 pcl->pageofs_in = erofs_blkoff(fe->inode->i_sb, map->m_pa); 814 pcl->tailpacking_size = map->m_plen; 815 } else { 816 pcl->obj.index = map->m_pa >> PAGE_SHIFT; 817 818 grp = erofs_insert_workgroup(fe->inode->i_sb, &pcl->obj); 819 if (IS_ERR(grp)) { 820 err = PTR_ERR(grp); 821 goto err_out; 822 } 823 824 if (grp != &pcl->obj) { 825 fe->pcl = container_of(grp, 826 struct z_erofs_pcluster, obj); 827 err = -EEXIST; 828 goto err_out; 829 } 830 } 831 /* used to check tail merging loop due to corrupted images */ 832 if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL) 833 fe->tailpcl = pcl; 834 fe->owned_head = &pcl->next; 835 fe->pcl = pcl; 836 return 0; 837 838 err_out: 839 mutex_unlock(&pcl->lock); 840 z_erofs_free_pcluster(pcl); 841 return err; 842 } 843 844 static int z_erofs_collector_begin(struct z_erofs_decompress_frontend *fe) 845 { 846 struct erofs_map_blocks *map = &fe->map; 847 struct erofs_workgroup *grp = NULL; 848 int ret; 849 850 DBG_BUGON(fe->pcl); 851 852 /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */ 853 DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_NIL); 854 DBG_BUGON(fe->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 855 856 if (!(map->m_flags & EROFS_MAP_META)) { 857 grp = erofs_find_workgroup(fe->inode->i_sb, 858 map->m_pa >> PAGE_SHIFT); 859 } else if ((map->m_pa & ~PAGE_MASK) + map->m_plen > PAGE_SIZE) { 860 DBG_BUGON(1); 861 return -EFSCORRUPTED; 862 } 863 864 if (grp) { 865 fe->pcl = container_of(grp, struct z_erofs_pcluster, obj); 866 ret = -EEXIST; 867 } else { 868 ret = z_erofs_register_pcluster(fe); 869 } 870 871 if (ret == -EEXIST) { 872 mutex_lock(&fe->pcl->lock); 873 /* used to check tail merging loop due to corrupted images */ 874 if (fe->owned_head == Z_EROFS_PCLUSTER_TAIL) 875 fe->tailpcl = fe->pcl; 876 877 z_erofs_try_to_claim_pcluster(fe); 878 } else if (ret) { 879 return ret; 880 } 881 z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset, 882 Z_EROFS_INLINE_BVECS, fe->pcl->vcnt); 883 /* since file-backed online pages are traversed in reverse order */ 884 fe->icur = z_erofs_pclusterpages(fe->pcl); 885 return 0; 886 } 887 888 /* 889 * keep in mind that no referenced pclusters will be freed 890 * only after a RCU grace period. 891 */ 892 static void z_erofs_rcu_callback(struct rcu_head *head) 893 { 894 z_erofs_free_pcluster(container_of(head, 895 struct z_erofs_pcluster, rcu)); 896 } 897 898 void erofs_workgroup_free_rcu(struct erofs_workgroup *grp) 899 { 900 struct z_erofs_pcluster *const pcl = 901 container_of(grp, struct z_erofs_pcluster, obj); 902 903 call_rcu(&pcl->rcu, z_erofs_rcu_callback); 904 } 905 906 static bool z_erofs_collector_end(struct z_erofs_decompress_frontend *fe) 907 { 908 struct z_erofs_pcluster *pcl = fe->pcl; 909 910 if (!pcl) 911 return false; 912 913 z_erofs_bvec_iter_end(&fe->biter); 914 mutex_unlock(&pcl->lock); 915 916 if (fe->candidate_bvpage) 917 fe->candidate_bvpage = NULL; 918 919 /* 920 * if all pending pages are added, don't hold its reference 921 * any longer if the pcluster isn't hosted by ourselves. 922 */ 923 if (fe->mode < Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE) 924 erofs_workgroup_put(&pcl->obj); 925 926 fe->pcl = NULL; 927 return true; 928 } 929 930 static int z_erofs_read_fragment(struct inode *inode, erofs_off_t pos, 931 struct page *page, unsigned int pageofs, 932 unsigned int len) 933 { 934 struct super_block *sb = inode->i_sb; 935 struct inode *packed_inode = EROFS_I_SB(inode)->packed_inode; 936 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 937 u8 *src, *dst; 938 unsigned int i, cnt; 939 940 if (!packed_inode) 941 return -EFSCORRUPTED; 942 943 buf.inode = packed_inode; 944 pos += EROFS_I(inode)->z_fragmentoff; 945 for (i = 0; i < len; i += cnt) { 946 cnt = min_t(unsigned int, len - i, 947 sb->s_blocksize - erofs_blkoff(sb, pos)); 948 src = erofs_bread(&buf, erofs_blknr(sb, pos), EROFS_KMAP); 949 if (IS_ERR(src)) { 950 erofs_put_metabuf(&buf); 951 return PTR_ERR(src); 952 } 953 954 dst = kmap_local_page(page); 955 memcpy(dst + pageofs + i, src + erofs_blkoff(sb, pos), cnt); 956 kunmap_local(dst); 957 pos += cnt; 958 } 959 erofs_put_metabuf(&buf); 960 return 0; 961 } 962 963 static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe, 964 struct page *page) 965 { 966 struct inode *const inode = fe->inode; 967 struct erofs_map_blocks *const map = &fe->map; 968 const loff_t offset = page_offset(page); 969 bool tight = true, exclusive; 970 unsigned int cur, end, spiltted; 971 int err = 0; 972 973 /* register locked file pages as online pages in pack */ 974 z_erofs_onlinepage_init(page); 975 976 spiltted = 0; 977 end = PAGE_SIZE; 978 repeat: 979 cur = end - 1; 980 981 if (offset + cur < map->m_la || 982 offset + cur >= map->m_la + map->m_llen) { 983 if (z_erofs_collector_end(fe)) 984 fe->backmost = false; 985 map->m_la = offset + cur; 986 map->m_llen = 0; 987 err = z_erofs_map_blocks_iter(inode, map, 0); 988 if (err) 989 goto out; 990 } else { 991 if (fe->pcl) 992 goto hitted; 993 /* didn't get a valid pcluster previously (very rare) */ 994 } 995 996 if (!(map->m_flags & EROFS_MAP_MAPPED) || 997 map->m_flags & EROFS_MAP_FRAGMENT) 998 goto hitted; 999 1000 err = z_erofs_collector_begin(fe); 1001 if (err) 1002 goto out; 1003 1004 if (z_erofs_is_inline_pcluster(fe->pcl)) { 1005 void *mp; 1006 1007 mp = erofs_read_metabuf(&fe->map.buf, inode->i_sb, 1008 erofs_blknr(inode->i_sb, map->m_pa), 1009 EROFS_NO_KMAP); 1010 if (IS_ERR(mp)) { 1011 err = PTR_ERR(mp); 1012 erofs_err(inode->i_sb, 1013 "failed to get inline page, err %d", err); 1014 goto out; 1015 } 1016 get_page(fe->map.buf.page); 1017 WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, 1018 fe->map.buf.page); 1019 fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; 1020 } else { 1021 /* bind cache first when cached decompression is preferred */ 1022 z_erofs_bind_cache(fe); 1023 } 1024 hitted: 1025 /* 1026 * Ensure the current partial page belongs to this submit chain rather 1027 * than other concurrent submit chains or the noio(bypass) chain since 1028 * those chains are handled asynchronously thus the page cannot be used 1029 * for inplace I/O or bvpage (should be processed in a strict order.) 1030 */ 1031 tight &= (fe->mode >= Z_EROFS_PCLUSTER_HOOKED && 1032 fe->mode != Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE); 1033 1034 cur = end - min_t(unsigned int, offset + end - map->m_la, end); 1035 if (!(map->m_flags & EROFS_MAP_MAPPED)) { 1036 zero_user_segment(page, cur, end); 1037 goto next_part; 1038 } 1039 if (map->m_flags & EROFS_MAP_FRAGMENT) { 1040 unsigned int pageofs, skip, len; 1041 1042 if (offset > map->m_la) { 1043 pageofs = 0; 1044 skip = offset - map->m_la; 1045 } else { 1046 pageofs = map->m_la & ~PAGE_MASK; 1047 skip = 0; 1048 } 1049 len = min_t(unsigned int, map->m_llen - skip, end - cur); 1050 err = z_erofs_read_fragment(inode, skip, page, pageofs, len); 1051 if (err) 1052 goto out; 1053 ++spiltted; 1054 tight = false; 1055 goto next_part; 1056 } 1057 1058 exclusive = (!cur && (!spiltted || tight)); 1059 if (cur) 1060 tight &= (fe->mode >= Z_EROFS_PCLUSTER_FOLLOWED); 1061 1062 err = z_erofs_attach_page(fe, &((struct z_erofs_bvec) { 1063 .page = page, 1064 .offset = offset - map->m_la, 1065 .end = end, 1066 }), exclusive); 1067 if (err) 1068 goto out; 1069 1070 z_erofs_onlinepage_split(page); 1071 /* bump up the number of spiltted parts of a page */ 1072 ++spiltted; 1073 if (fe->pcl->pageofs_out != (map->m_la & ~PAGE_MASK)) 1074 fe->pcl->multibases = true; 1075 if (fe->pcl->length < offset + end - map->m_la) { 1076 fe->pcl->length = offset + end - map->m_la; 1077 fe->pcl->pageofs_out = map->m_la & ~PAGE_MASK; 1078 } 1079 if ((map->m_flags & EROFS_MAP_FULL_MAPPED) && 1080 !(map->m_flags & EROFS_MAP_PARTIAL_REF) && 1081 fe->pcl->length == map->m_llen) 1082 fe->pcl->partial = false; 1083 next_part: 1084 /* shorten the remaining extent to update progress */ 1085 map->m_llen = offset + cur - map->m_la; 1086 map->m_flags &= ~EROFS_MAP_FULL_MAPPED; 1087 1088 end = cur; 1089 if (end > 0) 1090 goto repeat; 1091 1092 out: 1093 if (err) 1094 z_erofs_page_mark_eio(page); 1095 z_erofs_onlinepage_endio(page); 1096 return err; 1097 } 1098 1099 static bool z_erofs_is_sync_decompress(struct erofs_sb_info *sbi, 1100 unsigned int readahead_pages) 1101 { 1102 /* auto: enable for read_folio, disable for readahead */ 1103 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) && 1104 !readahead_pages) 1105 return true; 1106 1107 if ((sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_FORCE_ON) && 1108 (readahead_pages <= sbi->opt.max_sync_decompress_pages)) 1109 return true; 1110 1111 return false; 1112 } 1113 1114 static bool z_erofs_page_is_invalidated(struct page *page) 1115 { 1116 return !page->mapping && !z_erofs_is_shortlived_page(page); 1117 } 1118 1119 struct z_erofs_decompress_backend { 1120 struct page *onstack_pages[Z_EROFS_ONSTACK_PAGES]; 1121 struct super_block *sb; 1122 struct z_erofs_pcluster *pcl; 1123 1124 /* pages with the longest decompressed length for deduplication */ 1125 struct page **decompressed_pages; 1126 /* pages to keep the compressed data */ 1127 struct page **compressed_pages; 1128 1129 struct list_head decompressed_secondary_bvecs; 1130 struct page **pagepool; 1131 unsigned int onstack_used, nr_pages; 1132 }; 1133 1134 struct z_erofs_bvec_item { 1135 struct z_erofs_bvec bvec; 1136 struct list_head list; 1137 }; 1138 1139 static void z_erofs_do_decompressed_bvec(struct z_erofs_decompress_backend *be, 1140 struct z_erofs_bvec *bvec) 1141 { 1142 struct z_erofs_bvec_item *item; 1143 1144 if (!((bvec->offset + be->pcl->pageofs_out) & ~PAGE_MASK)) { 1145 unsigned int pgnr; 1146 1147 pgnr = (bvec->offset + be->pcl->pageofs_out) >> PAGE_SHIFT; 1148 DBG_BUGON(pgnr >= be->nr_pages); 1149 if (!be->decompressed_pages[pgnr]) { 1150 be->decompressed_pages[pgnr] = bvec->page; 1151 return; 1152 } 1153 } 1154 1155 /* (cold path) one pcluster is requested multiple times */ 1156 item = kmalloc(sizeof(*item), GFP_KERNEL | __GFP_NOFAIL); 1157 item->bvec = *bvec; 1158 list_add(&item->list, &be->decompressed_secondary_bvecs); 1159 } 1160 1161 static void z_erofs_fill_other_copies(struct z_erofs_decompress_backend *be, 1162 int err) 1163 { 1164 unsigned int off0 = be->pcl->pageofs_out; 1165 struct list_head *p, *n; 1166 1167 list_for_each_safe(p, n, &be->decompressed_secondary_bvecs) { 1168 struct z_erofs_bvec_item *bvi; 1169 unsigned int end, cur; 1170 void *dst, *src; 1171 1172 bvi = container_of(p, struct z_erofs_bvec_item, list); 1173 cur = bvi->bvec.offset < 0 ? -bvi->bvec.offset : 0; 1174 end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset, 1175 bvi->bvec.end); 1176 dst = kmap_local_page(bvi->bvec.page); 1177 while (cur < end) { 1178 unsigned int pgnr, scur, len; 1179 1180 pgnr = (bvi->bvec.offset + cur + off0) >> PAGE_SHIFT; 1181 DBG_BUGON(pgnr >= be->nr_pages); 1182 1183 scur = bvi->bvec.offset + cur - 1184 ((pgnr << PAGE_SHIFT) - off0); 1185 len = min_t(unsigned int, end - cur, PAGE_SIZE - scur); 1186 if (!be->decompressed_pages[pgnr]) { 1187 err = -EFSCORRUPTED; 1188 cur += len; 1189 continue; 1190 } 1191 src = kmap_local_page(be->decompressed_pages[pgnr]); 1192 memcpy(dst + cur, src + scur, len); 1193 kunmap_local(src); 1194 cur += len; 1195 } 1196 kunmap_local(dst); 1197 if (err) 1198 z_erofs_page_mark_eio(bvi->bvec.page); 1199 z_erofs_onlinepage_endio(bvi->bvec.page); 1200 list_del(p); 1201 kfree(bvi); 1202 } 1203 } 1204 1205 static void z_erofs_parse_out_bvecs(struct z_erofs_decompress_backend *be) 1206 { 1207 struct z_erofs_pcluster *pcl = be->pcl; 1208 struct z_erofs_bvec_iter biter; 1209 struct page *old_bvpage; 1210 int i; 1211 1212 z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0); 1213 for (i = 0; i < pcl->vcnt; ++i) { 1214 struct z_erofs_bvec bvec; 1215 1216 z_erofs_bvec_dequeue(&biter, &bvec, &old_bvpage); 1217 1218 if (old_bvpage) 1219 z_erofs_put_shortlivedpage(be->pagepool, old_bvpage); 1220 1221 DBG_BUGON(z_erofs_page_is_invalidated(bvec.page)); 1222 z_erofs_do_decompressed_bvec(be, &bvec); 1223 } 1224 1225 old_bvpage = z_erofs_bvec_iter_end(&biter); 1226 if (old_bvpage) 1227 z_erofs_put_shortlivedpage(be->pagepool, old_bvpage); 1228 } 1229 1230 static int z_erofs_parse_in_bvecs(struct z_erofs_decompress_backend *be, 1231 bool *overlapped) 1232 { 1233 struct z_erofs_pcluster *pcl = be->pcl; 1234 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 1235 int i, err = 0; 1236 1237 *overlapped = false; 1238 for (i = 0; i < pclusterpages; ++i) { 1239 struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i]; 1240 struct page *page = bvec->page; 1241 1242 /* compressed pages ought to be present before decompressing */ 1243 if (!page) { 1244 DBG_BUGON(1); 1245 continue; 1246 } 1247 be->compressed_pages[i] = page; 1248 1249 if (z_erofs_is_inline_pcluster(pcl)) { 1250 if (!PageUptodate(page)) 1251 err = -EIO; 1252 continue; 1253 } 1254 1255 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1256 if (!z_erofs_is_shortlived_page(page)) { 1257 if (erofs_page_is_managed(EROFS_SB(be->sb), page)) { 1258 if (!PageUptodate(page)) 1259 err = -EIO; 1260 continue; 1261 } 1262 z_erofs_do_decompressed_bvec(be, bvec); 1263 *overlapped = true; 1264 } 1265 } 1266 1267 if (err) 1268 return err; 1269 return 0; 1270 } 1271 1272 static int z_erofs_decompress_pcluster(struct z_erofs_decompress_backend *be, 1273 int err) 1274 { 1275 struct erofs_sb_info *const sbi = EROFS_SB(be->sb); 1276 struct z_erofs_pcluster *pcl = be->pcl; 1277 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); 1278 const struct z_erofs_decompressor *decompressor = 1279 &erofs_decompressors[pcl->algorithmformat]; 1280 unsigned int i, inputsize; 1281 int err2; 1282 struct page *page; 1283 bool overlapped; 1284 1285 mutex_lock(&pcl->lock); 1286 be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT; 1287 1288 /* allocate (de)compressed page arrays if cannot be kept on stack */ 1289 be->decompressed_pages = NULL; 1290 be->compressed_pages = NULL; 1291 be->onstack_used = 0; 1292 if (be->nr_pages <= Z_EROFS_ONSTACK_PAGES) { 1293 be->decompressed_pages = be->onstack_pages; 1294 be->onstack_used = be->nr_pages; 1295 memset(be->decompressed_pages, 0, 1296 sizeof(struct page *) * be->nr_pages); 1297 } 1298 1299 if (pclusterpages + be->onstack_used <= Z_EROFS_ONSTACK_PAGES) 1300 be->compressed_pages = be->onstack_pages + be->onstack_used; 1301 1302 if (!be->decompressed_pages) 1303 be->decompressed_pages = 1304 kvcalloc(be->nr_pages, sizeof(struct page *), 1305 GFP_KERNEL | __GFP_NOFAIL); 1306 if (!be->compressed_pages) 1307 be->compressed_pages = 1308 kvcalloc(pclusterpages, sizeof(struct page *), 1309 GFP_KERNEL | __GFP_NOFAIL); 1310 1311 z_erofs_parse_out_bvecs(be); 1312 err2 = z_erofs_parse_in_bvecs(be, &overlapped); 1313 if (err2) 1314 err = err2; 1315 if (err) 1316 goto out; 1317 1318 if (z_erofs_is_inline_pcluster(pcl)) 1319 inputsize = pcl->tailpacking_size; 1320 else 1321 inputsize = pclusterpages * PAGE_SIZE; 1322 1323 err = decompressor->decompress(&(struct z_erofs_decompress_req) { 1324 .sb = be->sb, 1325 .in = be->compressed_pages, 1326 .out = be->decompressed_pages, 1327 .pageofs_in = pcl->pageofs_in, 1328 .pageofs_out = pcl->pageofs_out, 1329 .inputsize = inputsize, 1330 .outputsize = pcl->length, 1331 .alg = pcl->algorithmformat, 1332 .inplace_io = overlapped, 1333 .partial_decoding = pcl->partial, 1334 .fillgaps = pcl->multibases, 1335 }, be->pagepool); 1336 1337 out: 1338 /* must handle all compressed pages before actual file pages */ 1339 if (z_erofs_is_inline_pcluster(pcl)) { 1340 page = pcl->compressed_bvecs[0].page; 1341 WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL); 1342 put_page(page); 1343 } else { 1344 for (i = 0; i < pclusterpages; ++i) { 1345 page = pcl->compressed_bvecs[i].page; 1346 1347 if (erofs_page_is_managed(sbi, page)) 1348 continue; 1349 1350 /* recycle all individual short-lived pages */ 1351 (void)z_erofs_put_shortlivedpage(be->pagepool, page); 1352 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); 1353 } 1354 } 1355 if (be->compressed_pages < be->onstack_pages || 1356 be->compressed_pages >= be->onstack_pages + Z_EROFS_ONSTACK_PAGES) 1357 kvfree(be->compressed_pages); 1358 z_erofs_fill_other_copies(be, err); 1359 1360 for (i = 0; i < be->nr_pages; ++i) { 1361 page = be->decompressed_pages[i]; 1362 if (!page) 1363 continue; 1364 1365 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1366 1367 /* recycle all individual short-lived pages */ 1368 if (z_erofs_put_shortlivedpage(be->pagepool, page)) 1369 continue; 1370 if (err) 1371 z_erofs_page_mark_eio(page); 1372 z_erofs_onlinepage_endio(page); 1373 } 1374 1375 if (be->decompressed_pages != be->onstack_pages) 1376 kvfree(be->decompressed_pages); 1377 1378 pcl->length = 0; 1379 pcl->partial = true; 1380 pcl->multibases = false; 1381 pcl->bvset.nextpage = NULL; 1382 pcl->vcnt = 0; 1383 1384 /* pcluster lock MUST be taken before the following line */ 1385 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL); 1386 mutex_unlock(&pcl->lock); 1387 return err; 1388 } 1389 1390 static void z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, 1391 struct page **pagepool) 1392 { 1393 struct z_erofs_decompress_backend be = { 1394 .sb = io->sb, 1395 .pagepool = pagepool, 1396 .decompressed_secondary_bvecs = 1397 LIST_HEAD_INIT(be.decompressed_secondary_bvecs), 1398 }; 1399 z_erofs_next_pcluster_t owned = io->head; 1400 1401 while (owned != Z_EROFS_PCLUSTER_TAIL_CLOSED) { 1402 /* impossible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */ 1403 DBG_BUGON(owned == Z_EROFS_PCLUSTER_TAIL); 1404 /* impossible that 'owned' equals Z_EROFS_PCLUSTER_NIL */ 1405 DBG_BUGON(owned == Z_EROFS_PCLUSTER_NIL); 1406 1407 be.pcl = container_of(owned, struct z_erofs_pcluster, next); 1408 owned = READ_ONCE(be.pcl->next); 1409 1410 z_erofs_decompress_pcluster(&be, io->eio ? -EIO : 0); 1411 erofs_workgroup_put(&be.pcl->obj); 1412 } 1413 } 1414 1415 static void z_erofs_decompressqueue_work(struct work_struct *work) 1416 { 1417 struct z_erofs_decompressqueue *bgq = 1418 container_of(work, struct z_erofs_decompressqueue, u.work); 1419 struct page *pagepool = NULL; 1420 1421 DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1422 z_erofs_decompress_queue(bgq, &pagepool); 1423 erofs_release_pages(&pagepool); 1424 kvfree(bgq); 1425 } 1426 1427 #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD 1428 static void z_erofs_decompressqueue_kthread_work(struct kthread_work *work) 1429 { 1430 z_erofs_decompressqueue_work((struct work_struct *)work); 1431 } 1432 #endif 1433 1434 static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, 1435 int bios) 1436 { 1437 struct erofs_sb_info *const sbi = EROFS_SB(io->sb); 1438 1439 /* wake up the caller thread for sync decompression */ 1440 if (io->sync) { 1441 if (!atomic_add_return(bios, &io->pending_bios)) 1442 complete(&io->u.done); 1443 return; 1444 } 1445 1446 if (atomic_add_return(bios, &io->pending_bios)) 1447 return; 1448 /* Use (kthread_)work and sync decompression for atomic contexts only */ 1449 if (in_atomic() || irqs_disabled()) { 1450 #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD 1451 struct kthread_worker *worker; 1452 1453 rcu_read_lock(); 1454 worker = rcu_dereference( 1455 z_erofs_pcpu_workers[raw_smp_processor_id()]); 1456 if (!worker) { 1457 INIT_WORK(&io->u.work, z_erofs_decompressqueue_work); 1458 queue_work(z_erofs_workqueue, &io->u.work); 1459 } else { 1460 kthread_queue_work(worker, &io->u.kthread_work); 1461 } 1462 rcu_read_unlock(); 1463 #else 1464 queue_work(z_erofs_workqueue, &io->u.work); 1465 #endif 1466 /* enable sync decompression for readahead */ 1467 if (sbi->opt.sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) 1468 sbi->opt.sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; 1469 return; 1470 } 1471 z_erofs_decompressqueue_work(&io->u.work); 1472 } 1473 1474 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, 1475 unsigned int nr, 1476 struct page **pagepool, 1477 struct address_space *mc) 1478 { 1479 const pgoff_t index = pcl->obj.index; 1480 gfp_t gfp = mapping_gfp_mask(mc); 1481 bool tocache = false; 1482 1483 struct address_space *mapping; 1484 struct page *oldpage, *page; 1485 int justfound; 1486 1487 repeat: 1488 page = READ_ONCE(pcl->compressed_bvecs[nr].page); 1489 oldpage = page; 1490 1491 if (!page) 1492 goto out_allocpage; 1493 1494 justfound = (unsigned long)page & 1UL; 1495 page = (struct page *)((unsigned long)page & ~1UL); 1496 1497 /* 1498 * preallocated cached pages, which is used to avoid direct reclaim 1499 * otherwise, it will go inplace I/O path instead. 1500 */ 1501 if (page->private == Z_EROFS_PREALLOCATED_PAGE) { 1502 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); 1503 set_page_private(page, 0); 1504 tocache = true; 1505 goto out_tocache; 1506 } 1507 mapping = READ_ONCE(page->mapping); 1508 1509 /* 1510 * file-backed online pages in plcuster are all locked steady, 1511 * therefore it is impossible for `mapping' to be NULL. 1512 */ 1513 if (mapping && mapping != mc) 1514 /* ought to be unmanaged pages */ 1515 goto out; 1516 1517 /* directly return for shortlived page as well */ 1518 if (z_erofs_is_shortlived_page(page)) 1519 goto out; 1520 1521 lock_page(page); 1522 1523 /* only true if page reclaim goes wrong, should never happen */ 1524 DBG_BUGON(justfound && PagePrivate(page)); 1525 1526 /* the page is still in manage cache */ 1527 if (page->mapping == mc) { 1528 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); 1529 1530 if (!PagePrivate(page)) { 1531 /* 1532 * impossible to be !PagePrivate(page) for 1533 * the current restriction as well if 1534 * the page is already in compressed_bvecs[]. 1535 */ 1536 DBG_BUGON(!justfound); 1537 1538 justfound = 0; 1539 set_page_private(page, (unsigned long)pcl); 1540 SetPagePrivate(page); 1541 } 1542 1543 /* no need to submit io if it is already up-to-date */ 1544 if (PageUptodate(page)) { 1545 unlock_page(page); 1546 page = NULL; 1547 } 1548 goto out; 1549 } 1550 1551 /* 1552 * the managed page has been truncated, it's unsafe to 1553 * reuse this one, let's allocate a new cache-managed page. 1554 */ 1555 DBG_BUGON(page->mapping); 1556 DBG_BUGON(!justfound); 1557 1558 tocache = true; 1559 unlock_page(page); 1560 put_page(page); 1561 out_allocpage: 1562 page = erofs_allocpage(pagepool, gfp | __GFP_NOFAIL); 1563 if (oldpage != cmpxchg(&pcl->compressed_bvecs[nr].page, 1564 oldpage, page)) { 1565 erofs_pagepool_add(pagepool, page); 1566 cond_resched(); 1567 goto repeat; 1568 } 1569 out_tocache: 1570 if (!tocache || add_to_page_cache_lru(page, mc, index + nr, gfp)) { 1571 /* turn into temporary page if fails (1 ref) */ 1572 set_page_private(page, Z_EROFS_SHORTLIVED_PAGE); 1573 goto out; 1574 } 1575 attach_page_private(page, pcl); 1576 /* drop a refcount added by allocpage (then we have 2 refs here) */ 1577 put_page(page); 1578 1579 out: /* the only exit (for tracing and debugging) */ 1580 return page; 1581 } 1582 1583 static struct z_erofs_decompressqueue *jobqueue_init(struct super_block *sb, 1584 struct z_erofs_decompressqueue *fgq, bool *fg) 1585 { 1586 struct z_erofs_decompressqueue *q; 1587 1588 if (fg && !*fg) { 1589 q = kvzalloc(sizeof(*q), GFP_KERNEL | __GFP_NOWARN); 1590 if (!q) { 1591 *fg = true; 1592 goto fg_out; 1593 } 1594 #ifdef CONFIG_EROFS_FS_PCPU_KTHREAD 1595 kthread_init_work(&q->u.kthread_work, 1596 z_erofs_decompressqueue_kthread_work); 1597 #else 1598 INIT_WORK(&q->u.work, z_erofs_decompressqueue_work); 1599 #endif 1600 } else { 1601 fg_out: 1602 q = fgq; 1603 init_completion(&fgq->u.done); 1604 atomic_set(&fgq->pending_bios, 0); 1605 q->eio = false; 1606 q->sync = true; 1607 } 1608 q->sb = sb; 1609 q->head = Z_EROFS_PCLUSTER_TAIL_CLOSED; 1610 return q; 1611 } 1612 1613 /* define decompression jobqueue types */ 1614 enum { 1615 JQ_BYPASS, 1616 JQ_SUBMIT, 1617 NR_JOBQUEUES, 1618 }; 1619 1620 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, 1621 z_erofs_next_pcluster_t qtail[], 1622 z_erofs_next_pcluster_t owned_head) 1623 { 1624 z_erofs_next_pcluster_t *const submit_qtail = qtail[JQ_SUBMIT]; 1625 z_erofs_next_pcluster_t *const bypass_qtail = qtail[JQ_BYPASS]; 1626 1627 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1628 if (owned_head == Z_EROFS_PCLUSTER_TAIL) 1629 owned_head = Z_EROFS_PCLUSTER_TAIL_CLOSED; 1630 1631 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED); 1632 1633 WRITE_ONCE(*submit_qtail, owned_head); 1634 WRITE_ONCE(*bypass_qtail, &pcl->next); 1635 1636 qtail[JQ_BYPASS] = &pcl->next; 1637 } 1638 1639 static void z_erofs_decompressqueue_endio(struct bio *bio) 1640 { 1641 struct z_erofs_decompressqueue *q = bio->bi_private; 1642 blk_status_t err = bio->bi_status; 1643 struct bio_vec *bvec; 1644 struct bvec_iter_all iter_all; 1645 1646 bio_for_each_segment_all(bvec, bio, iter_all) { 1647 struct page *page = bvec->bv_page; 1648 1649 DBG_BUGON(PageUptodate(page)); 1650 DBG_BUGON(z_erofs_page_is_invalidated(page)); 1651 1652 if (erofs_page_is_managed(EROFS_SB(q->sb), page)) { 1653 if (!err) 1654 SetPageUptodate(page); 1655 unlock_page(page); 1656 } 1657 } 1658 if (err) 1659 q->eio = true; 1660 z_erofs_decompress_kickoff(q, -1); 1661 bio_put(bio); 1662 } 1663 1664 static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, 1665 struct z_erofs_decompressqueue *fgq, 1666 bool *force_fg, bool readahead) 1667 { 1668 struct super_block *sb = f->inode->i_sb; 1669 struct address_space *mc = MNGD_MAPPING(EROFS_SB(sb)); 1670 z_erofs_next_pcluster_t qtail[NR_JOBQUEUES]; 1671 struct z_erofs_decompressqueue *q[NR_JOBQUEUES]; 1672 z_erofs_next_pcluster_t owned_head = f->owned_head; 1673 /* bio is NULL initially, so no need to initialize last_{index,bdev} */ 1674 pgoff_t last_index; 1675 struct block_device *last_bdev; 1676 unsigned int nr_bios = 0; 1677 struct bio *bio = NULL; 1678 unsigned long pflags; 1679 int memstall = 0; 1680 1681 /* 1682 * if managed cache is enabled, bypass jobqueue is needed, 1683 * no need to read from device for all pclusters in this queue. 1684 */ 1685 q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL); 1686 q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, force_fg); 1687 1688 qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; 1689 qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head; 1690 1691 /* by default, all need io submission */ 1692 q[JQ_SUBMIT]->head = owned_head; 1693 1694 do { 1695 struct erofs_map_dev mdev; 1696 struct z_erofs_pcluster *pcl; 1697 pgoff_t cur, end; 1698 unsigned int i = 0; 1699 bool bypass = true; 1700 1701 /* no possible 'owned_head' equals the following */ 1702 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED); 1703 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_NIL); 1704 1705 pcl = container_of(owned_head, struct z_erofs_pcluster, next); 1706 1707 /* close the main owned chain at first */ 1708 owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL, 1709 Z_EROFS_PCLUSTER_TAIL_CLOSED); 1710 if (z_erofs_is_inline_pcluster(pcl)) { 1711 move_to_bypass_jobqueue(pcl, qtail, owned_head); 1712 continue; 1713 } 1714 1715 /* no device id here, thus it will always succeed */ 1716 mdev = (struct erofs_map_dev) { 1717 .m_pa = erofs_pos(sb, pcl->obj.index), 1718 }; 1719 (void)erofs_map_dev(sb, &mdev); 1720 1721 cur = erofs_blknr(sb, mdev.m_pa); 1722 end = cur + pcl->pclusterpages; 1723 1724 do { 1725 struct page *page; 1726 1727 page = pickup_page_for_submission(pcl, i++, 1728 &f->pagepool, mc); 1729 if (!page) 1730 continue; 1731 1732 if (bio && (cur != last_index + 1 || 1733 last_bdev != mdev.m_bdev)) { 1734 submit_bio_retry: 1735 submit_bio(bio); 1736 if (memstall) { 1737 psi_memstall_leave(&pflags); 1738 memstall = 0; 1739 } 1740 bio = NULL; 1741 } 1742 1743 if (unlikely(PageWorkingset(page)) && !memstall) { 1744 psi_memstall_enter(&pflags); 1745 memstall = 1; 1746 } 1747 1748 if (!bio) { 1749 bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS, 1750 REQ_OP_READ, GFP_NOIO); 1751 bio->bi_end_io = z_erofs_decompressqueue_endio; 1752 1753 last_bdev = mdev.m_bdev; 1754 bio->bi_iter.bi_sector = (sector_t)cur << 1755 (sb->s_blocksize_bits - 9); 1756 bio->bi_private = q[JQ_SUBMIT]; 1757 if (readahead) 1758 bio->bi_opf |= REQ_RAHEAD; 1759 ++nr_bios; 1760 } 1761 1762 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) 1763 goto submit_bio_retry; 1764 1765 last_index = cur; 1766 bypass = false; 1767 } while (++cur < end); 1768 1769 if (!bypass) 1770 qtail[JQ_SUBMIT] = &pcl->next; 1771 else 1772 move_to_bypass_jobqueue(pcl, qtail, owned_head); 1773 } while (owned_head != Z_EROFS_PCLUSTER_TAIL); 1774 1775 if (bio) { 1776 submit_bio(bio); 1777 if (memstall) 1778 psi_memstall_leave(&pflags); 1779 } 1780 1781 /* 1782 * although background is preferred, no one is pending for submission. 1783 * don't issue decompression but drop it directly instead. 1784 */ 1785 if (!*force_fg && !nr_bios) { 1786 kvfree(q[JQ_SUBMIT]); 1787 return; 1788 } 1789 z_erofs_decompress_kickoff(q[JQ_SUBMIT], nr_bios); 1790 } 1791 1792 static void z_erofs_runqueue(struct z_erofs_decompress_frontend *f, 1793 bool force_fg, bool ra) 1794 { 1795 struct z_erofs_decompressqueue io[NR_JOBQUEUES]; 1796 1797 if (f->owned_head == Z_EROFS_PCLUSTER_TAIL) 1798 return; 1799 z_erofs_submit_queue(f, io, &force_fg, ra); 1800 1801 /* handle bypass queue (no i/o pclusters) immediately */ 1802 z_erofs_decompress_queue(&io[JQ_BYPASS], &f->pagepool); 1803 1804 if (!force_fg) 1805 return; 1806 1807 /* wait until all bios are completed */ 1808 wait_for_completion_io(&io[JQ_SUBMIT].u.done); 1809 1810 /* handle synchronous decompress queue in the caller context */ 1811 z_erofs_decompress_queue(&io[JQ_SUBMIT], &f->pagepool); 1812 } 1813 1814 /* 1815 * Since partial uptodate is still unimplemented for now, we have to use 1816 * approximate readmore strategies as a start. 1817 */ 1818 static void z_erofs_pcluster_readmore(struct z_erofs_decompress_frontend *f, 1819 struct readahead_control *rac, bool backmost) 1820 { 1821 struct inode *inode = f->inode; 1822 struct erofs_map_blocks *map = &f->map; 1823 erofs_off_t cur, end, headoffset = f->headoffset; 1824 int err; 1825 1826 if (backmost) { 1827 if (rac) 1828 end = headoffset + readahead_length(rac) - 1; 1829 else 1830 end = headoffset + PAGE_SIZE - 1; 1831 map->m_la = end; 1832 err = z_erofs_map_blocks_iter(inode, map, 1833 EROFS_GET_BLOCKS_READMORE); 1834 if (err) 1835 return; 1836 1837 /* expand ra for the trailing edge if readahead */ 1838 if (rac) { 1839 cur = round_up(map->m_la + map->m_llen, PAGE_SIZE); 1840 readahead_expand(rac, headoffset, cur - headoffset); 1841 return; 1842 } 1843 end = round_up(end, PAGE_SIZE); 1844 } else { 1845 end = round_up(map->m_la, PAGE_SIZE); 1846 1847 if (!map->m_llen) 1848 return; 1849 } 1850 1851 cur = map->m_la + map->m_llen - 1; 1852 while (cur >= end) { 1853 pgoff_t index = cur >> PAGE_SHIFT; 1854 struct page *page; 1855 1856 page = erofs_grab_cache_page_nowait(inode->i_mapping, index); 1857 if (page) { 1858 if (PageUptodate(page)) { 1859 unlock_page(page); 1860 } else { 1861 err = z_erofs_do_read_page(f, page); 1862 if (err) 1863 erofs_err(inode->i_sb, 1864 "readmore error at page %lu @ nid %llu", 1865 index, EROFS_I(inode)->nid); 1866 } 1867 put_page(page); 1868 } 1869 1870 if (cur < PAGE_SIZE) 1871 break; 1872 cur = (index << PAGE_SHIFT) - 1; 1873 } 1874 } 1875 1876 static int z_erofs_read_folio(struct file *file, struct folio *folio) 1877 { 1878 struct page *page = &folio->page; 1879 struct inode *const inode = page->mapping->host; 1880 struct erofs_sb_info *const sbi = EROFS_I_SB(inode); 1881 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); 1882 int err; 1883 1884 trace_erofs_readpage(page, false); 1885 f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT; 1886 1887 z_erofs_pcluster_readmore(&f, NULL, true); 1888 err = z_erofs_do_read_page(&f, page); 1889 z_erofs_pcluster_readmore(&f, NULL, false); 1890 (void)z_erofs_collector_end(&f); 1891 1892 /* if some compressed cluster ready, need submit them anyway */ 1893 z_erofs_runqueue(&f, z_erofs_is_sync_decompress(sbi, 0), false); 1894 1895 if (err) 1896 erofs_err(inode->i_sb, "failed to read, err [%d]", err); 1897 1898 erofs_put_metabuf(&f.map.buf); 1899 erofs_release_pages(&f.pagepool); 1900 return err; 1901 } 1902 1903 static void z_erofs_readahead(struct readahead_control *rac) 1904 { 1905 struct inode *const inode = rac->mapping->host; 1906 struct erofs_sb_info *const sbi = EROFS_I_SB(inode); 1907 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode); 1908 struct page *head = NULL, *page; 1909 unsigned int nr_pages; 1910 1911 f.headoffset = readahead_pos(rac); 1912 1913 z_erofs_pcluster_readmore(&f, rac, true); 1914 nr_pages = readahead_count(rac); 1915 trace_erofs_readpages(inode, readahead_index(rac), nr_pages, false); 1916 1917 while ((page = readahead_page(rac))) { 1918 set_page_private(page, (unsigned long)head); 1919 head = page; 1920 } 1921 1922 while (head) { 1923 struct page *page = head; 1924 int err; 1925 1926 /* traversal in reverse order */ 1927 head = (void *)page_private(page); 1928 1929 err = z_erofs_do_read_page(&f, page); 1930 if (err) 1931 erofs_err(inode->i_sb, 1932 "readahead error at page %lu @ nid %llu", 1933 page->index, EROFS_I(inode)->nid); 1934 put_page(page); 1935 } 1936 z_erofs_pcluster_readmore(&f, rac, false); 1937 (void)z_erofs_collector_end(&f); 1938 1939 z_erofs_runqueue(&f, z_erofs_is_sync_decompress(sbi, nr_pages), true); 1940 erofs_put_metabuf(&f.map.buf); 1941 erofs_release_pages(&f.pagepool); 1942 } 1943 1944 const struct address_space_operations z_erofs_aops = { 1945 .read_folio = z_erofs_read_folio, 1946 .readahead = z_erofs_readahead, 1947 }; 1948