Lines Matching refs:pcl
111 static inline bool z_erofs_is_inline_pcluster(struct z_erofs_pcluster *pcl) in z_erofs_is_inline_pcluster() argument
113 return !pcl->obj.index; in z_erofs_is_inline_pcluster()
116 static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl) in z_erofs_pclusterpages() argument
118 if (z_erofs_is_inline_pcluster(pcl)) in z_erofs_pclusterpages()
120 return pcl->pclusterpages; in z_erofs_pclusterpages()
307 struct z_erofs_pcluster *pcl; in z_erofs_alloc_pcluster() local
312 pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS); in z_erofs_alloc_pcluster()
313 if (!pcl) in z_erofs_alloc_pcluster()
315 pcl->pclusterpages = nrpages; in z_erofs_alloc_pcluster()
316 return pcl; in z_erofs_alloc_pcluster()
321 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) in z_erofs_free_pcluster() argument
323 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_free_pcluster()
332 kmem_cache_free(pcs->slab, pcl); in z_erofs_free_pcluster()
527 struct z_erofs_pcluster *pcl; member
561 struct z_erofs_pcluster *pcl = fe->pcl; in z_erofs_bind_cache() local
575 for (i = 0; i < pcl->pclusterpages; ++i) { in z_erofs_bind_cache()
581 if (READ_ONCE(pcl->compressed_bvecs[i].page)) in z_erofs_bind_cache()
584 page = find_get_page(mc, pcl->obj.index + i); in z_erofs_bind_cache()
606 if (!cmpxchg_relaxed(&pcl->compressed_bvecs[i].page, NULL, t)) in z_erofs_bind_cache()
627 struct z_erofs_pcluster *const pcl = in erofs_try_to_free_all_cached_pages() local
631 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); in erofs_try_to_free_all_cached_pages()
636 for (i = 0; i < pcl->pclusterpages; ++i) { in erofs_try_to_free_all_cached_pages()
637 struct page *page = pcl->compressed_bvecs[i].page; in erofs_try_to_free_all_cached_pages()
650 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in erofs_try_to_free_all_cached_pages()
659 struct z_erofs_pcluster *pcl = folio_get_private(folio); in z_erofs_cache_release_folio() local
667 spin_lock(&pcl->obj.lockref.lock); in z_erofs_cache_release_folio()
668 if (pcl->obj.lockref.count > 0) in z_erofs_cache_release_folio()
671 DBG_BUGON(z_erofs_is_inline_pcluster(pcl)); in z_erofs_cache_release_folio()
672 for (i = 0; i < pcl->pclusterpages; ++i) { in z_erofs_cache_release_folio()
673 if (pcl->compressed_bvecs[i].page == &folio->page) { in z_erofs_cache_release_folio()
674 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in z_erofs_cache_release_folio()
682 spin_unlock(&pcl->obj.lockref.lock); in z_erofs_cache_release_folio()
727 struct z_erofs_pcluster *const pcl = fe->pcl; in z_erofs_try_inplace_io() local
730 if (!cmpxchg(&pcl->compressed_bvecs[--fe->icur].page, in z_erofs_try_inplace_io()
732 pcl->compressed_bvecs[fe->icur] = *bvec; in z_erofs_try_inplace_io()
756 fe->pcl->vcnt += (ret >= 0); in z_erofs_attach_page()
762 struct z_erofs_pcluster *pcl = f->pcl; in z_erofs_try_to_claim_pcluster() local
766 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL, in z_erofs_try_to_claim_pcluster()
768 *owned_head = &pcl->next; in z_erofs_try_to_claim_pcluster()
782 struct z_erofs_pcluster *pcl; in z_erofs_register_pcluster() local
793 pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 : in z_erofs_register_pcluster()
795 if (IS_ERR(pcl)) in z_erofs_register_pcluster()
796 return PTR_ERR(pcl); in z_erofs_register_pcluster()
798 spin_lock_init(&pcl->obj.lockref.lock); in z_erofs_register_pcluster()
799 pcl->obj.lockref.count = 1; /* one ref for this request */ in z_erofs_register_pcluster()
800 pcl->algorithmformat = map->m_algorithmformat; in z_erofs_register_pcluster()
801 pcl->length = 0; in z_erofs_register_pcluster()
802 pcl->partial = true; in z_erofs_register_pcluster()
805 pcl->next = fe->owned_head; in z_erofs_register_pcluster()
806 pcl->pageofs_out = map->m_la & ~PAGE_MASK; in z_erofs_register_pcluster()
813 mutex_init(&pcl->lock); in z_erofs_register_pcluster()
814 DBG_BUGON(!mutex_trylock(&pcl->lock)); in z_erofs_register_pcluster()
817 pcl->obj.index = 0; /* which indicates ztailpacking */ in z_erofs_register_pcluster()
818 pcl->tailpacking_size = map->m_plen; in z_erofs_register_pcluster()
820 pcl->obj.index = map->m_pa >> PAGE_SHIFT; in z_erofs_register_pcluster()
822 grp = erofs_insert_workgroup(fe->inode->i_sb, &pcl->obj); in z_erofs_register_pcluster()
828 if (grp != &pcl->obj) { in z_erofs_register_pcluster()
829 fe->pcl = container_of(grp, in z_erofs_register_pcluster()
835 fe->owned_head = &pcl->next; in z_erofs_register_pcluster()
836 fe->pcl = pcl; in z_erofs_register_pcluster()
840 mutex_unlock(&pcl->lock); in z_erofs_register_pcluster()
841 z_erofs_free_pcluster(pcl); in z_erofs_register_pcluster()
853 DBG_BUGON(fe->pcl); in z_erofs_pcluster_begin()
866 fe->pcl = container_of(grp, struct z_erofs_pcluster, obj); in z_erofs_pcluster_begin()
873 mutex_lock(&fe->pcl->lock); in z_erofs_pcluster_begin()
879 z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset, in z_erofs_pcluster_begin()
880 Z_EROFS_INLINE_BVECS, fe->pcl->vcnt); in z_erofs_pcluster_begin()
881 if (!z_erofs_is_inline_pcluster(fe->pcl)) { in z_erofs_pcluster_begin()
894 WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, map->buf.page); in z_erofs_pcluster_begin()
895 fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK; in z_erofs_pcluster_begin()
899 fe->icur = z_erofs_pclusterpages(fe->pcl); in z_erofs_pcluster_begin()
915 struct z_erofs_pcluster *const pcl = in erofs_workgroup_free_rcu() local
918 call_rcu(&pcl->rcu, z_erofs_rcu_callback); in erofs_workgroup_free_rcu()
923 struct z_erofs_pcluster *pcl = fe->pcl; in z_erofs_pcluster_end() local
925 if (!pcl) in z_erofs_pcluster_end()
929 mutex_unlock(&pcl->lock); in z_erofs_pcluster_end()
939 erofs_workgroup_put(&pcl->obj); in z_erofs_pcluster_end()
941 fe->pcl = NULL; in z_erofs_pcluster_end()
1017 if (!fe->pcl) { in z_erofs_do_read_page()
1043 if (fe->pcl->pageofs_out != (map->m_la & ~PAGE_MASK)) in z_erofs_do_read_page()
1044 fe->pcl->multibases = true; in z_erofs_do_read_page()
1045 if (fe->pcl->length < offset + end - map->m_la) { in z_erofs_do_read_page()
1046 fe->pcl->length = offset + end - map->m_la; in z_erofs_do_read_page()
1047 fe->pcl->pageofs_out = map->m_la & ~PAGE_MASK; in z_erofs_do_read_page()
1051 fe->pcl->length == map->m_llen) in z_erofs_do_read_page()
1052 fe->pcl->partial = false; in z_erofs_do_read_page()
1090 struct z_erofs_pcluster *pcl; member
1113 if (!((bvec->offset + be->pcl->pageofs_out) & ~PAGE_MASK) && in z_erofs_do_decompressed_bvec()
1115 bvec->offset + bvec->end == be->pcl->length)) { in z_erofs_do_decompressed_bvec()
1116 pgnr = (bvec->offset + be->pcl->pageofs_out) >> PAGE_SHIFT; in z_erofs_do_decompressed_bvec()
1133 unsigned int off0 = be->pcl->pageofs_out; in z_erofs_fill_other_copies()
1143 end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset, in z_erofs_fill_other_copies()
1174 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_parse_out_bvecs() local
1179 z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0); in z_erofs_parse_out_bvecs()
1180 for (i = 0; i < pcl->vcnt; ++i) { in z_erofs_parse_out_bvecs()
1200 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_parse_in_bvecs() local
1201 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_parse_in_bvecs()
1206 struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i]; in z_erofs_parse_in_bvecs()
1216 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_parse_in_bvecs()
1243 struct z_erofs_pcluster *pcl = be->pcl; in z_erofs_decompress_pcluster() local
1244 unsigned int pclusterpages = z_erofs_pclusterpages(pcl); in z_erofs_decompress_pcluster()
1246 &erofs_decompressors[pcl->algorithmformat]; in z_erofs_decompress_pcluster()
1252 mutex_lock(&pcl->lock); in z_erofs_decompress_pcluster()
1253 be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT; in z_erofs_decompress_pcluster()
1285 if (z_erofs_is_inline_pcluster(pcl)) in z_erofs_decompress_pcluster()
1286 inputsize = pcl->tailpacking_size; in z_erofs_decompress_pcluster()
1294 .pageofs_in = pcl->pageofs_in, in z_erofs_decompress_pcluster()
1295 .pageofs_out = pcl->pageofs_out, in z_erofs_decompress_pcluster()
1297 .outputsize = pcl->length, in z_erofs_decompress_pcluster()
1298 .alg = pcl->algorithmformat, in z_erofs_decompress_pcluster()
1300 .partial_decoding = pcl->partial, in z_erofs_decompress_pcluster()
1301 .fillgaps = pcl->multibases, in z_erofs_decompress_pcluster()
1306 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_decompress_pcluster()
1307 page = pcl->compressed_bvecs[0].page; in z_erofs_decompress_pcluster()
1308 WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL); in z_erofs_decompress_pcluster()
1318 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); in z_erofs_decompress_pcluster()
1342 pcl->length = 0; in z_erofs_decompress_pcluster()
1343 pcl->partial = true; in z_erofs_decompress_pcluster()
1344 pcl->multibases = false; in z_erofs_decompress_pcluster()
1345 pcl->bvset.nextpage = NULL; in z_erofs_decompress_pcluster()
1346 pcl->vcnt = 0; in z_erofs_decompress_pcluster()
1349 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL); in z_erofs_decompress_pcluster()
1350 mutex_unlock(&pcl->lock); in z_erofs_decompress_pcluster()
1368 be.pcl = container_of(owned, struct z_erofs_pcluster, next); in z_erofs_decompress_queue()
1369 owned = READ_ONCE(be.pcl->next); in z_erofs_decompress_queue()
1372 if (z_erofs_is_inline_pcluster(be.pcl)) in z_erofs_decompress_queue()
1373 z_erofs_free_pcluster(be.pcl); in z_erofs_decompress_queue()
1375 erofs_workgroup_put(&be.pcl->obj); in z_erofs_decompress_queue()
1438 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl, in pickup_page_for_submission() argument
1443 const pgoff_t index = pcl->obj.index; in pickup_page_for_submission()
1452 page = READ_ONCE(pcl->compressed_bvecs[nr].page); in pickup_page_for_submission()
1466 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); in pickup_page_for_submission()
1487 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page); in pickup_page_for_submission()
1502 set_page_private(page, (unsigned long)pcl); in pickup_page_for_submission()
1506 if (likely(page->private == (unsigned long)pcl)) { in pickup_page_for_submission()
1528 if (oldpage != cmpxchg(&pcl->compressed_bvecs[nr].page, in pickup_page_for_submission()
1540 attach_page_private(page, pcl); in pickup_page_for_submission()
1585 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl, in move_to_bypass_jobqueue() argument
1592 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL); in move_to_bypass_jobqueue()
1595 WRITE_ONCE(*bypass_qtail, &pcl->next); in move_to_bypass_jobqueue()
1597 qtail[JQ_BYPASS] = &pcl->next; in move_to_bypass_jobqueue()
1657 struct z_erofs_pcluster *pcl; in z_erofs_submit_queue() local
1663 pcl = container_of(owned_head, struct z_erofs_pcluster, next); in z_erofs_submit_queue()
1664 owned_head = READ_ONCE(pcl->next); in z_erofs_submit_queue()
1666 if (z_erofs_is_inline_pcluster(pcl)) { in z_erofs_submit_queue()
1667 move_to_bypass_jobqueue(pcl, qtail, owned_head); in z_erofs_submit_queue()
1673 .m_pa = erofs_pos(sb, pcl->obj.index), in z_erofs_submit_queue()
1678 end = cur + pcl->pclusterpages; in z_erofs_submit_queue()
1695 page = pickup_page_for_submission(pcl, i++, in z_erofs_submit_queue()
1728 qtail[JQ_SUBMIT] = &pcl->next; in z_erofs_submit_queue()
1730 move_to_bypass_jobqueue(pcl, qtail, owned_head); in z_erofs_submit_queue()