147e4937aSGao Xiang // SPDX-License-Identifier: GPL-2.0-only 247e4937aSGao Xiang /* 347e4937aSGao Xiang * Copyright (C) 2018-2019 HUAWEI, Inc. 4592e7cd0SAlexander A. Klimov * https://www.huawei.com/ 547e4937aSGao Xiang * Created by Gao Xiang <gaoxiang25@huawei.com> 647e4937aSGao Xiang */ 747e4937aSGao Xiang #include "internal.h" 847e4937aSGao Xiang #include <asm/unaligned.h> 947e4937aSGao Xiang #include <trace/events/erofs.h> 1047e4937aSGao Xiang 1147e4937aSGao Xiang int z_erofs_fill_inode(struct inode *inode) 1247e4937aSGao Xiang { 13a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(inode); 14cec6e93bSGao Xiang struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); 1547e4937aSGao Xiang 16cec6e93bSGao Xiang if (!erofs_sb_has_big_pcluster(sbi) && 17cec6e93bSGao Xiang vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { 1847e4937aSGao Xiang vi->z_advise = 0; 1947e4937aSGao Xiang vi->z_algorithmtype[0] = 0; 2047e4937aSGao Xiang vi->z_algorithmtype[1] = 0; 2147e4937aSGao Xiang vi->z_logical_clusterbits = LOG_BLOCK_SIZE; 22a5876e24SGao Xiang set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 2347e4937aSGao Xiang } 240c638f70SGao Xiang inode->i_mapping->a_ops = &z_erofs_aops; 2547e4937aSGao Xiang return 0; 2647e4937aSGao Xiang } 2747e4937aSGao Xiang 280c638f70SGao Xiang static int z_erofs_fill_inode_lazy(struct inode *inode) 2947e4937aSGao Xiang { 30a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(inode); 3147e4937aSGao Xiang struct super_block *const sb = inode->i_sb; 3247e4937aSGao Xiang int err; 3347e4937aSGao Xiang erofs_off_t pos; 3447e4937aSGao Xiang struct page *page; 3547e4937aSGao Xiang void *kaddr; 3647e4937aSGao Xiang struct z_erofs_map_header *h; 3747e4937aSGao Xiang 38ce063129SGao Xiang if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { 39ce063129SGao Xiang /* 40ce063129SGao Xiang * paired with smp_mb() at the end of the function to ensure 41ce063129SGao Xiang * fields will only be observed after the bit is set. 42ce063129SGao Xiang */ 43ce063129SGao Xiang smp_mb(); 4447e4937aSGao Xiang return 0; 45ce063129SGao Xiang } 4647e4937aSGao Xiang 47a5876e24SGao Xiang if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) 4847e4937aSGao Xiang return -ERESTARTSYS; 4947e4937aSGao Xiang 5047e4937aSGao Xiang err = 0; 51a5876e24SGao Xiang if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) 5247e4937aSGao Xiang goto out_unlock; 5347e4937aSGao Xiang 54cec6e93bSGao Xiang DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && 55cec6e93bSGao Xiang vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY); 5647e4937aSGao Xiang 5747e4937aSGao Xiang pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize + 5847e4937aSGao Xiang vi->xattr_isize, 8); 59e655b5b3SGao Xiang page = erofs_get_meta_page(sb, erofs_blknr(pos)); 6047e4937aSGao Xiang if (IS_ERR(page)) { 6147e4937aSGao Xiang err = PTR_ERR(page); 6247e4937aSGao Xiang goto out_unlock; 6347e4937aSGao Xiang } 6447e4937aSGao Xiang 6547e4937aSGao Xiang kaddr = kmap_atomic(page); 6647e4937aSGao Xiang 6747e4937aSGao Xiang h = kaddr + erofs_blkoff(pos); 6847e4937aSGao Xiang vi->z_advise = le16_to_cpu(h->h_advise); 6947e4937aSGao Xiang vi->z_algorithmtype[0] = h->h_algorithmtype & 15; 7047e4937aSGao Xiang vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; 7147e4937aSGao Xiang 7247e4937aSGao Xiang if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX) { 734f761fa2SGao Xiang erofs_err(sb, "unknown compression format %u for nid %llu, please upgrade kernel", 7447e4937aSGao Xiang vi->z_algorithmtype[0], vi->nid); 7547e4937aSGao Xiang err = -EOPNOTSUPP; 7647e4937aSGao Xiang goto unmap_done; 7747e4937aSGao Xiang } 7847e4937aSGao Xiang 7947e4937aSGao Xiang vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7); 80b86269f4SGao Xiang if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && 81b86269f4SGao Xiang vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 82b86269f4SGao Xiang Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 83b86269f4SGao Xiang erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", 84b86269f4SGao Xiang vi->nid); 85b86269f4SGao Xiang err = -EFSCORRUPTED; 86b86269f4SGao Xiang goto unmap_done; 87b86269f4SGao Xiang } 88b86269f4SGao Xiang if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION && 89b86269f4SGao Xiang !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ 90b86269f4SGao Xiang !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 91b86269f4SGao Xiang erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", 92b86269f4SGao Xiang vi->nid); 93b86269f4SGao Xiang err = -EFSCORRUPTED; 94b86269f4SGao Xiang goto unmap_done; 95b86269f4SGao Xiang } 96ce063129SGao Xiang /* paired with smp_mb() at the beginning of the function */ 97ce063129SGao Xiang smp_mb(); 98a5876e24SGao Xiang set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 9947e4937aSGao Xiang unmap_done: 10047e4937aSGao Xiang kunmap_atomic(kaddr); 10147e4937aSGao Xiang unlock_page(page); 10247e4937aSGao Xiang put_page(page); 10347e4937aSGao Xiang out_unlock: 104a5876e24SGao Xiang clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); 10547e4937aSGao Xiang return err; 10647e4937aSGao Xiang } 10747e4937aSGao Xiang 10847e4937aSGao Xiang struct z_erofs_maprecorder { 10947e4937aSGao Xiang struct inode *inode; 11047e4937aSGao Xiang struct erofs_map_blocks *map; 11147e4937aSGao Xiang void *kaddr; 11247e4937aSGao Xiang 11347e4937aSGao Xiang unsigned long lcn; 11447e4937aSGao Xiang /* compression extent information gathered */ 11547e4937aSGao Xiang u8 type; 11647e4937aSGao Xiang u16 clusterofs; 11747e4937aSGao Xiang u16 delta[2]; 118cec6e93bSGao Xiang erofs_blk_t pblk, compressedlcs; 11947e4937aSGao Xiang }; 12047e4937aSGao Xiang 12147e4937aSGao Xiang static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, 12247e4937aSGao Xiang erofs_blk_t eblk) 12347e4937aSGao Xiang { 12447e4937aSGao Xiang struct super_block *const sb = m->inode->i_sb; 12547e4937aSGao Xiang struct erofs_map_blocks *const map = m->map; 12647e4937aSGao Xiang struct page *mpage = map->mpage; 12747e4937aSGao Xiang 12847e4937aSGao Xiang if (mpage) { 12947e4937aSGao Xiang if (mpage->index == eblk) { 13047e4937aSGao Xiang if (!m->kaddr) 13147e4937aSGao Xiang m->kaddr = kmap_atomic(mpage); 13247e4937aSGao Xiang return 0; 13347e4937aSGao Xiang } 13447e4937aSGao Xiang 13547e4937aSGao Xiang if (m->kaddr) { 13647e4937aSGao Xiang kunmap_atomic(m->kaddr); 13747e4937aSGao Xiang m->kaddr = NULL; 13847e4937aSGao Xiang } 13947e4937aSGao Xiang put_page(mpage); 14047e4937aSGao Xiang } 14147e4937aSGao Xiang 142e655b5b3SGao Xiang mpage = erofs_get_meta_page(sb, eblk); 14347e4937aSGao Xiang if (IS_ERR(mpage)) { 14447e4937aSGao Xiang map->mpage = NULL; 14547e4937aSGao Xiang return PTR_ERR(mpage); 14647e4937aSGao Xiang } 14747e4937aSGao Xiang m->kaddr = kmap_atomic(mpage); 14847e4937aSGao Xiang unlock_page(mpage); 14947e4937aSGao Xiang map->mpage = mpage; 15047e4937aSGao Xiang return 0; 15147e4937aSGao Xiang } 15247e4937aSGao Xiang 1530c638f70SGao Xiang static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, 15447e4937aSGao Xiang unsigned long lcn) 15547e4937aSGao Xiang { 15647e4937aSGao Xiang struct inode *const inode = m->inode; 157a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(inode); 15847e4937aSGao Xiang const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid); 15947e4937aSGao Xiang const erofs_off_t pos = 16047e4937aSGao Xiang Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize + 16147e4937aSGao Xiang vi->xattr_isize) + 16247e4937aSGao Xiang lcn * sizeof(struct z_erofs_vle_decompressed_index); 16347e4937aSGao Xiang struct z_erofs_vle_decompressed_index *di; 16447e4937aSGao Xiang unsigned int advise, type; 16547e4937aSGao Xiang int err; 16647e4937aSGao Xiang 16747e4937aSGao Xiang err = z_erofs_reload_indexes(m, erofs_blknr(pos)); 16847e4937aSGao Xiang if (err) 16947e4937aSGao Xiang return err; 17047e4937aSGao Xiang 17147e4937aSGao Xiang m->lcn = lcn; 17247e4937aSGao Xiang di = m->kaddr + erofs_blkoff(pos); 17347e4937aSGao Xiang 17447e4937aSGao Xiang advise = le16_to_cpu(di->di_advise); 17547e4937aSGao Xiang type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) & 17647e4937aSGao Xiang ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1); 17747e4937aSGao Xiang switch (type) { 17847e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 17947e4937aSGao Xiang m->clusterofs = 1 << vi->z_logical_clusterbits; 18047e4937aSGao Xiang m->delta[0] = le16_to_cpu(di->di_u.delta[0]); 181cec6e93bSGao Xiang if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) { 182cec6e93bSGao Xiang if (!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) { 183cec6e93bSGao Xiang DBG_BUGON(1); 184cec6e93bSGao Xiang return -EFSCORRUPTED; 185cec6e93bSGao Xiang } 186cec6e93bSGao Xiang m->compressedlcs = m->delta[0] & 187cec6e93bSGao Xiang ~Z_EROFS_VLE_DI_D0_CBLKCNT; 188cec6e93bSGao Xiang m->delta[0] = 1; 189cec6e93bSGao Xiang } 19047e4937aSGao Xiang m->delta[1] = le16_to_cpu(di->di_u.delta[1]); 19147e4937aSGao Xiang break; 19247e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 19347e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: 19447e4937aSGao Xiang m->clusterofs = le16_to_cpu(di->di_clusterofs); 19547e4937aSGao Xiang m->pblk = le32_to_cpu(di->di_u.blkaddr); 19647e4937aSGao Xiang break; 19747e4937aSGao Xiang default: 19847e4937aSGao Xiang DBG_BUGON(1); 19947e4937aSGao Xiang return -EOPNOTSUPP; 20047e4937aSGao Xiang } 20147e4937aSGao Xiang m->type = type; 20247e4937aSGao Xiang return 0; 20347e4937aSGao Xiang } 20447e4937aSGao Xiang 20547e4937aSGao Xiang static unsigned int decode_compactedbits(unsigned int lobits, 20647e4937aSGao Xiang unsigned int lomask, 20747e4937aSGao Xiang u8 *in, unsigned int pos, u8 *type) 20847e4937aSGao Xiang { 20947e4937aSGao Xiang const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); 21047e4937aSGao Xiang const unsigned int lo = v & lomask; 21147e4937aSGao Xiang 21247e4937aSGao Xiang *type = (v >> lobits) & 3; 21347e4937aSGao Xiang return lo; 21447e4937aSGao Xiang } 21547e4937aSGao Xiang 21647e4937aSGao Xiang static int unpack_compacted_index(struct z_erofs_maprecorder *m, 21747e4937aSGao Xiang unsigned int amortizedshift, 21847e4937aSGao Xiang unsigned int eofs) 21947e4937aSGao Xiang { 220a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(m->inode); 22147e4937aSGao Xiang const unsigned int lclusterbits = vi->z_logical_clusterbits; 22247e4937aSGao Xiang const unsigned int lomask = (1 << lclusterbits) - 1; 22347e4937aSGao Xiang unsigned int vcnt, base, lo, encodebits, nblk; 22447e4937aSGao Xiang int i; 22547e4937aSGao Xiang u8 *in, type; 226b86269f4SGao Xiang bool big_pcluster; 22747e4937aSGao Xiang 22847e4937aSGao Xiang if (1 << amortizedshift == 4) 22947e4937aSGao Xiang vcnt = 2; 23047e4937aSGao Xiang else if (1 << amortizedshift == 2 && lclusterbits == 12) 23147e4937aSGao Xiang vcnt = 16; 23247e4937aSGao Xiang else 23347e4937aSGao Xiang return -EOPNOTSUPP; 23447e4937aSGao Xiang 235b86269f4SGao Xiang big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; 23647e4937aSGao Xiang encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; 23747e4937aSGao Xiang base = round_down(eofs, vcnt << amortizedshift); 23847e4937aSGao Xiang in = m->kaddr + base; 23947e4937aSGao Xiang 24047e4937aSGao Xiang i = (eofs - base) >> amortizedshift; 24147e4937aSGao Xiang 24247e4937aSGao Xiang lo = decode_compactedbits(lclusterbits, lomask, 24347e4937aSGao Xiang in, encodebits * i, &type); 24447e4937aSGao Xiang m->type = type; 24547e4937aSGao Xiang if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 24647e4937aSGao Xiang m->clusterofs = 1 << lclusterbits; 247b86269f4SGao Xiang if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { 248b86269f4SGao Xiang if (!big_pcluster) { 249b86269f4SGao Xiang DBG_BUGON(1); 250b86269f4SGao Xiang return -EFSCORRUPTED; 251b86269f4SGao Xiang } 252b86269f4SGao Xiang m->compressedlcs = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; 253b86269f4SGao Xiang m->delta[0] = 1; 254b86269f4SGao Xiang return 0; 255b86269f4SGao Xiang } else if (i + 1 != (int)vcnt) { 25647e4937aSGao Xiang m->delta[0] = lo; 25747e4937aSGao Xiang return 0; 25847e4937aSGao Xiang } 25947e4937aSGao Xiang /* 26047e4937aSGao Xiang * since the last lcluster in the pack is special, 26147e4937aSGao Xiang * of which lo saves delta[1] rather than delta[0]. 26247e4937aSGao Xiang * Hence, get delta[0] by the previous lcluster indirectly. 26347e4937aSGao Xiang */ 26447e4937aSGao Xiang lo = decode_compactedbits(lclusterbits, lomask, 26547e4937aSGao Xiang in, encodebits * (i - 1), &type); 26647e4937aSGao Xiang if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 26747e4937aSGao Xiang lo = 0; 268b86269f4SGao Xiang else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) 269b86269f4SGao Xiang lo = 1; 27047e4937aSGao Xiang m->delta[0] = lo + 1; 27147e4937aSGao Xiang return 0; 27247e4937aSGao Xiang } 27347e4937aSGao Xiang m->clusterofs = lo; 27447e4937aSGao Xiang m->delta[0] = 0; 27547e4937aSGao Xiang /* figout out blkaddr (pblk) for HEAD lclusters */ 276b86269f4SGao Xiang if (!big_pcluster) { 27747e4937aSGao Xiang nblk = 1; 27847e4937aSGao Xiang while (i > 0) { 27947e4937aSGao Xiang --i; 28047e4937aSGao Xiang lo = decode_compactedbits(lclusterbits, lomask, 28147e4937aSGao Xiang in, encodebits * i, &type); 28247e4937aSGao Xiang if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 28347e4937aSGao Xiang i -= lo; 28447e4937aSGao Xiang 28547e4937aSGao Xiang if (i >= 0) 28647e4937aSGao Xiang ++nblk; 28747e4937aSGao Xiang } 288b86269f4SGao Xiang } else { 289b86269f4SGao Xiang nblk = 0; 290b86269f4SGao Xiang while (i > 0) { 291b86269f4SGao Xiang --i; 292b86269f4SGao Xiang lo = decode_compactedbits(lclusterbits, lomask, 293b86269f4SGao Xiang in, encodebits * i, &type); 294b86269f4SGao Xiang if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 295b86269f4SGao Xiang if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { 296b86269f4SGao Xiang --i; 297b86269f4SGao Xiang nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; 298b86269f4SGao Xiang continue; 299b86269f4SGao Xiang } 300b86269f4SGao Xiang /* bigpcluster shouldn't have plain d0 == 1 */ 301b86269f4SGao Xiang if (lo <= 1) { 302b86269f4SGao Xiang DBG_BUGON(1); 303b86269f4SGao Xiang return -EFSCORRUPTED; 304b86269f4SGao Xiang } 305b86269f4SGao Xiang i -= lo - 2; 306b86269f4SGao Xiang continue; 307b86269f4SGao Xiang } 308b86269f4SGao Xiang ++nblk; 309b86269f4SGao Xiang } 310b86269f4SGao Xiang } 31147e4937aSGao Xiang in += (vcnt << amortizedshift) - sizeof(__le32); 31247e4937aSGao Xiang m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; 31347e4937aSGao Xiang return 0; 31447e4937aSGao Xiang } 31547e4937aSGao Xiang 31647e4937aSGao Xiang static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, 31747e4937aSGao Xiang unsigned long lcn) 31847e4937aSGao Xiang { 31947e4937aSGao Xiang struct inode *const inode = m->inode; 320a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(inode); 32147e4937aSGao Xiang const unsigned int lclusterbits = vi->z_logical_clusterbits; 32247e4937aSGao Xiang const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) + 32347e4937aSGao Xiang vi->inode_isize + vi->xattr_isize, 8) + 32447e4937aSGao Xiang sizeof(struct z_erofs_map_header); 32547e4937aSGao Xiang const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); 32647e4937aSGao Xiang unsigned int compacted_4b_initial, compacted_2b; 32747e4937aSGao Xiang unsigned int amortizedshift; 32847e4937aSGao Xiang erofs_off_t pos; 32947e4937aSGao Xiang int err; 33047e4937aSGao Xiang 33147e4937aSGao Xiang if (lclusterbits != 12) 33247e4937aSGao Xiang return -EOPNOTSUPP; 33347e4937aSGao Xiang 33447e4937aSGao Xiang if (lcn >= totalidx) 33547e4937aSGao Xiang return -EINVAL; 33647e4937aSGao Xiang 33747e4937aSGao Xiang m->lcn = lcn; 33847e4937aSGao Xiang /* used to align to 32-byte (compacted_2b) alignment */ 33947e4937aSGao Xiang compacted_4b_initial = (32 - ebase % 32) / 4; 34047e4937aSGao Xiang if (compacted_4b_initial == 32 / 4) 34147e4937aSGao Xiang compacted_4b_initial = 0; 34247e4937aSGao Xiang 34347e4937aSGao Xiang if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) 34447e4937aSGao Xiang compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); 34547e4937aSGao Xiang else 34647e4937aSGao Xiang compacted_2b = 0; 34747e4937aSGao Xiang 34847e4937aSGao Xiang pos = ebase; 34947e4937aSGao Xiang if (lcn < compacted_4b_initial) { 35047e4937aSGao Xiang amortizedshift = 2; 35147e4937aSGao Xiang goto out; 35247e4937aSGao Xiang } 35347e4937aSGao Xiang pos += compacted_4b_initial * 4; 35447e4937aSGao Xiang lcn -= compacted_4b_initial; 35547e4937aSGao Xiang 35647e4937aSGao Xiang if (lcn < compacted_2b) { 35747e4937aSGao Xiang amortizedshift = 1; 35847e4937aSGao Xiang goto out; 35947e4937aSGao Xiang } 36047e4937aSGao Xiang pos += compacted_2b * 2; 36147e4937aSGao Xiang lcn -= compacted_2b; 36247e4937aSGao Xiang amortizedshift = 2; 36347e4937aSGao Xiang out: 36447e4937aSGao Xiang pos += lcn * (1 << amortizedshift); 36547e4937aSGao Xiang err = z_erofs_reload_indexes(m, erofs_blknr(pos)); 36647e4937aSGao Xiang if (err) 36747e4937aSGao Xiang return err; 36847e4937aSGao Xiang return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos)); 36947e4937aSGao Xiang } 37047e4937aSGao Xiang 3710c638f70SGao Xiang static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, 37247e4937aSGao Xiang unsigned int lcn) 37347e4937aSGao Xiang { 374a5876e24SGao Xiang const unsigned int datamode = EROFS_I(m->inode)->datalayout; 37547e4937aSGao Xiang 37647e4937aSGao Xiang if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY) 3770c638f70SGao Xiang return legacy_load_cluster_from_disk(m, lcn); 37847e4937aSGao Xiang 37947e4937aSGao Xiang if (datamode == EROFS_INODE_FLAT_COMPRESSION) 38047e4937aSGao Xiang return compacted_load_cluster_from_disk(m, lcn); 38147e4937aSGao Xiang 38247e4937aSGao Xiang return -EINVAL; 38347e4937aSGao Xiang } 38447e4937aSGao Xiang 3850c638f70SGao Xiang static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, 38647e4937aSGao Xiang unsigned int lookback_distance) 38747e4937aSGao Xiang { 388a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(m->inode); 38947e4937aSGao Xiang struct erofs_map_blocks *const map = m->map; 39047e4937aSGao Xiang const unsigned int lclusterbits = vi->z_logical_clusterbits; 39147e4937aSGao Xiang unsigned long lcn = m->lcn; 39247e4937aSGao Xiang int err; 39347e4937aSGao Xiang 39447e4937aSGao Xiang if (lcn < lookback_distance) { 3954f761fa2SGao Xiang erofs_err(m->inode->i_sb, 3964f761fa2SGao Xiang "bogus lookback distance @ nid %llu", vi->nid); 39747e4937aSGao Xiang DBG_BUGON(1); 39847e4937aSGao Xiang return -EFSCORRUPTED; 39947e4937aSGao Xiang } 40047e4937aSGao Xiang 40147e4937aSGao Xiang /* load extent head logical cluster if needed */ 40247e4937aSGao Xiang lcn -= lookback_distance; 4030c638f70SGao Xiang err = z_erofs_load_cluster_from_disk(m, lcn); 40447e4937aSGao Xiang if (err) 40547e4937aSGao Xiang return err; 40647e4937aSGao Xiang 40747e4937aSGao Xiang switch (m->type) { 40847e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 4098d8a09b0SGao Xiang if (!m->delta[0]) { 4104f761fa2SGao Xiang erofs_err(m->inode->i_sb, 4114f761fa2SGao Xiang "invalid lookback distance 0 @ nid %llu", 41247e4937aSGao Xiang vi->nid); 41347e4937aSGao Xiang DBG_BUGON(1); 41447e4937aSGao Xiang return -EFSCORRUPTED; 41547e4937aSGao Xiang } 4160c638f70SGao Xiang return z_erofs_extent_lookback(m, m->delta[0]); 41747e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 41847e4937aSGao Xiang map->m_flags &= ~EROFS_MAP_ZIPPED; 419df561f66SGustavo A. R. Silva fallthrough; 42047e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: 42147e4937aSGao Xiang map->m_la = (lcn << lclusterbits) | m->clusterofs; 42247e4937aSGao Xiang break; 42347e4937aSGao Xiang default: 4244f761fa2SGao Xiang erofs_err(m->inode->i_sb, 4254f761fa2SGao Xiang "unknown type %u @ lcn %lu of nid %llu", 42647e4937aSGao Xiang m->type, lcn, vi->nid); 42747e4937aSGao Xiang DBG_BUGON(1); 42847e4937aSGao Xiang return -EOPNOTSUPP; 42947e4937aSGao Xiang } 43047e4937aSGao Xiang return 0; 43147e4937aSGao Xiang } 43247e4937aSGao Xiang 433cec6e93bSGao Xiang static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, 434cec6e93bSGao Xiang unsigned int initial_lcn) 435cec6e93bSGao Xiang { 436cec6e93bSGao Xiang struct erofs_inode *const vi = EROFS_I(m->inode); 437cec6e93bSGao Xiang struct erofs_map_blocks *const map = m->map; 438cec6e93bSGao Xiang const unsigned int lclusterbits = vi->z_logical_clusterbits; 439cec6e93bSGao Xiang unsigned long lcn; 440cec6e93bSGao Xiang int err; 441cec6e93bSGao Xiang 442cec6e93bSGao Xiang DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN && 443cec6e93bSGao Xiang m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD); 444cec6e93bSGao Xiang if (!(map->m_flags & EROFS_MAP_ZIPPED) || 445cec6e93bSGao Xiang !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) { 446cec6e93bSGao Xiang map->m_plen = 1 << lclusterbits; 447cec6e93bSGao Xiang return 0; 448cec6e93bSGao Xiang } 449cec6e93bSGao Xiang 450cec6e93bSGao Xiang lcn = m->lcn + 1; 451cec6e93bSGao Xiang if (m->compressedlcs) 452cec6e93bSGao Xiang goto out; 453cec6e93bSGao Xiang 454cec6e93bSGao Xiang err = z_erofs_load_cluster_from_disk(m, lcn); 455cec6e93bSGao Xiang if (err) 456cec6e93bSGao Xiang return err; 457cec6e93bSGao Xiang 458*0852b6caSGao Xiang /* 459*0852b6caSGao Xiang * If the 1st NONHEAD lcluster has already been handled initially w/o 460*0852b6caSGao Xiang * valid compressedlcs, which means at least it mustn't be CBLKCNT, or 461*0852b6caSGao Xiang * an internal implemenatation error is detected. 462*0852b6caSGao Xiang * 463*0852b6caSGao Xiang * The following code can also handle it properly anyway, but let's 464*0852b6caSGao Xiang * BUG_ON in the debugging mode only for developers to notice that. 465*0852b6caSGao Xiang */ 466*0852b6caSGao Xiang DBG_BUGON(lcn == initial_lcn && 467*0852b6caSGao Xiang m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD); 468*0852b6caSGao Xiang 469cec6e93bSGao Xiang switch (m->type) { 470*0852b6caSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 471*0852b6caSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: 472*0852b6caSGao Xiang /* 473*0852b6caSGao Xiang * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type 474*0852b6caSGao Xiang * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. 475*0852b6caSGao Xiang */ 476*0852b6caSGao Xiang m->compressedlcs = 1; 477*0852b6caSGao Xiang break; 478cec6e93bSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 479cec6e93bSGao Xiang if (m->delta[0] != 1) 480cec6e93bSGao Xiang goto err_bonus_cblkcnt; 481cec6e93bSGao Xiang if (m->compressedlcs) 482cec6e93bSGao Xiang break; 483cec6e93bSGao Xiang fallthrough; 484cec6e93bSGao Xiang default: 485cec6e93bSGao Xiang erofs_err(m->inode->i_sb, 486cec6e93bSGao Xiang "cannot found CBLKCNT @ lcn %lu of nid %llu", 487cec6e93bSGao Xiang lcn, vi->nid); 488cec6e93bSGao Xiang DBG_BUGON(1); 489cec6e93bSGao Xiang return -EFSCORRUPTED; 490cec6e93bSGao Xiang } 491cec6e93bSGao Xiang out: 492cec6e93bSGao Xiang map->m_plen = m->compressedlcs << lclusterbits; 493cec6e93bSGao Xiang return 0; 494cec6e93bSGao Xiang err_bonus_cblkcnt: 495cec6e93bSGao Xiang erofs_err(m->inode->i_sb, 496cec6e93bSGao Xiang "bogus CBLKCNT @ lcn %lu of nid %llu", 497cec6e93bSGao Xiang lcn, vi->nid); 498cec6e93bSGao Xiang DBG_BUGON(1); 499cec6e93bSGao Xiang return -EFSCORRUPTED; 500cec6e93bSGao Xiang } 501cec6e93bSGao Xiang 50247e4937aSGao Xiang int z_erofs_map_blocks_iter(struct inode *inode, 50347e4937aSGao Xiang struct erofs_map_blocks *map, 50447e4937aSGao Xiang int flags) 50547e4937aSGao Xiang { 506a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(inode); 50747e4937aSGao Xiang struct z_erofs_maprecorder m = { 50847e4937aSGao Xiang .inode = inode, 50947e4937aSGao Xiang .map = map, 51047e4937aSGao Xiang }; 51147e4937aSGao Xiang int err = 0; 51247e4937aSGao Xiang unsigned int lclusterbits, endoff; 513cec6e93bSGao Xiang unsigned long initial_lcn; 51447e4937aSGao Xiang unsigned long long ofs, end; 51547e4937aSGao Xiang 51647e4937aSGao Xiang trace_z_erofs_map_blocks_iter_enter(inode, map, flags); 51747e4937aSGao Xiang 51847e4937aSGao Xiang /* when trying to read beyond EOF, leave it unmapped */ 5198d8a09b0SGao Xiang if (map->m_la >= inode->i_size) { 52047e4937aSGao Xiang map->m_llen = map->m_la + 1 - inode->i_size; 52147e4937aSGao Xiang map->m_la = inode->i_size; 52247e4937aSGao Xiang map->m_flags = 0; 52347e4937aSGao Xiang goto out; 52447e4937aSGao Xiang } 52547e4937aSGao Xiang 5260c638f70SGao Xiang err = z_erofs_fill_inode_lazy(inode); 52747e4937aSGao Xiang if (err) 52847e4937aSGao Xiang goto out; 52947e4937aSGao Xiang 53047e4937aSGao Xiang lclusterbits = vi->z_logical_clusterbits; 53147e4937aSGao Xiang ofs = map->m_la; 532cec6e93bSGao Xiang initial_lcn = ofs >> lclusterbits; 53347e4937aSGao Xiang endoff = ofs & ((1 << lclusterbits) - 1); 53447e4937aSGao Xiang 535cec6e93bSGao Xiang err = z_erofs_load_cluster_from_disk(&m, initial_lcn); 53647e4937aSGao Xiang if (err) 53747e4937aSGao Xiang goto unmap_out; 53847e4937aSGao Xiang 53947e4937aSGao Xiang map->m_flags = EROFS_MAP_ZIPPED; /* by default, compressed */ 54047e4937aSGao Xiang end = (m.lcn + 1ULL) << lclusterbits; 54147e4937aSGao Xiang 54247e4937aSGao Xiang switch (m.type) { 54347e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 54447e4937aSGao Xiang if (endoff >= m.clusterofs) 54547e4937aSGao Xiang map->m_flags &= ~EROFS_MAP_ZIPPED; 546df561f66SGustavo A. R. Silva fallthrough; 54747e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_HEAD: 54847e4937aSGao Xiang if (endoff >= m.clusterofs) { 54947e4937aSGao Xiang map->m_la = (m.lcn << lclusterbits) | m.clusterofs; 55047e4937aSGao Xiang break; 55147e4937aSGao Xiang } 55247e4937aSGao Xiang /* m.lcn should be >= 1 if endoff < m.clusterofs */ 5538d8a09b0SGao Xiang if (!m.lcn) { 5544f761fa2SGao Xiang erofs_err(inode->i_sb, 5554f761fa2SGao Xiang "invalid logical cluster 0 at nid %llu", 55647e4937aSGao Xiang vi->nid); 55747e4937aSGao Xiang err = -EFSCORRUPTED; 55847e4937aSGao Xiang goto unmap_out; 55947e4937aSGao Xiang } 56047e4937aSGao Xiang end = (m.lcn << lclusterbits) | m.clusterofs; 56147e4937aSGao Xiang map->m_flags |= EROFS_MAP_FULL_MAPPED; 56247e4937aSGao Xiang m.delta[0] = 1; 563df561f66SGustavo A. R. Silva fallthrough; 56447e4937aSGao Xiang case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 565fe6adcceSRuiqi Gong /* get the corresponding first chunk */ 5660c638f70SGao Xiang err = z_erofs_extent_lookback(&m, m.delta[0]); 5678d8a09b0SGao Xiang if (err) 56847e4937aSGao Xiang goto unmap_out; 56947e4937aSGao Xiang break; 57047e4937aSGao Xiang default: 5714f761fa2SGao Xiang erofs_err(inode->i_sb, 5724f761fa2SGao Xiang "unknown type %u @ offset %llu of nid %llu", 57347e4937aSGao Xiang m.type, ofs, vi->nid); 57447e4937aSGao Xiang err = -EOPNOTSUPP; 57547e4937aSGao Xiang goto unmap_out; 57647e4937aSGao Xiang } 57747e4937aSGao Xiang 57847e4937aSGao Xiang map->m_llen = end - map->m_la; 57947e4937aSGao Xiang map->m_pa = blknr_to_addr(m.pblk); 58047e4937aSGao Xiang map->m_flags |= EROFS_MAP_MAPPED; 58147e4937aSGao Xiang 582cec6e93bSGao Xiang err = z_erofs_get_extent_compressedlen(&m, initial_lcn); 583cec6e93bSGao Xiang if (err) 584cec6e93bSGao Xiang goto out; 58547e4937aSGao Xiang unmap_out: 58647e4937aSGao Xiang if (m.kaddr) 58747e4937aSGao Xiang kunmap_atomic(m.kaddr); 58847e4937aSGao Xiang 58947e4937aSGao Xiang out: 5904f761fa2SGao Xiang erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o", 59147e4937aSGao Xiang __func__, map->m_la, map->m_pa, 59247e4937aSGao Xiang map->m_llen, map->m_plen, map->m_flags); 59347e4937aSGao Xiang 59447e4937aSGao Xiang trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); 59547e4937aSGao Xiang 59647e4937aSGao Xiang /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */ 59747e4937aSGao Xiang DBG_BUGON(err < 0 && err != -ENOMEM); 59847e4937aSGao Xiang return err; 59947e4937aSGao Xiang } 60047e4937aSGao Xiang 601