147e4937aSGao Xiang // SPDX-License-Identifier: GPL-2.0-only
247e4937aSGao Xiang /*
347e4937aSGao Xiang * Copyright (C) 2018-2019 HUAWEI, Inc.
4592e7cd0SAlexander A. Klimov * https://www.huawei.com/
547e4937aSGao Xiang */
647e4937aSGao Xiang #include "internal.h"
747e4937aSGao Xiang #include <asm/unaligned.h>
847e4937aSGao Xiang #include <trace/events/erofs.h>
947e4937aSGao Xiang
1047e4937aSGao Xiang struct z_erofs_maprecorder {
1147e4937aSGao Xiang struct inode *inode;
1247e4937aSGao Xiang struct erofs_map_blocks *map;
1347e4937aSGao Xiang void *kaddr;
1447e4937aSGao Xiang
1547e4937aSGao Xiang unsigned long lcn;
1647e4937aSGao Xiang /* compression extent information gathered */
178f899262SGao Xiang u8 type, headtype;
1847e4937aSGao Xiang u16 clusterofs;
1947e4937aSGao Xiang u16 delta[2];
20ea0b7b0dSYue Hu erofs_blk_t pblk, compressedblks;
21ab92184fSYue Hu erofs_off_t nextpackoff;
225c2a6425SGao Xiang bool partialref;
2347e4937aSGao Xiang };
2447e4937aSGao Xiang
z_erofs_load_full_lcluster(struct z_erofs_maprecorder * m,unsigned long lcn)258241fdd3SGao Xiang static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m,
2647e4937aSGao Xiang unsigned long lcn)
2747e4937aSGao Xiang {
2847e4937aSGao Xiang struct inode *const inode = m->inode;
29a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(inode);
301c7f49a7SGao Xiang const erofs_off_t pos = Z_EROFS_FULL_INDEX_ALIGN(erofs_iloc(inode) +
31b780d3fcSGao Xiang vi->inode_isize + vi->xattr_isize) +
321c7f49a7SGao Xiang lcn * sizeof(struct z_erofs_lcluster_index);
331c7f49a7SGao Xiang struct z_erofs_lcluster_index *di;
3447e4937aSGao Xiang unsigned int advise, type;
3547e4937aSGao Xiang
3631da107fSYue Hu m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
373acea5fcSJingbo Xu erofs_blknr(inode->i_sb, pos), EROFS_KMAP);
3831da107fSYue Hu if (IS_ERR(m->kaddr))
3931da107fSYue Hu return PTR_ERR(m->kaddr);
4047e4937aSGao Xiang
411c7f49a7SGao Xiang m->nextpackoff = pos + sizeof(struct z_erofs_lcluster_index);
4247e4937aSGao Xiang m->lcn = lcn;
433acea5fcSJingbo Xu di = m->kaddr + erofs_blkoff(inode->i_sb, pos);
4447e4937aSGao Xiang
4547e4937aSGao Xiang advise = le16_to_cpu(di->di_advise);
461c7f49a7SGao Xiang type = (advise >> Z_EROFS_LI_LCLUSTER_TYPE_BIT) &
471c7f49a7SGao Xiang ((1 << Z_EROFS_LI_LCLUSTER_TYPE_BITS) - 1);
4847e4937aSGao Xiang switch (type) {
491c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_NONHEAD:
5047e4937aSGao Xiang m->clusterofs = 1 << vi->z_logical_clusterbits;
5147e4937aSGao Xiang m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
521c7f49a7SGao Xiang if (m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) {
5372bb5262SGao Xiang if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
5472bb5262SGao Xiang Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
55cec6e93bSGao Xiang DBG_BUGON(1);
56cec6e93bSGao Xiang return -EFSCORRUPTED;
57cec6e93bSGao Xiang }
58ea0b7b0dSYue Hu m->compressedblks = m->delta[0] &
591c7f49a7SGao Xiang ~Z_EROFS_LI_D0_CBLKCNT;
60cec6e93bSGao Xiang m->delta[0] = 1;
61cec6e93bSGao Xiang }
6247e4937aSGao Xiang m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
6347e4937aSGao Xiang break;
641c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_PLAIN:
651c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_HEAD1:
661c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_HEAD2:
671c7f49a7SGao Xiang if (advise & Z_EROFS_LI_PARTIAL_REF)
685c2a6425SGao Xiang m->partialref = true;
6947e4937aSGao Xiang m->clusterofs = le16_to_cpu(di->di_clusterofs);
70cc4efd3dSGao Xiang if (m->clusterofs >= 1 << vi->z_logical_clusterbits) {
71cc4efd3dSGao Xiang DBG_BUGON(1);
72cc4efd3dSGao Xiang return -EFSCORRUPTED;
73cc4efd3dSGao Xiang }
7447e4937aSGao Xiang m->pblk = le32_to_cpu(di->di_u.blkaddr);
7547e4937aSGao Xiang break;
7647e4937aSGao Xiang default:
7747e4937aSGao Xiang DBG_BUGON(1);
7847e4937aSGao Xiang return -EOPNOTSUPP;
7947e4937aSGao Xiang }
8047e4937aSGao Xiang m->type = type;
8147e4937aSGao Xiang return 0;
8247e4937aSGao Xiang }
8347e4937aSGao Xiang
decode_compactedbits(unsigned int lobits,u8 * in,unsigned int pos,u8 * type)8447e4937aSGao Xiang static unsigned int decode_compactedbits(unsigned int lobits,
8547e4937aSGao Xiang u8 *in, unsigned int pos, u8 *type)
8647e4937aSGao Xiang {
8747e4937aSGao Xiang const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
88*3f691aa6SGao Xiang const unsigned int lo = v & ((1 << lobits) - 1);
8947e4937aSGao Xiang
9047e4937aSGao Xiang *type = (v >> lobits) & 3;
9147e4937aSGao Xiang return lo;
9247e4937aSGao Xiang }
9347e4937aSGao Xiang
get_compacted_la_distance(unsigned int lobits,unsigned int encodebits,unsigned int vcnt,u8 * in,int i)94*3f691aa6SGao Xiang static int get_compacted_la_distance(unsigned int lobits,
95d95ae5e2SGao Xiang unsigned int encodebits,
96d95ae5e2SGao Xiang unsigned int vcnt, u8 *in, int i)
97d95ae5e2SGao Xiang {
98d95ae5e2SGao Xiang unsigned int lo, d1 = 0;
99d95ae5e2SGao Xiang u8 type;
100d95ae5e2SGao Xiang
101d95ae5e2SGao Xiang DBG_BUGON(i >= vcnt);
102d95ae5e2SGao Xiang
103d95ae5e2SGao Xiang do {
104*3f691aa6SGao Xiang lo = decode_compactedbits(lobits, in, encodebits * i, &type);
105d95ae5e2SGao Xiang
1061c7f49a7SGao Xiang if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)
107d95ae5e2SGao Xiang return d1;
108d95ae5e2SGao Xiang ++d1;
109d95ae5e2SGao Xiang } while (++i < vcnt);
110d95ae5e2SGao Xiang
1111c7f49a7SGao Xiang /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */
1121c7f49a7SGao Xiang if (!(lo & Z_EROFS_LI_D0_CBLKCNT))
113d95ae5e2SGao Xiang d1 += lo - 1;
114d95ae5e2SGao Xiang return d1;
115d95ae5e2SGao Xiang }
116d95ae5e2SGao Xiang
unpack_compacted_index(struct z_erofs_maprecorder * m,unsigned int amortizedshift,erofs_off_t pos,bool lookahead)11747e4937aSGao Xiang static int unpack_compacted_index(struct z_erofs_maprecorder *m,
11847e4937aSGao Xiang unsigned int amortizedshift,
119ab92184fSYue Hu erofs_off_t pos, bool lookahead)
12047e4937aSGao Xiang {
121a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(m->inode);
12247e4937aSGao Xiang const unsigned int lclusterbits = vi->z_logical_clusterbits;
123*3f691aa6SGao Xiang unsigned int vcnt, base, lo, lobits, encodebits, nblk, eofs;
12447e4937aSGao Xiang int i;
12547e4937aSGao Xiang u8 *in, type;
126b86269f4SGao Xiang bool big_pcluster;
12747e4937aSGao Xiang
128001b8ccdSGao Xiang if (1 << amortizedshift == 4 && lclusterbits <= 14)
12947e4937aSGao Xiang vcnt = 2;
130*3f691aa6SGao Xiang else if (1 << amortizedshift == 2 && lclusterbits <= 12)
13147e4937aSGao Xiang vcnt = 16;
13247e4937aSGao Xiang else
13347e4937aSGao Xiang return -EOPNOTSUPP;
13447e4937aSGao Xiang
135ab92184fSYue Hu /* it doesn't equal to round_up(..) */
136ab92184fSYue Hu m->nextpackoff = round_down(pos, vcnt << amortizedshift) +
137ab92184fSYue Hu (vcnt << amortizedshift);
138b86269f4SGao Xiang big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
139*3f691aa6SGao Xiang lobits = max(lclusterbits, ilog2(Z_EROFS_LI_D0_CBLKCNT) + 1U);
14047e4937aSGao Xiang encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
1413acea5fcSJingbo Xu eofs = erofs_blkoff(m->inode->i_sb, pos);
14247e4937aSGao Xiang base = round_down(eofs, vcnt << amortizedshift);
14347e4937aSGao Xiang in = m->kaddr + base;
14447e4937aSGao Xiang
14547e4937aSGao Xiang i = (eofs - base) >> amortizedshift;
14647e4937aSGao Xiang
147*3f691aa6SGao Xiang lo = decode_compactedbits(lobits, in, encodebits * i, &type);
14847e4937aSGao Xiang m->type = type;
1491c7f49a7SGao Xiang if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
15047e4937aSGao Xiang m->clusterofs = 1 << lclusterbits;
151d95ae5e2SGao Xiang
152d95ae5e2SGao Xiang /* figure out lookahead_distance: delta[1] if needed */
153d95ae5e2SGao Xiang if (lookahead)
154*3f691aa6SGao Xiang m->delta[1] = get_compacted_la_distance(lobits,
155d95ae5e2SGao Xiang encodebits, vcnt, in, i);
1561c7f49a7SGao Xiang if (lo & Z_EROFS_LI_D0_CBLKCNT) {
157b86269f4SGao Xiang if (!big_pcluster) {
158b86269f4SGao Xiang DBG_BUGON(1);
159b86269f4SGao Xiang return -EFSCORRUPTED;
160b86269f4SGao Xiang }
1611c7f49a7SGao Xiang m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT;
162b86269f4SGao Xiang m->delta[0] = 1;
163b86269f4SGao Xiang return 0;
164b86269f4SGao Xiang } else if (i + 1 != (int)vcnt) {
16547e4937aSGao Xiang m->delta[0] = lo;
16647e4937aSGao Xiang return 0;
16747e4937aSGao Xiang }
16847e4937aSGao Xiang /*
16947e4937aSGao Xiang * since the last lcluster in the pack is special,
17047e4937aSGao Xiang * of which lo saves delta[1] rather than delta[0].
17147e4937aSGao Xiang * Hence, get delta[0] by the previous lcluster indirectly.
17247e4937aSGao Xiang */
173*3f691aa6SGao Xiang lo = decode_compactedbits(lobits, in,
174*3f691aa6SGao Xiang encodebits * (i - 1), &type);
1751c7f49a7SGao Xiang if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD)
17647e4937aSGao Xiang lo = 0;
1771c7f49a7SGao Xiang else if (lo & Z_EROFS_LI_D0_CBLKCNT)
178b86269f4SGao Xiang lo = 1;
17947e4937aSGao Xiang m->delta[0] = lo + 1;
18047e4937aSGao Xiang return 0;
18147e4937aSGao Xiang }
18247e4937aSGao Xiang m->clusterofs = lo;
18347e4937aSGao Xiang m->delta[0] = 0;
18447e4937aSGao Xiang /* figout out blkaddr (pblk) for HEAD lclusters */
185b86269f4SGao Xiang if (!big_pcluster) {
18647e4937aSGao Xiang nblk = 1;
18747e4937aSGao Xiang while (i > 0) {
18847e4937aSGao Xiang --i;
189*3f691aa6SGao Xiang lo = decode_compactedbits(lobits, in,
190*3f691aa6SGao Xiang encodebits * i, &type);
1911c7f49a7SGao Xiang if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD)
19247e4937aSGao Xiang i -= lo;
19347e4937aSGao Xiang
19447e4937aSGao Xiang if (i >= 0)
19547e4937aSGao Xiang ++nblk;
19647e4937aSGao Xiang }
197b86269f4SGao Xiang } else {
198b86269f4SGao Xiang nblk = 0;
199b86269f4SGao Xiang while (i > 0) {
200b86269f4SGao Xiang --i;
201*3f691aa6SGao Xiang lo = decode_compactedbits(lobits, in,
202*3f691aa6SGao Xiang encodebits * i, &type);
2031c7f49a7SGao Xiang if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
2041c7f49a7SGao Xiang if (lo & Z_EROFS_LI_D0_CBLKCNT) {
205b86269f4SGao Xiang --i;
2061c7f49a7SGao Xiang nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT;
207b86269f4SGao Xiang continue;
208b86269f4SGao Xiang }
209b86269f4SGao Xiang /* bigpcluster shouldn't have plain d0 == 1 */
210b86269f4SGao Xiang if (lo <= 1) {
211b86269f4SGao Xiang DBG_BUGON(1);
212b86269f4SGao Xiang return -EFSCORRUPTED;
213b86269f4SGao Xiang }
214b86269f4SGao Xiang i -= lo - 2;
215b86269f4SGao Xiang continue;
216b86269f4SGao Xiang }
217b86269f4SGao Xiang ++nblk;
218b86269f4SGao Xiang }
219b86269f4SGao Xiang }
22047e4937aSGao Xiang in += (vcnt << amortizedshift) - sizeof(__le32);
22147e4937aSGao Xiang m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
22247e4937aSGao Xiang return 0;
22347e4937aSGao Xiang }
22447e4937aSGao Xiang
z_erofs_load_compact_lcluster(struct z_erofs_maprecorder * m,unsigned long lcn,bool lookahead)2258241fdd3SGao Xiang static int z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m,
226d95ae5e2SGao Xiang unsigned long lcn, bool lookahead)
22747e4937aSGao Xiang {
22847e4937aSGao Xiang struct inode *const inode = m->inode;
229a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(inode);
230b780d3fcSGao Xiang const erofs_off_t ebase = sizeof(struct z_erofs_map_header) +
231b780d3fcSGao Xiang ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);
2323acea5fcSJingbo Xu unsigned int totalidx = erofs_iblks(inode);
23347e4937aSGao Xiang unsigned int compacted_4b_initial, compacted_2b;
23447e4937aSGao Xiang unsigned int amortizedshift;
23547e4937aSGao Xiang erofs_off_t pos;
23647e4937aSGao Xiang
23747e4937aSGao Xiang if (lcn >= totalidx)
23847e4937aSGao Xiang return -EINVAL;
23947e4937aSGao Xiang
24047e4937aSGao Xiang m->lcn = lcn;
24147e4937aSGao Xiang /* used to align to 32-byte (compacted_2b) alignment */
24247e4937aSGao Xiang compacted_4b_initial = (32 - ebase % 32) / 4;
24347e4937aSGao Xiang if (compacted_4b_initial == 32 / 4)
24447e4937aSGao Xiang compacted_4b_initial = 0;
24547e4937aSGao Xiang
246c40dd3caSYue Hu if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) &&
247c40dd3caSYue Hu compacted_4b_initial < totalidx)
24847e4937aSGao Xiang compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
24947e4937aSGao Xiang else
25047e4937aSGao Xiang compacted_2b = 0;
25147e4937aSGao Xiang
25247e4937aSGao Xiang pos = ebase;
25347e4937aSGao Xiang if (lcn < compacted_4b_initial) {
25447e4937aSGao Xiang amortizedshift = 2;
25547e4937aSGao Xiang goto out;
25647e4937aSGao Xiang }
25747e4937aSGao Xiang pos += compacted_4b_initial * 4;
25847e4937aSGao Xiang lcn -= compacted_4b_initial;
25947e4937aSGao Xiang
26047e4937aSGao Xiang if (lcn < compacted_2b) {
26147e4937aSGao Xiang amortizedshift = 1;
26247e4937aSGao Xiang goto out;
26347e4937aSGao Xiang }
26447e4937aSGao Xiang pos += compacted_2b * 2;
26547e4937aSGao Xiang lcn -= compacted_2b;
26647e4937aSGao Xiang amortizedshift = 2;
26747e4937aSGao Xiang out:
26847e4937aSGao Xiang pos += lcn * (1 << amortizedshift);
26931da107fSYue Hu m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
2703acea5fcSJingbo Xu erofs_blknr(inode->i_sb, pos), EROFS_KMAP);
27131da107fSYue Hu if (IS_ERR(m->kaddr))
27231da107fSYue Hu return PTR_ERR(m->kaddr);
273ab92184fSYue Hu return unpack_compacted_index(m, amortizedshift, pos, lookahead);
27447e4937aSGao Xiang }
27547e4937aSGao Xiang
z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder * m,unsigned int lcn,bool lookahead)2768241fdd3SGao Xiang static int z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m,
277d95ae5e2SGao Xiang unsigned int lcn, bool lookahead)
27847e4937aSGao Xiang {
2798241fdd3SGao Xiang switch (EROFS_I(m->inode)->datalayout) {
2808241fdd3SGao Xiang case EROFS_INODE_COMPRESSED_FULL:
2818241fdd3SGao Xiang return z_erofs_load_full_lcluster(m, lcn);
2828241fdd3SGao Xiang case EROFS_INODE_COMPRESSED_COMPACT:
2838241fdd3SGao Xiang return z_erofs_load_compact_lcluster(m, lcn, lookahead);
2848241fdd3SGao Xiang default:
28547e4937aSGao Xiang return -EINVAL;
28647e4937aSGao Xiang }
2878241fdd3SGao Xiang }
28847e4937aSGao Xiang
z_erofs_extent_lookback(struct z_erofs_maprecorder * m,unsigned int lookback_distance)2890c638f70SGao Xiang static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
29047e4937aSGao Xiang unsigned int lookback_distance)
29147e4937aSGao Xiang {
2928241fdd3SGao Xiang struct super_block *sb = m->inode->i_sb;
293a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(m->inode);
29447e4937aSGao Xiang const unsigned int lclusterbits = vi->z_logical_clusterbits;
295ab474fccSGao Xiang
296ab474fccSGao Xiang while (m->lcn >= lookback_distance) {
297ab474fccSGao Xiang unsigned long lcn = m->lcn - lookback_distance;
29847e4937aSGao Xiang int err;
29947e4937aSGao Xiang
3008241fdd3SGao Xiang err = z_erofs_load_lcluster_from_disk(m, lcn, false);
30147e4937aSGao Xiang if (err)
30247e4937aSGao Xiang return err;
30347e4937aSGao Xiang
30447e4937aSGao Xiang switch (m->type) {
3051c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_NONHEAD:
306ab474fccSGao Xiang lookback_distance = m->delta[0];
3078241fdd3SGao Xiang if (!lookback_distance)
3088241fdd3SGao Xiang goto err_bogus;
309ab474fccSGao Xiang continue;
3101c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_PLAIN:
3111c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_HEAD1:
3121c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_HEAD2:
3138f899262SGao Xiang m->headtype = m->type;
314ab474fccSGao Xiang m->map->m_la = (lcn << lclusterbits) | m->clusterofs;
315ab474fccSGao Xiang return 0;
31647e4937aSGao Xiang default:
3178241fdd3SGao Xiang erofs_err(sb, "unknown type %u @ lcn %lu of nid %llu",
31847e4937aSGao Xiang m->type, lcn, vi->nid);
31947e4937aSGao Xiang DBG_BUGON(1);
32047e4937aSGao Xiang return -EOPNOTSUPP;
32147e4937aSGao Xiang }
322ab474fccSGao Xiang }
3238241fdd3SGao Xiang err_bogus:
3248241fdd3SGao Xiang erofs_err(sb, "bogus lookback distance %u @ lcn %lu of nid %llu",
3258241fdd3SGao Xiang lookback_distance, m->lcn, vi->nid);
326ab474fccSGao Xiang DBG_BUGON(1);
327ab474fccSGao Xiang return -EFSCORRUPTED;
32847e4937aSGao Xiang }
32947e4937aSGao Xiang
z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder * m,unsigned int initial_lcn)330cec6e93bSGao Xiang static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
331cec6e93bSGao Xiang unsigned int initial_lcn)
332cec6e93bSGao Xiang {
3333acea5fcSJingbo Xu struct super_block *sb = m->inode->i_sb;
334cec6e93bSGao Xiang struct erofs_inode *const vi = EROFS_I(m->inode);
335cec6e93bSGao Xiang struct erofs_map_blocks *const map = m->map;
336cec6e93bSGao Xiang const unsigned int lclusterbits = vi->z_logical_clusterbits;
337cec6e93bSGao Xiang unsigned long lcn;
338cec6e93bSGao Xiang int err;
339cec6e93bSGao Xiang
3401c7f49a7SGao Xiang DBG_BUGON(m->type != Z_EROFS_LCLUSTER_TYPE_PLAIN &&
3411c7f49a7SGao Xiang m->type != Z_EROFS_LCLUSTER_TYPE_HEAD1 &&
3421c7f49a7SGao Xiang m->type != Z_EROFS_LCLUSTER_TYPE_HEAD2);
34372bb5262SGao Xiang DBG_BUGON(m->type != m->headtype);
34472bb5262SGao Xiang
3451c7f49a7SGao Xiang if (m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN ||
3461c7f49a7SGao Xiang ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1) &&
34772bb5262SGao Xiang !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) ||
3481c7f49a7SGao Xiang ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) &&
34972bb5262SGao Xiang !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
350d467e980SGao Xiang map->m_plen = 1ULL << lclusterbits;
351cec6e93bSGao Xiang return 0;
352cec6e93bSGao Xiang }
353cec6e93bSGao Xiang lcn = m->lcn + 1;
354ea0b7b0dSYue Hu if (m->compressedblks)
355cec6e93bSGao Xiang goto out;
356cec6e93bSGao Xiang
3578241fdd3SGao Xiang err = z_erofs_load_lcluster_from_disk(m, lcn, false);
358cec6e93bSGao Xiang if (err)
359cec6e93bSGao Xiang return err;
360cec6e93bSGao Xiang
3610852b6caSGao Xiang /*
3620852b6caSGao Xiang * If the 1st NONHEAD lcluster has already been handled initially w/o
363ea0b7b0dSYue Hu * valid compressedblks, which means at least it mustn't be CBLKCNT, or
3640852b6caSGao Xiang * an internal implemenatation error is detected.
3650852b6caSGao Xiang *
3660852b6caSGao Xiang * The following code can also handle it properly anyway, but let's
3670852b6caSGao Xiang * BUG_ON in the debugging mode only for developers to notice that.
3680852b6caSGao Xiang */
3690852b6caSGao Xiang DBG_BUGON(lcn == initial_lcn &&
3701c7f49a7SGao Xiang m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD);
3710852b6caSGao Xiang
372cec6e93bSGao Xiang switch (m->type) {
3731c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_PLAIN:
3741c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_HEAD1:
3751c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_HEAD2:
3760852b6caSGao Xiang /*
3770852b6caSGao Xiang * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type
3780852b6caSGao Xiang * rather than CBLKCNT, it's a 1 lcluster-sized pcluster.
3790852b6caSGao Xiang */
3803acea5fcSJingbo Xu m->compressedblks = 1 << (lclusterbits - sb->s_blocksize_bits);
3810852b6caSGao Xiang break;
3821c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_NONHEAD:
383cec6e93bSGao Xiang if (m->delta[0] != 1)
384cec6e93bSGao Xiang goto err_bonus_cblkcnt;
385ea0b7b0dSYue Hu if (m->compressedblks)
386cec6e93bSGao Xiang break;
387cec6e93bSGao Xiang fallthrough;
388cec6e93bSGao Xiang default:
3898241fdd3SGao Xiang erofs_err(sb, "cannot found CBLKCNT @ lcn %lu of nid %llu", lcn,
3908241fdd3SGao Xiang vi->nid);
391cec6e93bSGao Xiang DBG_BUGON(1);
392cec6e93bSGao Xiang return -EFSCORRUPTED;
393cec6e93bSGao Xiang }
394cec6e93bSGao Xiang out:
3953acea5fcSJingbo Xu map->m_plen = erofs_pos(sb, m->compressedblks);
396cec6e93bSGao Xiang return 0;
397cec6e93bSGao Xiang err_bonus_cblkcnt:
3988241fdd3SGao Xiang erofs_err(sb, "bogus CBLKCNT @ lcn %lu of nid %llu", lcn, vi->nid);
399cec6e93bSGao Xiang DBG_BUGON(1);
400cec6e93bSGao Xiang return -EFSCORRUPTED;
401cec6e93bSGao Xiang }
402cec6e93bSGao Xiang
z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder * m)403d95ae5e2SGao Xiang static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m)
404d95ae5e2SGao Xiang {
405d95ae5e2SGao Xiang struct inode *inode = m->inode;
406d95ae5e2SGao Xiang struct erofs_inode *vi = EROFS_I(inode);
407d95ae5e2SGao Xiang struct erofs_map_blocks *map = m->map;
408d95ae5e2SGao Xiang unsigned int lclusterbits = vi->z_logical_clusterbits;
409d95ae5e2SGao Xiang u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits;
410d95ae5e2SGao Xiang int err;
411d95ae5e2SGao Xiang
412d95ae5e2SGao Xiang do {
413d95ae5e2SGao Xiang /* handle the last EOF pcluster (no next HEAD lcluster) */
414d95ae5e2SGao Xiang if ((lcn << lclusterbits) >= inode->i_size) {
415d95ae5e2SGao Xiang map->m_llen = inode->i_size - map->m_la;
416d95ae5e2SGao Xiang return 0;
417d95ae5e2SGao Xiang }
418d95ae5e2SGao Xiang
4198241fdd3SGao Xiang err = z_erofs_load_lcluster_from_disk(m, lcn, true);
420d95ae5e2SGao Xiang if (err)
421d95ae5e2SGao Xiang return err;
422d95ae5e2SGao Xiang
4231c7f49a7SGao Xiang if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) {
424d95ae5e2SGao Xiang DBG_BUGON(!m->delta[1] &&
425d95ae5e2SGao Xiang m->clusterofs != 1 << lclusterbits);
4261c7f49a7SGao Xiang } else if (m->type == Z_EROFS_LCLUSTER_TYPE_PLAIN ||
4271c7f49a7SGao Xiang m->type == Z_EROFS_LCLUSTER_TYPE_HEAD1 ||
4281c7f49a7SGao Xiang m->type == Z_EROFS_LCLUSTER_TYPE_HEAD2) {
429d95ae5e2SGao Xiang /* go on until the next HEAD lcluster */
430d95ae5e2SGao Xiang if (lcn != headlcn)
431d95ae5e2SGao Xiang break;
432d95ae5e2SGao Xiang m->delta[1] = 1;
433d95ae5e2SGao Xiang } else {
434d95ae5e2SGao Xiang erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu",
435d95ae5e2SGao Xiang m->type, lcn, vi->nid);
436d95ae5e2SGao Xiang DBG_BUGON(1);
437d95ae5e2SGao Xiang return -EOPNOTSUPP;
438d95ae5e2SGao Xiang }
439d95ae5e2SGao Xiang lcn += m->delta[1];
440d95ae5e2SGao Xiang } while (m->delta[1]);
441d95ae5e2SGao Xiang
442d95ae5e2SGao Xiang map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la;
443d95ae5e2SGao Xiang return 0;
444d95ae5e2SGao Xiang }
445d95ae5e2SGao Xiang
z_erofs_do_map_blocks(struct inode * inode,struct erofs_map_blocks * map,int flags)446ab92184fSYue Hu static int z_erofs_do_map_blocks(struct inode *inode,
4471c7f49a7SGao Xiang struct erofs_map_blocks *map, int flags)
44847e4937aSGao Xiang {
449a5876e24SGao Xiang struct erofs_inode *const vi = EROFS_I(inode);
450ab92184fSYue Hu bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER;
451b15b2e30SYue Hu bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
45247e4937aSGao Xiang struct z_erofs_maprecorder m = {
45347e4937aSGao Xiang .inode = inode,
45447e4937aSGao Xiang .map = map,
45547e4937aSGao Xiang };
45647e4937aSGao Xiang int err = 0;
457823ba1d2SGao Xiang unsigned int lclusterbits, endoff, afmt;
458cec6e93bSGao Xiang unsigned long initial_lcn;
45947e4937aSGao Xiang unsigned long long ofs, end;
46047e4937aSGao Xiang
46147e4937aSGao Xiang lclusterbits = vi->z_logical_clusterbits;
462ab92184fSYue Hu ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la;
463cec6e93bSGao Xiang initial_lcn = ofs >> lclusterbits;
46447e4937aSGao Xiang endoff = ofs & ((1 << lclusterbits) - 1);
46547e4937aSGao Xiang
4668241fdd3SGao Xiang err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false);
46747e4937aSGao Xiang if (err)
46847e4937aSGao Xiang goto unmap_out;
46947e4937aSGao Xiang
470ab92184fSYue Hu if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL))
471ab92184fSYue Hu vi->z_idataoff = m.nextpackoff;
472ab92184fSYue Hu
4738f899262SGao Xiang map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
47447e4937aSGao Xiang end = (m.lcn + 1ULL) << lclusterbits;
47547e4937aSGao Xiang
47647e4937aSGao Xiang switch (m.type) {
4771c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_PLAIN:
4781c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_HEAD1:
4791c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_HEAD2:
48047e4937aSGao Xiang if (endoff >= m.clusterofs) {
4818f899262SGao Xiang m.headtype = m.type;
48247e4937aSGao Xiang map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
48324331050SGao Xiang /*
48424331050SGao Xiang * For ztailpacking files, in order to inline data more
48524331050SGao Xiang * effectively, special EOF lclusters are now supported
48624331050SGao Xiang * which can have three parts at most.
48724331050SGao Xiang */
48824331050SGao Xiang if (ztailpacking && end > inode->i_size)
48924331050SGao Xiang end = inode->i_size;
49047e4937aSGao Xiang break;
49147e4937aSGao Xiang }
49247e4937aSGao Xiang /* m.lcn should be >= 1 if endoff < m.clusterofs */
4938d8a09b0SGao Xiang if (!m.lcn) {
4944f761fa2SGao Xiang erofs_err(inode->i_sb,
4954f761fa2SGao Xiang "invalid logical cluster 0 at nid %llu",
49647e4937aSGao Xiang vi->nid);
49747e4937aSGao Xiang err = -EFSCORRUPTED;
49847e4937aSGao Xiang goto unmap_out;
49947e4937aSGao Xiang }
50047e4937aSGao Xiang end = (m.lcn << lclusterbits) | m.clusterofs;
50147e4937aSGao Xiang map->m_flags |= EROFS_MAP_FULL_MAPPED;
50247e4937aSGao Xiang m.delta[0] = 1;
503df561f66SGustavo A. R. Silva fallthrough;
5041c7f49a7SGao Xiang case Z_EROFS_LCLUSTER_TYPE_NONHEAD:
505fe6adcceSRuiqi Gong /* get the corresponding first chunk */
5060c638f70SGao Xiang err = z_erofs_extent_lookback(&m, m.delta[0]);
5078d8a09b0SGao Xiang if (err)
50847e4937aSGao Xiang goto unmap_out;
50947e4937aSGao Xiang break;
51047e4937aSGao Xiang default:
5114f761fa2SGao Xiang erofs_err(inode->i_sb,
5124f761fa2SGao Xiang "unknown type %u @ offset %llu of nid %llu",
51347e4937aSGao Xiang m.type, ofs, vi->nid);
51447e4937aSGao Xiang err = -EOPNOTSUPP;
51547e4937aSGao Xiang goto unmap_out;
51647e4937aSGao Xiang }
5175c2a6425SGao Xiang if (m.partialref)
5185c2a6425SGao Xiang map->m_flags |= EROFS_MAP_PARTIAL_REF;
51947e4937aSGao Xiang map->m_llen = end - map->m_la;
52047e4937aSGao Xiang
521b15b2e30SYue Hu if (flags & EROFS_GET_BLOCKS_FINDTAIL) {
522ab92184fSYue Hu vi->z_tailextent_headlcn = m.lcn;
523b15b2e30SYue Hu /* for non-compact indexes, fragmentoff is 64 bits */
5248241fdd3SGao Xiang if (fragment && vi->datalayout == EROFS_INODE_COMPRESSED_FULL)
525b15b2e30SYue Hu vi->z_fragmentoff |= (u64)m.pblk << 32;
526b15b2e30SYue Hu }
527ab92184fSYue Hu if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {
528ab92184fSYue Hu map->m_flags |= EROFS_MAP_META;
529ab92184fSYue Hu map->m_pa = vi->z_idataoff;
530ab92184fSYue Hu map->m_plen = vi->z_idata_size;
531b15b2e30SYue Hu } else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
532b15b2e30SYue Hu map->m_flags |= EROFS_MAP_FRAGMENT;
533ab92184fSYue Hu } else {
5343acea5fcSJingbo Xu map->m_pa = erofs_pos(inode->i_sb, m.pblk);
535cec6e93bSGao Xiang err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
536cec6e93bSGao Xiang if (err)
537d5d188b8SGao Xiang goto unmap_out;
538ab92184fSYue Hu }
539d95ae5e2SGao Xiang
5401c7f49a7SGao Xiang if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) {
541c505febaSGao Xiang if (map->m_llen > map->m_plen) {
542c505febaSGao Xiang DBG_BUGON(1);
543c505febaSGao Xiang err = -EFSCORRUPTED;
544c505febaSGao Xiang goto unmap_out;
545c505febaSGao Xiang }
546823ba1d2SGao Xiang afmt = vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER ?
547823ba1d2SGao Xiang Z_EROFS_COMPRESSION_INTERLACED :
548fdffc091SYue Hu Z_EROFS_COMPRESSION_SHIFTED;
549fdffc091SYue Hu } else {
550823ba1d2SGao Xiang afmt = m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2 ?
551823ba1d2SGao Xiang vi->z_algorithmtype[1] : vi->z_algorithmtype[0];
552823ba1d2SGao Xiang if (!(EROFS_I_SB(inode)->available_compr_algs & (1 << afmt))) {
553823ba1d2SGao Xiang erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu",
554823ba1d2SGao Xiang afmt, vi->nid);
555823ba1d2SGao Xiang err = -EFSCORRUPTED;
556823ba1d2SGao Xiang goto unmap_out;
557fdffc091SYue Hu }
558823ba1d2SGao Xiang }
559823ba1d2SGao Xiang map->m_algorithmformat = afmt;
5608f899262SGao Xiang
561622ceaddSGao Xiang if ((flags & EROFS_GET_BLOCKS_FIEMAP) ||
562622ceaddSGao Xiang ((flags & EROFS_GET_BLOCKS_READMORE) &&
563ffa09b3bSGao Xiang (map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA ||
564ffa09b3bSGao Xiang map->m_algorithmformat == Z_EROFS_COMPRESSION_DEFLATE) &&
5653acea5fcSJingbo Xu map->m_llen >= i_blocksize(inode))) {
566d95ae5e2SGao Xiang err = z_erofs_get_extent_decompressedlen(&m);
567d95ae5e2SGao Xiang if (!err)
568d95ae5e2SGao Xiang map->m_flags |= EROFS_MAP_FULL_MAPPED;
569d95ae5e2SGao Xiang }
570d5d188b8SGao Xiang
57147e4937aSGao Xiang unmap_out:
57209c54379SGao Xiang erofs_unmap_metabuf(&m.map->buf);
573ab92184fSYue Hu return err;
574ab92184fSYue Hu }
575ab92184fSYue Hu
z_erofs_fill_inode_lazy(struct inode * inode)576999f2f9aSGao Xiang static int z_erofs_fill_inode_lazy(struct inode *inode)
577999f2f9aSGao Xiang {
578999f2f9aSGao Xiang struct erofs_inode *const vi = EROFS_I(inode);
579999f2f9aSGao Xiang struct super_block *const sb = inode->i_sb;
580999f2f9aSGao Xiang int err, headnr;
581999f2f9aSGao Xiang erofs_off_t pos;
582999f2f9aSGao Xiang struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
583999f2f9aSGao Xiang void *kaddr;
584999f2f9aSGao Xiang struct z_erofs_map_header *h;
585999f2f9aSGao Xiang
586999f2f9aSGao Xiang if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) {
587999f2f9aSGao Xiang /*
588999f2f9aSGao Xiang * paired with smp_mb() at the end of the function to ensure
589999f2f9aSGao Xiang * fields will only be observed after the bit is set.
590999f2f9aSGao Xiang */
591999f2f9aSGao Xiang smp_mb();
592999f2f9aSGao Xiang return 0;
593999f2f9aSGao Xiang }
594999f2f9aSGao Xiang
595999f2f9aSGao Xiang if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
596999f2f9aSGao Xiang return -ERESTARTSYS;
597999f2f9aSGao Xiang
598999f2f9aSGao Xiang err = 0;
599999f2f9aSGao Xiang if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
600999f2f9aSGao Xiang goto out_unlock;
601999f2f9aSGao Xiang
602999f2f9aSGao Xiang pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);
6033acea5fcSJingbo Xu kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(sb, pos), EROFS_KMAP);
604999f2f9aSGao Xiang if (IS_ERR(kaddr)) {
605999f2f9aSGao Xiang err = PTR_ERR(kaddr);
606999f2f9aSGao Xiang goto out_unlock;
607999f2f9aSGao Xiang }
608999f2f9aSGao Xiang
6093acea5fcSJingbo Xu h = kaddr + erofs_blkoff(sb, pos);
610999f2f9aSGao Xiang /*
611999f2f9aSGao Xiang * if the highest bit of the 8-byte map header is set, the whole file
612999f2f9aSGao Xiang * is stored in the packed inode. The rest bits keeps z_fragmentoff.
613999f2f9aSGao Xiang */
614999f2f9aSGao Xiang if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) {
615999f2f9aSGao Xiang vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
616999f2f9aSGao Xiang vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63);
617999f2f9aSGao Xiang vi->z_tailextent_headlcn = 0;
618999f2f9aSGao Xiang goto done;
619999f2f9aSGao Xiang }
620999f2f9aSGao Xiang vi->z_advise = le16_to_cpu(h->h_advise);
621999f2f9aSGao Xiang vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
622999f2f9aSGao Xiang vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
623999f2f9aSGao Xiang
624999f2f9aSGao Xiang headnr = 0;
625999f2f9aSGao Xiang if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX ||
626999f2f9aSGao Xiang vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) {
627999f2f9aSGao Xiang erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel",
628999f2f9aSGao Xiang headnr + 1, vi->z_algorithmtype[headnr], vi->nid);
629999f2f9aSGao Xiang err = -EOPNOTSUPP;
630999f2f9aSGao Xiang goto out_put_metabuf;
631999f2f9aSGao Xiang }
632999f2f9aSGao Xiang
6333acea5fcSJingbo Xu vi->z_logical_clusterbits = sb->s_blocksize_bits + (h->h_clusterbits & 7);
634999f2f9aSGao Xiang if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
635999f2f9aSGao Xiang vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
636999f2f9aSGao Xiang Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
637999f2f9aSGao Xiang erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu",
638999f2f9aSGao Xiang vi->nid);
639999f2f9aSGao Xiang err = -EFSCORRUPTED;
640999f2f9aSGao Xiang goto out_put_metabuf;
641999f2f9aSGao Xiang }
6421c7f49a7SGao Xiang if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT &&
643999f2f9aSGao Xiang !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
644999f2f9aSGao Xiang !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
645999f2f9aSGao Xiang erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
646999f2f9aSGao Xiang vi->nid);
647999f2f9aSGao Xiang err = -EFSCORRUPTED;
648999f2f9aSGao Xiang goto out_put_metabuf;
649999f2f9aSGao Xiang }
650999f2f9aSGao Xiang
651999f2f9aSGao Xiang if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) {
652999f2f9aSGao Xiang struct erofs_map_blocks map = {
653999f2f9aSGao Xiang .buf = __EROFS_BUF_INITIALIZER
654999f2f9aSGao Xiang };
655999f2f9aSGao Xiang
656999f2f9aSGao Xiang vi->z_idata_size = le16_to_cpu(h->h_idata_size);
657999f2f9aSGao Xiang err = z_erofs_do_map_blocks(inode, &map,
658999f2f9aSGao Xiang EROFS_GET_BLOCKS_FINDTAIL);
659999f2f9aSGao Xiang erofs_put_metabuf(&map.buf);
660999f2f9aSGao Xiang
661999f2f9aSGao Xiang if (!map.m_plen ||
6623acea5fcSJingbo Xu erofs_blkoff(sb, map.m_pa) + map.m_plen > sb->s_blocksize) {
663999f2f9aSGao Xiang erofs_err(sb, "invalid tail-packing pclustersize %llu",
664999f2f9aSGao Xiang map.m_plen);
665999f2f9aSGao Xiang err = -EFSCORRUPTED;
666999f2f9aSGao Xiang }
667999f2f9aSGao Xiang if (err < 0)
668999f2f9aSGao Xiang goto out_put_metabuf;
669999f2f9aSGao Xiang }
670999f2f9aSGao Xiang
671999f2f9aSGao Xiang if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER &&
672999f2f9aSGao Xiang !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) {
673999f2f9aSGao Xiang struct erofs_map_blocks map = {
674999f2f9aSGao Xiang .buf = __EROFS_BUF_INITIALIZER
675999f2f9aSGao Xiang };
676999f2f9aSGao Xiang
677999f2f9aSGao Xiang vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff);
678999f2f9aSGao Xiang err = z_erofs_do_map_blocks(inode, &map,
679999f2f9aSGao Xiang EROFS_GET_BLOCKS_FINDTAIL);
680999f2f9aSGao Xiang erofs_put_metabuf(&map.buf);
681999f2f9aSGao Xiang if (err < 0)
682999f2f9aSGao Xiang goto out_put_metabuf;
683999f2f9aSGao Xiang }
684999f2f9aSGao Xiang done:
685999f2f9aSGao Xiang /* paired with smp_mb() at the beginning of the function */
686999f2f9aSGao Xiang smp_mb();
687999f2f9aSGao Xiang set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
688999f2f9aSGao Xiang out_put_metabuf:
689999f2f9aSGao Xiang erofs_put_metabuf(&buf);
690999f2f9aSGao Xiang out_unlock:
691999f2f9aSGao Xiang clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
692999f2f9aSGao Xiang return err;
693999f2f9aSGao Xiang }
694999f2f9aSGao Xiang
z_erofs_map_blocks_iter(struct inode * inode,struct erofs_map_blocks * map,int flags)69553a7f996SGao Xiang int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,
696ab92184fSYue Hu int flags)
697ab92184fSYue Hu {
698b15b2e30SYue Hu struct erofs_inode *const vi = EROFS_I(inode);
699ab92184fSYue Hu int err = 0;
700ab92184fSYue Hu
701ab92184fSYue Hu trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
702ab92184fSYue Hu
703ab92184fSYue Hu /* when trying to read beyond EOF, leave it unmapped */
704ab92184fSYue Hu if (map->m_la >= inode->i_size) {
705ab92184fSYue Hu map->m_llen = map->m_la + 1 - inode->i_size;
706ab92184fSYue Hu map->m_la = inode->i_size;
707ab92184fSYue Hu map->m_flags = 0;
708ab92184fSYue Hu goto out;
709ab92184fSYue Hu }
710ab92184fSYue Hu
711ab92184fSYue Hu err = z_erofs_fill_inode_lazy(inode);
712ab92184fSYue Hu if (err)
713ab92184fSYue Hu goto out;
714ab92184fSYue Hu
715b15b2e30SYue Hu if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) &&
716b15b2e30SYue Hu !vi->z_tailextent_headlcn) {
717b15b2e30SYue Hu map->m_la = 0;
718b15b2e30SYue Hu map->m_llen = inode->i_size;
719b15b2e30SYue Hu map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED |
720b15b2e30SYue Hu EROFS_MAP_FRAGMENT;
721b15b2e30SYue Hu goto out;
722b15b2e30SYue Hu }
723b15b2e30SYue Hu
724ab92184fSYue Hu err = z_erofs_do_map_blocks(inode, map, flags);
725ab92184fSYue Hu out:
72647e4937aSGao Xiang trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
72747e4937aSGao Xiang return err;
72847e4937aSGao Xiang }
729eadcd6b5SGao Xiang
z_erofs_iomap_begin_report(struct inode * inode,loff_t offset,loff_t length,unsigned int flags,struct iomap * iomap,struct iomap * srcmap)730eadcd6b5SGao Xiang static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,
731eadcd6b5SGao Xiang loff_t length, unsigned int flags,
732eadcd6b5SGao Xiang struct iomap *iomap, struct iomap *srcmap)
733eadcd6b5SGao Xiang {
734eadcd6b5SGao Xiang int ret;
735eadcd6b5SGao Xiang struct erofs_map_blocks map = { .m_la = offset };
736eadcd6b5SGao Xiang
737eadcd6b5SGao Xiang ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP);
73809c54379SGao Xiang erofs_put_metabuf(&map.buf);
739eadcd6b5SGao Xiang if (ret < 0)
740eadcd6b5SGao Xiang return ret;
741eadcd6b5SGao Xiang
742eadcd6b5SGao Xiang iomap->bdev = inode->i_sb->s_bdev;
743eadcd6b5SGao Xiang iomap->offset = map.m_la;
744eadcd6b5SGao Xiang iomap->length = map.m_llen;
745eadcd6b5SGao Xiang if (map.m_flags & EROFS_MAP_MAPPED) {
746eadcd6b5SGao Xiang iomap->type = IOMAP_MAPPED;
747b15b2e30SYue Hu iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ?
748b15b2e30SYue Hu IOMAP_NULL_ADDR : map.m_pa;
749eadcd6b5SGao Xiang } else {
750eadcd6b5SGao Xiang iomap->type = IOMAP_HOLE;
751eadcd6b5SGao Xiang iomap->addr = IOMAP_NULL_ADDR;
752eadcd6b5SGao Xiang /*
7536acd87d5SSiddh Raman Pant * No strict rule on how to describe extents for post EOF, yet
7546acd87d5SSiddh Raman Pant * we need to do like below. Otherwise, iomap itself will get
755eadcd6b5SGao Xiang * into an endless loop on post EOF.
7566acd87d5SSiddh Raman Pant *
7576acd87d5SSiddh Raman Pant * Calculate the effective offset by subtracting extent start
7586acd87d5SSiddh Raman Pant * (map.m_la) from the requested offset, and add it to length.
7596acd87d5SSiddh Raman Pant * (NB: offset >= map.m_la always)
760eadcd6b5SGao Xiang */
761eadcd6b5SGao Xiang if (iomap->offset >= inode->i_size)
7626acd87d5SSiddh Raman Pant iomap->length = length + offset - map.m_la;
763eadcd6b5SGao Xiang }
764eadcd6b5SGao Xiang iomap->flags = 0;
765eadcd6b5SGao Xiang return 0;
766eadcd6b5SGao Xiang }
767eadcd6b5SGao Xiang
768eadcd6b5SGao Xiang const struct iomap_ops z_erofs_iomap_report_ops = {
769eadcd6b5SGao Xiang .iomap_begin = z_erofs_iomap_begin_report,
770eadcd6b5SGao Xiang };
771