zmap.c (1758047057dbe329be712a31b79db7151b5871f8) zmap.c (ab92184ff8f12979f3d3dd5ed601ed85770d81ba)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * https://www.huawei.com/
5 */
6#include "internal.h"
7#include <asm/unaligned.h>
8#include <trace/events/erofs.h>
9
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * https://www.huawei.com/
5 */
6#include "internal.h"
7#include <asm/unaligned.h>
8#include <trace/events/erofs.h>
9
10static int z_erofs_do_map_blocks(struct inode *inode,
11 struct erofs_map_blocks *map,
12 int flags);
13
10int z_erofs_fill_inode(struct inode *inode)
11{
12 struct erofs_inode *const vi = EROFS_I(inode);
13 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
14
15 if (!erofs_sb_has_big_pcluster(sbi) &&
14int z_erofs_fill_inode(struct inode *inode)
15{
16 struct erofs_inode *const vi = EROFS_I(inode);
17 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
18
19 if (!erofs_sb_has_big_pcluster(sbi) &&
20 !erofs_sb_has_ztailpacking(sbi) &&
16 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
17 vi->z_advise = 0;
18 vi->z_algorithmtype[0] = 0;
19 vi->z_algorithmtype[1] = 0;
20 vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
21 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
22 }
23 inode->i_mapping->a_ops = &z_erofs_aops;

--- 22 unchanged lines hidden (view full) ---

46 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
47 return -ERESTARTSYS;
48
49 err = 0;
50 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
51 goto out_unlock;
52
53 DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
21 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
22 vi->z_advise = 0;
23 vi->z_algorithmtype[0] = 0;
24 vi->z_algorithmtype[1] = 0;
25 vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
26 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
27 }
28 inode->i_mapping->a_ops = &z_erofs_aops;

--- 22 unchanged lines hidden (view full) ---

51 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
52 return -ERESTARTSYS;
53
54 err = 0;
55 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
56 goto out_unlock;
57
58 DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
59 !erofs_sb_has_ztailpacking(EROFS_SB(sb)) &&
54 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
55
56 pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
57 vi->xattr_isize, 8);
58 page = erofs_get_meta_page(sb, erofs_blknr(pos));
59 if (IS_ERR(page)) {
60 err = PTR_ERR(page);
61 goto out_unlock;

--- 27 unchanged lines hidden (view full) ---

89 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION &&
90 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
91 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
92 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
93 vi->nid);
94 err = -EFSCORRUPTED;
95 goto unmap_done;
96 }
60 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
61
62 pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
63 vi->xattr_isize, 8);
64 page = erofs_get_meta_page(sb, erofs_blknr(pos));
65 if (IS_ERR(page)) {
66 err = PTR_ERR(page);
67 goto out_unlock;

--- 27 unchanged lines hidden (view full) ---

95 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION &&
96 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
97 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
98 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
99 vi->nid);
100 err = -EFSCORRUPTED;
101 goto unmap_done;
102 }
97 /* paired with smp_mb() at the beginning of the function */
98 smp_mb();
99 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
100unmap_done:
101 kunmap_atomic(kaddr);
102 unlock_page(page);
103 put_page(page);
103unmap_done:
104 kunmap_atomic(kaddr);
105 unlock_page(page);
106 put_page(page);
107 if (err)
108 goto out_unlock;
109
110 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) {
111 struct erofs_map_blocks map = { .mpage = NULL };
112
113 vi->z_idata_size = le16_to_cpu(h->h_idata_size);
114 err = z_erofs_do_map_blocks(inode, &map,
115 EROFS_GET_BLOCKS_FINDTAIL);
116 if (map.mpage)
117 put_page(map.mpage);
118
119 if (!map.m_plen ||
120 erofs_blkoff(map.m_pa) + map.m_plen > EROFS_BLKSIZ) {
121 erofs_err(sb, "invalid tail-packing pclustersize %llu",
122 map.m_plen);
123 err = -EFSCORRUPTED;
124 }
125 if (err < 0)
126 goto out_unlock;
127 }
128 /* paired with smp_mb() at the beginning of the function */
129 smp_mb();
130 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
104out_unlock:
105 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
106 return err;
107}
108
109struct z_erofs_maprecorder {
110 struct inode *inode;
111 struct erofs_map_blocks *map;
112 void *kaddr;
113
114 unsigned long lcn;
115 /* compression extent information gathered */
116 u8 type, headtype;
117 u16 clusterofs;
118 u16 delta[2];
119 erofs_blk_t pblk, compressedlcs;
131out_unlock:
132 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
133 return err;
134}
135
136struct z_erofs_maprecorder {
137 struct inode *inode;
138 struct erofs_map_blocks *map;
139 void *kaddr;
140
141 unsigned long lcn;
142 /* compression extent information gathered */
143 u8 type, headtype;
144 u16 clusterofs;
145 u16 delta[2];
146 erofs_blk_t pblk, compressedlcs;
147 erofs_off_t nextpackoff;
120};
121
122static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
123 erofs_blk_t eblk)
124{
125 struct super_block *const sb = m->inode->i_sb;
126 struct erofs_map_blocks *const map = m->map;
127 struct page *mpage = map->mpage;

--- 36 unchanged lines hidden (view full) ---

164 struct z_erofs_vle_decompressed_index *di;
165 unsigned int advise, type;
166 int err;
167
168 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
169 if (err)
170 return err;
171
148};
149
150static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
151 erofs_blk_t eblk)
152{
153 struct super_block *const sb = m->inode->i_sb;
154 struct erofs_map_blocks *const map = m->map;
155 struct page *mpage = map->mpage;

--- 36 unchanged lines hidden (view full) ---

192 struct z_erofs_vle_decompressed_index *di;
193 unsigned int advise, type;
194 int err;
195
196 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
197 if (err)
198 return err;
199
200 m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index);
172 m->lcn = lcn;
173 di = m->kaddr + erofs_blkoff(pos);
174
175 advise = le16_to_cpu(di->di_advise);
176 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
177 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
178 switch (type) {
179 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:

--- 58 unchanged lines hidden (view full) ---

238 /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */
239 if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT))
240 d1 += lo - 1;
241 return d1;
242}
243
244static int unpack_compacted_index(struct z_erofs_maprecorder *m,
245 unsigned int amortizedshift,
201 m->lcn = lcn;
202 di = m->kaddr + erofs_blkoff(pos);
203
204 advise = le16_to_cpu(di->di_advise);
205 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
206 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
207 switch (type) {
208 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:

--- 58 unchanged lines hidden (view full) ---

267 /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */
268 if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT))
269 d1 += lo - 1;
270 return d1;
271}
272
273static int unpack_compacted_index(struct z_erofs_maprecorder *m,
274 unsigned int amortizedshift,
246 unsigned int eofs, bool lookahead)
275 erofs_off_t pos, bool lookahead)
247{
248 struct erofs_inode *const vi = EROFS_I(m->inode);
249 const unsigned int lclusterbits = vi->z_logical_clusterbits;
250 const unsigned int lomask = (1 << lclusterbits) - 1;
276{
277 struct erofs_inode *const vi = EROFS_I(m->inode);
278 const unsigned int lclusterbits = vi->z_logical_clusterbits;
279 const unsigned int lomask = (1 << lclusterbits) - 1;
251 unsigned int vcnt, base, lo, encodebits, nblk;
280 unsigned int vcnt, base, lo, encodebits, nblk, eofs;
252 int i;
253 u8 *in, type;
254 bool big_pcluster;
255
256 if (1 << amortizedshift == 4)
257 vcnt = 2;
258 else if (1 << amortizedshift == 2 && lclusterbits == 12)
259 vcnt = 16;
260 else
261 return -EOPNOTSUPP;
262
281 int i;
282 u8 *in, type;
283 bool big_pcluster;
284
285 if (1 << amortizedshift == 4)
286 vcnt = 2;
287 else if (1 << amortizedshift == 2 && lclusterbits == 12)
288 vcnt = 16;
289 else
290 return -EOPNOTSUPP;
291
292 /* it doesn't equal to round_up(..) */
293 m->nextpackoff = round_down(pos, vcnt << amortizedshift) +
294 (vcnt << amortizedshift);
263 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
264 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
295 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
296 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
297 eofs = erofs_blkoff(pos);
265 base = round_down(eofs, vcnt << amortizedshift);
266 in = m->kaddr + base;
267
268 i = (eofs - base) >> amortizedshift;
269
270 lo = decode_compactedbits(lclusterbits, lomask,
271 in, encodebits * i, &type);
272 m->type = type;

--- 121 unchanged lines hidden (view full) ---

394 pos += compacted_2b * 2;
395 lcn -= compacted_2b;
396 amortizedshift = 2;
397out:
398 pos += lcn * (1 << amortizedshift);
399 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
400 if (err)
401 return err;
298 base = round_down(eofs, vcnt << amortizedshift);
299 in = m->kaddr + base;
300
301 i = (eofs - base) >> amortizedshift;
302
303 lo = decode_compactedbits(lclusterbits, lomask,
304 in, encodebits * i, &type);
305 m->type = type;

--- 121 unchanged lines hidden (view full) ---

427 pos += compacted_2b * 2;
428 lcn -= compacted_2b;
429 amortizedshift = 2;
430out:
431 pos += lcn * (1 << amortizedshift);
432 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
433 if (err)
434 return err;
402 return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos),
403 lookahead);
435 return unpack_compacted_index(m, amortizedshift, pos, lookahead);
404}
405
406static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
407 unsigned int lcn, bool lookahead)
408{
409 const unsigned int datamode = EROFS_I(m->inode)->datalayout;
410
411 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)

--- 166 unchanged lines hidden (view full) ---

578 }
579 lcn += m->delta[1];
580 } while (m->delta[1]);
581
582 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la;
583 return 0;
584}
585
436}
437
438static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
439 unsigned int lcn, bool lookahead)
440{
441 const unsigned int datamode = EROFS_I(m->inode)->datalayout;
442
443 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)

--- 166 unchanged lines hidden (view full) ---

610 }
611 lcn += m->delta[1];
612 } while (m->delta[1]);
613
614 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la;
615 return 0;
616}
617
586int z_erofs_map_blocks_iter(struct inode *inode,
587 struct erofs_map_blocks *map,
588 int flags)
618static int z_erofs_do_map_blocks(struct inode *inode,
619 struct erofs_map_blocks *map,
620 int flags)
589{
590 struct erofs_inode *const vi = EROFS_I(inode);
621{
622 struct erofs_inode *const vi = EROFS_I(inode);
623 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER;
591 struct z_erofs_maprecorder m = {
592 .inode = inode,
593 .map = map,
594 };
595 int err = 0;
596 unsigned int lclusterbits, endoff;
597 unsigned long initial_lcn;
598 unsigned long long ofs, end;
599
624 struct z_erofs_maprecorder m = {
625 .inode = inode,
626 .map = map,
627 };
628 int err = 0;
629 unsigned int lclusterbits, endoff;
630 unsigned long initial_lcn;
631 unsigned long long ofs, end;
632
600 trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
601
602 /* when trying to read beyond EOF, leave it unmapped */
603 if (map->m_la >= inode->i_size) {
604 map->m_llen = map->m_la + 1 - inode->i_size;
605 map->m_la = inode->i_size;
606 map->m_flags = 0;
607 goto out;
608 }
609
610 err = z_erofs_fill_inode_lazy(inode);
611 if (err)
612 goto out;
613
614 lclusterbits = vi->z_logical_clusterbits;
633 lclusterbits = vi->z_logical_clusterbits;
615 ofs = map->m_la;
634 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la;
616 initial_lcn = ofs >> lclusterbits;
617 endoff = ofs & ((1 << lclusterbits) - 1);
618
619 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false);
620 if (err)
621 goto unmap_out;
622
635 initial_lcn = ofs >> lclusterbits;
636 endoff = ofs & ((1 << lclusterbits) - 1);
637
638 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false);
639 if (err)
640 goto unmap_out;
641
642 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL))
643 vi->z_idataoff = m.nextpackoff;
644
623 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
624 end = (m.lcn + 1ULL) << lclusterbits;
625
626 switch (m.type) {
627 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
628 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
629 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
630 if (endoff >= m.clusterofs) {

--- 23 unchanged lines hidden (view full) ---

654 erofs_err(inode->i_sb,
655 "unknown type %u @ offset %llu of nid %llu",
656 m.type, ofs, vi->nid);
657 err = -EOPNOTSUPP;
658 goto unmap_out;
659 }
660
661 map->m_llen = end - map->m_la;
645 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
646 end = (m.lcn + 1ULL) << lclusterbits;
647
648 switch (m.type) {
649 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
650 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
651 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
652 if (endoff >= m.clusterofs) {

--- 23 unchanged lines hidden (view full) ---

676 erofs_err(inode->i_sb,
677 "unknown type %u @ offset %llu of nid %llu",
678 m.type, ofs, vi->nid);
679 err = -EOPNOTSUPP;
680 goto unmap_out;
681 }
682
683 map->m_llen = end - map->m_la;
662 map->m_pa = blknr_to_addr(m.pblk);
663
684
664 err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
665 if (err)
666 goto out;
685 if (flags & EROFS_GET_BLOCKS_FINDTAIL)
686 vi->z_tailextent_headlcn = m.lcn;
687 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {
688 map->m_flags |= EROFS_MAP_META;
689 map->m_pa = vi->z_idataoff;
690 map->m_plen = vi->z_idata_size;
691 } else {
692 map->m_pa = blknr_to_addr(m.pblk);
693 err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
694 if (err)
695 goto out;
696 }
667
668 if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN)
669 map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
670 else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2)
671 map->m_algorithmformat = vi->z_algorithmtype[1];
672 else
673 map->m_algorithmformat = vi->z_algorithmtype[0];
674

--- 9 unchanged lines hidden (view full) ---

684 if (m.kaddr)
685 kunmap_atomic(m.kaddr);
686
687out:
688 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
689 __func__, map->m_la, map->m_pa,
690 map->m_llen, map->m_plen, map->m_flags);
691
697
698 if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN)
699 map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
700 else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2)
701 map->m_algorithmformat = vi->z_algorithmtype[1];
702 else
703 map->m_algorithmformat = vi->z_algorithmtype[0];
704

--- 9 unchanged lines hidden (view full) ---

714 if (m.kaddr)
715 kunmap_atomic(m.kaddr);
716
717out:
718 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
719 __func__, map->m_la, map->m_pa,
720 map->m_llen, map->m_plen, map->m_flags);
721
722 return err;
723}
724
725int z_erofs_map_blocks_iter(struct inode *inode,
726 struct erofs_map_blocks *map,
727 int flags)
728{
729 int err = 0;
730
731 trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
732
733 /* when trying to read beyond EOF, leave it unmapped */
734 if (map->m_la >= inode->i_size) {
735 map->m_llen = map->m_la + 1 - inode->i_size;
736 map->m_la = inode->i_size;
737 map->m_flags = 0;
738 goto out;
739 }
740
741 err = z_erofs_fill_inode_lazy(inode);
742 if (err)
743 goto out;
744
745 err = z_erofs_do_map_blocks(inode, map, flags);
746out:
692 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
693
694 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */
695 DBG_BUGON(err < 0 && err != -ENOMEM);
696 return err;
697}
698
699static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,

--- 36 unchanged lines hidden --- 		747 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
748
749 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */
750 DBG_BUGON(err < 0 && err != -ENOMEM);
751 return err;
752}
753
754static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,

--- 36 unchanged lines hidden ---