1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018-2019 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 */ 6 #include "internal.h" 7 #include <asm/unaligned.h> 8 #include <trace/events/erofs.h> 9 10 int z_erofs_fill_inode(struct inode *inode) 11 { 12 struct erofs_inode *const vi = EROFS_I(inode); 13 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); 14 15 if (!erofs_sb_has_big_pcluster(sbi) && 16 !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) && 17 vi->datalayout == EROFS_INODE_COMPRESSED_FULL) { 18 vi->z_advise = 0; 19 vi->z_algorithmtype[0] = 0; 20 vi->z_algorithmtype[1] = 0; 21 vi->z_logical_clusterbits = inode->i_sb->s_blocksize_bits; 22 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 23 } 24 inode->i_mapping->a_ops = &z_erofs_aops; 25 return 0; 26 } 27 28 struct z_erofs_maprecorder { 29 struct inode *inode; 30 struct erofs_map_blocks *map; 31 void *kaddr; 32 33 unsigned long lcn; 34 /* compression extent information gathered */ 35 u8 type, headtype; 36 u16 clusterofs; 37 u16 delta[2]; 38 erofs_blk_t pblk, compressedblks; 39 erofs_off_t nextpackoff; 40 bool partialref; 41 }; 42 43 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, 44 unsigned long lcn) 45 { 46 struct inode *const inode = m->inode; 47 struct erofs_inode *const vi = EROFS_I(inode); 48 const erofs_off_t pos = Z_EROFS_FULL_INDEX_ALIGN(erofs_iloc(inode) + 49 vi->inode_isize + vi->xattr_isize) + 50 lcn * sizeof(struct z_erofs_lcluster_index); 51 struct z_erofs_lcluster_index *di; 52 unsigned int advise, type; 53 54 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, 55 erofs_blknr(inode->i_sb, pos), EROFS_KMAP); 56 if (IS_ERR(m->kaddr)) 57 return PTR_ERR(m->kaddr); 58 59 m->nextpackoff = pos + sizeof(struct z_erofs_lcluster_index); 60 m->lcn = lcn; 61 di = m->kaddr + erofs_blkoff(inode->i_sb, pos); 62 63 advise = le16_to_cpu(di->di_advise); 64 type = (advise >> Z_EROFS_LI_LCLUSTER_TYPE_BIT) & 65 ((1 << Z_EROFS_LI_LCLUSTER_TYPE_BITS) - 1); 66 switch (type) { 67 case Z_EROFS_LCLUSTER_TYPE_NONHEAD: 68 m->clusterofs = 1 << vi->z_logical_clusterbits; 69 m->delta[0] = le16_to_cpu(di->di_u.delta[0]); 70 if (m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) { 71 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 72 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 73 DBG_BUGON(1); 74 return -EFSCORRUPTED; 75 } 76 m->compressedblks = m->delta[0] & 77 ~Z_EROFS_LI_D0_CBLKCNT; 78 m->delta[0] = 1; 79 } 80 m->delta[1] = le16_to_cpu(di->di_u.delta[1]); 81 break; 82 case Z_EROFS_LCLUSTER_TYPE_PLAIN: 83 case Z_EROFS_LCLUSTER_TYPE_HEAD1: 84 case Z_EROFS_LCLUSTER_TYPE_HEAD2: 85 if (advise & Z_EROFS_LI_PARTIAL_REF) 86 m->partialref = true; 87 m->clusterofs = le16_to_cpu(di->di_clusterofs); 88 m->pblk = le32_to_cpu(di->di_u.blkaddr); 89 break; 90 default: 91 DBG_BUGON(1); 92 return -EOPNOTSUPP; 93 } 94 m->type = type; 95 return 0; 96 } 97 98 static unsigned int decode_compactedbits(unsigned int lobits, 99 unsigned int lomask, 100 u8 *in, unsigned int pos, u8 *type) 101 { 102 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); 103 const unsigned int lo = v & lomask; 104 105 *type = (v >> lobits) & 3; 106 return lo; 107 } 108 109 static int get_compacted_la_distance(unsigned int lclusterbits, 110 unsigned int encodebits, 111 unsigned int vcnt, u8 *in, int i) 112 { 113 const unsigned int lomask = (1 << lclusterbits) - 1; 114 unsigned int lo, d1 = 0; 115 u8 type; 116 117 DBG_BUGON(i >= vcnt); 118 119 do { 120 lo = decode_compactedbits(lclusterbits, lomask, 121 in, encodebits * i, &type); 122 123 if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) 124 return d1; 125 ++d1; 126 } while (++i < vcnt); 127 128 /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */ 129 if (!(lo & Z_EROFS_LI_D0_CBLKCNT)) 130 d1 += lo - 1; 131 return d1; 132 } 133 134 static int unpack_compacted_index(struct z_erofs_maprecorder *m, 135 unsigned int amortizedshift, 136 erofs_off_t pos, bool lookahead) 137 { 138 struct erofs_inode *const vi = EROFS_I(m->inode); 139 const unsigned int lclusterbits = vi->z_logical_clusterbits; 140 const unsigned int lomask = (1 << lclusterbits) - 1; 141 unsigned int vcnt, base, lo, encodebits, nblk, eofs; 142 int i; 143 u8 *in, type; 144 bool big_pcluster; 145 146 if (1 << amortizedshift == 4) 147 vcnt = 2; 148 else if (1 << amortizedshift == 2 && lclusterbits == 12) 149 vcnt = 16; 150 else 151 return -EOPNOTSUPP; 152 153 /* it doesn't equal to round_up(..) */ 154 m->nextpackoff = round_down(pos, vcnt << amortizedshift) + 155 (vcnt << amortizedshift); 156 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; 157 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; 158 eofs = erofs_blkoff(m->inode->i_sb, pos); 159 base = round_down(eofs, vcnt << amortizedshift); 160 in = m->kaddr + base; 161 162 i = (eofs - base) >> amortizedshift; 163 164 lo = decode_compactedbits(lclusterbits, lomask, 165 in, encodebits * i, &type); 166 m->type = type; 167 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { 168 m->clusterofs = 1 << lclusterbits; 169 170 /* figure out lookahead_distance: delta[1] if needed */ 171 if (lookahead) 172 m->delta[1] = get_compacted_la_distance(lclusterbits, 173 encodebits, vcnt, in, i); 174 if (lo & Z_EROFS_LI_D0_CBLKCNT) { 175 if (!big_pcluster) { 176 DBG_BUGON(1); 177 return -EFSCORRUPTED; 178 } 179 m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; 180 m->delta[0] = 1; 181 return 0; 182 } else if (i + 1 != (int)vcnt) { 183 m->delta[0] = lo; 184 return 0; 185 } 186 /* 187 * since the last lcluster in the pack is special, 188 * of which lo saves delta[1] rather than delta[0]. 189 * Hence, get delta[0] by the previous lcluster indirectly. 190 */ 191 lo = decode_compactedbits(lclusterbits, lomask, 192 in, encodebits * (i - 1), &type); 193 if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) 194 lo = 0; 195 else if (lo & Z_EROFS_LI_D0_CBLKCNT) 196 lo = 1; 197 m->delta[0] = lo + 1; 198 return 0; 199 } 200 m->clusterofs = lo; 201 m->delta[0] = 0; 202 /* figout out blkaddr (pblk) for HEAD lclusters */ 203 if (!big_pcluster) { 204 nblk = 1; 205 while (i > 0) { 206 --i; 207 lo = decode_compactedbits(lclusterbits, lomask, 208 in, encodebits * i, &type); 209 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) 210 i -= lo; 211 212 if (i >= 0) 213 ++nblk; 214 } 215 } else { 216 nblk = 0; 217 while (i > 0) { 218 --i; 219 lo = decode_compactedbits(lclusterbits, lomask, 220 in, encodebits * i, &type); 221 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { 222 if (lo & Z_EROFS_LI_D0_CBLKCNT) { 223 --i; 224 nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; 225 continue; 226 } 227 /* bigpcluster shouldn't have plain d0 == 1 */ 228 if (lo <= 1) { 229 DBG_BUGON(1); 230 return -EFSCORRUPTED; 231 } 232 i -= lo - 2; 233 continue; 234 } 235 ++nblk; 236 } 237 } 238 in += (vcnt << amortizedshift) - sizeof(__le32); 239 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; 240 return 0; 241 } 242 243 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, 244 unsigned long lcn, bool lookahead) 245 { 246 struct inode *const inode = m->inode; 247 struct erofs_inode *const vi = EROFS_I(inode); 248 const unsigned int lclusterbits = vi->z_logical_clusterbits; 249 const erofs_off_t ebase = sizeof(struct z_erofs_map_header) + 250 ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); 251 unsigned int totalidx = erofs_iblks(inode); 252 unsigned int compacted_4b_initial, compacted_2b; 253 unsigned int amortizedshift; 254 erofs_off_t pos; 255 256 if (lclusterbits != 12) 257 return -EOPNOTSUPP; 258 259 if (lcn >= totalidx) 260 return -EINVAL; 261 262 m->lcn = lcn; 263 /* used to align to 32-byte (compacted_2b) alignment */ 264 compacted_4b_initial = (32 - ebase % 32) / 4; 265 if (compacted_4b_initial == 32 / 4) 266 compacted_4b_initial = 0; 267 268 if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && 269 compacted_4b_initial < totalidx) 270 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); 271 else 272 compacted_2b = 0; 273 274 pos = ebase; 275 if (lcn < compacted_4b_initial) { 276 amortizedshift = 2; 277 goto out; 278 } 279 pos += compacted_4b_initial * 4; 280 lcn -= compacted_4b_initial; 281 282 if (lcn < compacted_2b) { 283 amortizedshift = 1; 284 goto out; 285 } 286 pos += compacted_2b * 2; 287 lcn -= compacted_2b; 288 amortizedshift = 2; 289 out: 290 pos += lcn * (1 << amortizedshift); 291 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, 292 erofs_blknr(inode->i_sb, pos), EROFS_KMAP); 293 if (IS_ERR(m->kaddr)) 294 return PTR_ERR(m->kaddr); 295 return unpack_compacted_index(m, amortizedshift, pos, lookahead); 296 } 297 298 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, 299 unsigned int lcn, bool lookahead) 300 { 301 const unsigned int datamode = EROFS_I(m->inode)->datalayout; 302 303 if (datamode == EROFS_INODE_COMPRESSED_FULL) 304 return legacy_load_cluster_from_disk(m, lcn); 305 306 if (datamode == EROFS_INODE_COMPRESSED_COMPACT) 307 return compacted_load_cluster_from_disk(m, lcn, lookahead); 308 309 return -EINVAL; 310 } 311 312 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, 313 unsigned int lookback_distance) 314 { 315 struct erofs_inode *const vi = EROFS_I(m->inode); 316 const unsigned int lclusterbits = vi->z_logical_clusterbits; 317 318 while (m->lcn >= lookback_distance) { 319 unsigned long lcn = m->lcn - lookback_distance; 320 int err; 321 322 /* load extent head logical cluster if needed */ 323 err = z_erofs_load_cluster_from_disk(m, lcn, false); 324 if (err) 325 return err; 326 327 switch (m->type) { 328 case Z_EROFS_LCLUSTER_TYPE_NONHEAD: 329 if (!m->delta[0]) { 330 erofs_err(m->inode->i_sb, 331 "invalid lookback distance 0 @ nid %llu", 332 vi->nid); 333 DBG_BUGON(1); 334 return -EFSCORRUPTED; 335 } 336 lookback_distance = m->delta[0]; 337 continue; 338 case Z_EROFS_LCLUSTER_TYPE_PLAIN: 339 case Z_EROFS_LCLUSTER_TYPE_HEAD1: 340 case Z_EROFS_LCLUSTER_TYPE_HEAD2: 341 m->headtype = m->type; 342 m->map->m_la = (lcn << lclusterbits) | m->clusterofs; 343 return 0; 344 default: 345 erofs_err(m->inode->i_sb, 346 "unknown type %u @ lcn %lu of nid %llu", 347 m->type, lcn, vi->nid); 348 DBG_BUGON(1); 349 return -EOPNOTSUPP; 350 } 351 } 352 353 erofs_err(m->inode->i_sb, "bogus lookback distance @ nid %llu", 354 vi->nid); 355 DBG_BUGON(1); 356 return -EFSCORRUPTED; 357 } 358 359 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, 360 unsigned int initial_lcn) 361 { 362 struct super_block *sb = m->inode->i_sb; 363 struct erofs_inode *const vi = EROFS_I(m->inode); 364 struct erofs_map_blocks *const map = m->map; 365 const unsigned int lclusterbits = vi->z_logical_clusterbits; 366 unsigned long lcn; 367 int err; 368 369 DBG_BUGON(m->type != Z_EROFS_LCLUSTER_TYPE_PLAIN && 370 m->type != Z_EROFS_LCLUSTER_TYPE_HEAD1 && 371 m->type != Z_EROFS_LCLUSTER_TYPE_HEAD2); 372 DBG_BUGON(m->type != m->headtype); 373 374 if (m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || 375 ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1) && 376 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) || 377 ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && 378 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 379 map->m_plen = 1ULL << lclusterbits; 380 return 0; 381 } 382 lcn = m->lcn + 1; 383 if (m->compressedblks) 384 goto out; 385 386 err = z_erofs_load_cluster_from_disk(m, lcn, false); 387 if (err) 388 return err; 389 390 /* 391 * If the 1st NONHEAD lcluster has already been handled initially w/o 392 * valid compressedblks, which means at least it mustn't be CBLKCNT, or 393 * an internal implemenatation error is detected. 394 * 395 * The following code can also handle it properly anyway, but let's 396 * BUG_ON in the debugging mode only for developers to notice that. 397 */ 398 DBG_BUGON(lcn == initial_lcn && 399 m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); 400 401 switch (m->type) { 402 case Z_EROFS_LCLUSTER_TYPE_PLAIN: 403 case Z_EROFS_LCLUSTER_TYPE_HEAD1: 404 case Z_EROFS_LCLUSTER_TYPE_HEAD2: 405 /* 406 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type 407 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. 408 */ 409 m->compressedblks = 1 << (lclusterbits - sb->s_blocksize_bits); 410 break; 411 case Z_EROFS_LCLUSTER_TYPE_NONHEAD: 412 if (m->delta[0] != 1) 413 goto err_bonus_cblkcnt; 414 if (m->compressedblks) 415 break; 416 fallthrough; 417 default: 418 erofs_err(m->inode->i_sb, 419 "cannot found CBLKCNT @ lcn %lu of nid %llu", 420 lcn, vi->nid); 421 DBG_BUGON(1); 422 return -EFSCORRUPTED; 423 } 424 out: 425 map->m_plen = erofs_pos(sb, m->compressedblks); 426 return 0; 427 err_bonus_cblkcnt: 428 erofs_err(m->inode->i_sb, 429 "bogus CBLKCNT @ lcn %lu of nid %llu", 430 lcn, vi->nid); 431 DBG_BUGON(1); 432 return -EFSCORRUPTED; 433 } 434 435 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) 436 { 437 struct inode *inode = m->inode; 438 struct erofs_inode *vi = EROFS_I(inode); 439 struct erofs_map_blocks *map = m->map; 440 unsigned int lclusterbits = vi->z_logical_clusterbits; 441 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits; 442 int err; 443 444 do { 445 /* handle the last EOF pcluster (no next HEAD lcluster) */ 446 if ((lcn << lclusterbits) >= inode->i_size) { 447 map->m_llen = inode->i_size - map->m_la; 448 return 0; 449 } 450 451 err = z_erofs_load_cluster_from_disk(m, lcn, true); 452 if (err) 453 return err; 454 455 if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { 456 DBG_BUGON(!m->delta[1] && 457 m->clusterofs != 1 << lclusterbits); 458 } else if (m->type == Z_EROFS_LCLUSTER_TYPE_PLAIN || 459 m->type == Z_EROFS_LCLUSTER_TYPE_HEAD1 || 460 m->type == Z_EROFS_LCLUSTER_TYPE_HEAD2) { 461 /* go on until the next HEAD lcluster */ 462 if (lcn != headlcn) 463 break; 464 m->delta[1] = 1; 465 } else { 466 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", 467 m->type, lcn, vi->nid); 468 DBG_BUGON(1); 469 return -EOPNOTSUPP; 470 } 471 lcn += m->delta[1]; 472 } while (m->delta[1]); 473 474 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la; 475 return 0; 476 } 477 478 static int z_erofs_do_map_blocks(struct inode *inode, 479 struct erofs_map_blocks *map, int flags) 480 { 481 struct erofs_inode *const vi = EROFS_I(inode); 482 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; 483 bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 484 struct z_erofs_maprecorder m = { 485 .inode = inode, 486 .map = map, 487 }; 488 int err = 0; 489 unsigned int lclusterbits, endoff; 490 unsigned long initial_lcn; 491 unsigned long long ofs, end; 492 493 lclusterbits = vi->z_logical_clusterbits; 494 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la; 495 initial_lcn = ofs >> lclusterbits; 496 endoff = ofs & ((1 << lclusterbits) - 1); 497 498 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false); 499 if (err) 500 goto unmap_out; 501 502 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL)) 503 vi->z_idataoff = m.nextpackoff; 504 505 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; 506 end = (m.lcn + 1ULL) << lclusterbits; 507 508 switch (m.type) { 509 case Z_EROFS_LCLUSTER_TYPE_PLAIN: 510 case Z_EROFS_LCLUSTER_TYPE_HEAD1: 511 case Z_EROFS_LCLUSTER_TYPE_HEAD2: 512 if (endoff >= m.clusterofs) { 513 m.headtype = m.type; 514 map->m_la = (m.lcn << lclusterbits) | m.clusterofs; 515 /* 516 * For ztailpacking files, in order to inline data more 517 * effectively, special EOF lclusters are now supported 518 * which can have three parts at most. 519 */ 520 if (ztailpacking && end > inode->i_size) 521 end = inode->i_size; 522 break; 523 } 524 /* m.lcn should be >= 1 if endoff < m.clusterofs */ 525 if (!m.lcn) { 526 erofs_err(inode->i_sb, 527 "invalid logical cluster 0 at nid %llu", 528 vi->nid); 529 err = -EFSCORRUPTED; 530 goto unmap_out; 531 } 532 end = (m.lcn << lclusterbits) | m.clusterofs; 533 map->m_flags |= EROFS_MAP_FULL_MAPPED; 534 m.delta[0] = 1; 535 fallthrough; 536 case Z_EROFS_LCLUSTER_TYPE_NONHEAD: 537 /* get the corresponding first chunk */ 538 err = z_erofs_extent_lookback(&m, m.delta[0]); 539 if (err) 540 goto unmap_out; 541 break; 542 default: 543 erofs_err(inode->i_sb, 544 "unknown type %u @ offset %llu of nid %llu", 545 m.type, ofs, vi->nid); 546 err = -EOPNOTSUPP; 547 goto unmap_out; 548 } 549 if (m.partialref) 550 map->m_flags |= EROFS_MAP_PARTIAL_REF; 551 map->m_llen = end - map->m_la; 552 553 if (flags & EROFS_GET_BLOCKS_FINDTAIL) { 554 vi->z_tailextent_headlcn = m.lcn; 555 /* for non-compact indexes, fragmentoff is 64 bits */ 556 if (fragment && 557 vi->datalayout == EROFS_INODE_COMPRESSED_FULL) 558 vi->z_fragmentoff |= (u64)m.pblk << 32; 559 } 560 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { 561 map->m_flags |= EROFS_MAP_META; 562 map->m_pa = vi->z_idataoff; 563 map->m_plen = vi->z_idata_size; 564 } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { 565 map->m_flags |= EROFS_MAP_FRAGMENT; 566 } else { 567 map->m_pa = erofs_pos(inode->i_sb, m.pblk); 568 err = z_erofs_get_extent_compressedlen(&m, initial_lcn); 569 if (err) 570 goto unmap_out; 571 } 572 573 if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { 574 if (map->m_llen > map->m_plen) { 575 DBG_BUGON(1); 576 err = -EFSCORRUPTED; 577 goto unmap_out; 578 } 579 if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) 580 map->m_algorithmformat = 581 Z_EROFS_COMPRESSION_INTERLACED; 582 else 583 map->m_algorithmformat = 584 Z_EROFS_COMPRESSION_SHIFTED; 585 } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { 586 map->m_algorithmformat = vi->z_algorithmtype[1]; 587 } else { 588 map->m_algorithmformat = vi->z_algorithmtype[0]; 589 } 590 591 if ((flags & EROFS_GET_BLOCKS_FIEMAP) || 592 ((flags & EROFS_GET_BLOCKS_READMORE) && 593 map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA && 594 map->m_llen >= i_blocksize(inode))) { 595 err = z_erofs_get_extent_decompressedlen(&m); 596 if (!err) 597 map->m_flags |= EROFS_MAP_FULL_MAPPED; 598 } 599 600 unmap_out: 601 erofs_unmap_metabuf(&m.map->buf); 602 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o", 603 __func__, map->m_la, map->m_pa, 604 map->m_llen, map->m_plen, map->m_flags); 605 return err; 606 } 607 608 static int z_erofs_fill_inode_lazy(struct inode *inode) 609 { 610 struct erofs_inode *const vi = EROFS_I(inode); 611 struct super_block *const sb = inode->i_sb; 612 int err, headnr; 613 erofs_off_t pos; 614 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 615 void *kaddr; 616 struct z_erofs_map_header *h; 617 618 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { 619 /* 620 * paired with smp_mb() at the end of the function to ensure 621 * fields will only be observed after the bit is set. 622 */ 623 smp_mb(); 624 return 0; 625 } 626 627 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) 628 return -ERESTARTSYS; 629 630 err = 0; 631 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) 632 goto out_unlock; 633 634 pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); 635 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(sb, pos), EROFS_KMAP); 636 if (IS_ERR(kaddr)) { 637 err = PTR_ERR(kaddr); 638 goto out_unlock; 639 } 640 641 h = kaddr + erofs_blkoff(sb, pos); 642 /* 643 * if the highest bit of the 8-byte map header is set, the whole file 644 * is stored in the packed inode. The rest bits keeps z_fragmentoff. 645 */ 646 if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { 647 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 648 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); 649 vi->z_tailextent_headlcn = 0; 650 goto done; 651 } 652 vi->z_advise = le16_to_cpu(h->h_advise); 653 vi->z_algorithmtype[0] = h->h_algorithmtype & 15; 654 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; 655 656 headnr = 0; 657 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX || 658 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) { 659 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", 660 headnr + 1, vi->z_algorithmtype[headnr], vi->nid); 661 err = -EOPNOTSUPP; 662 goto out_put_metabuf; 663 } 664 665 vi->z_logical_clusterbits = sb->s_blocksize_bits + (h->h_clusterbits & 7); 666 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && 667 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 668 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 669 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", 670 vi->nid); 671 err = -EFSCORRUPTED; 672 goto out_put_metabuf; 673 } 674 if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && 675 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ 676 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 677 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", 678 vi->nid); 679 err = -EFSCORRUPTED; 680 goto out_put_metabuf; 681 } 682 683 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) { 684 struct erofs_map_blocks map = { 685 .buf = __EROFS_BUF_INITIALIZER 686 }; 687 688 vi->z_idata_size = le16_to_cpu(h->h_idata_size); 689 err = z_erofs_do_map_blocks(inode, &map, 690 EROFS_GET_BLOCKS_FINDTAIL); 691 erofs_put_metabuf(&map.buf); 692 693 if (!map.m_plen || 694 erofs_blkoff(sb, map.m_pa) + map.m_plen > sb->s_blocksize) { 695 erofs_err(sb, "invalid tail-packing pclustersize %llu", 696 map.m_plen); 697 err = -EFSCORRUPTED; 698 } 699 if (err < 0) 700 goto out_put_metabuf; 701 } 702 703 if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER && 704 !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) { 705 struct erofs_map_blocks map = { 706 .buf = __EROFS_BUF_INITIALIZER 707 }; 708 709 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); 710 err = z_erofs_do_map_blocks(inode, &map, 711 EROFS_GET_BLOCKS_FINDTAIL); 712 erofs_put_metabuf(&map.buf); 713 if (err < 0) 714 goto out_put_metabuf; 715 } 716 done: 717 /* paired with smp_mb() at the beginning of the function */ 718 smp_mb(); 719 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 720 out_put_metabuf: 721 erofs_put_metabuf(&buf); 722 out_unlock: 723 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); 724 return err; 725 } 726 727 int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, 728 int flags) 729 { 730 struct erofs_inode *const vi = EROFS_I(inode); 731 int err = 0; 732 733 trace_z_erofs_map_blocks_iter_enter(inode, map, flags); 734 735 /* when trying to read beyond EOF, leave it unmapped */ 736 if (map->m_la >= inode->i_size) { 737 map->m_llen = map->m_la + 1 - inode->i_size; 738 map->m_la = inode->i_size; 739 map->m_flags = 0; 740 goto out; 741 } 742 743 err = z_erofs_fill_inode_lazy(inode); 744 if (err) 745 goto out; 746 747 if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) && 748 !vi->z_tailextent_headlcn) { 749 map->m_la = 0; 750 map->m_llen = inode->i_size; 751 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | 752 EROFS_MAP_FRAGMENT; 753 goto out; 754 } 755 756 err = z_erofs_do_map_blocks(inode, map, flags); 757 out: 758 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); 759 return err; 760 } 761 762 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, 763 loff_t length, unsigned int flags, 764 struct iomap *iomap, struct iomap *srcmap) 765 { 766 int ret; 767 struct erofs_map_blocks map = { .m_la = offset }; 768 769 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP); 770 erofs_put_metabuf(&map.buf); 771 if (ret < 0) 772 return ret; 773 774 iomap->bdev = inode->i_sb->s_bdev; 775 iomap->offset = map.m_la; 776 iomap->length = map.m_llen; 777 if (map.m_flags & EROFS_MAP_MAPPED) { 778 iomap->type = IOMAP_MAPPED; 779 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ? 780 IOMAP_NULL_ADDR : map.m_pa; 781 } else { 782 iomap->type = IOMAP_HOLE; 783 iomap->addr = IOMAP_NULL_ADDR; 784 /* 785 * No strict rule on how to describe extents for post EOF, yet 786 * we need to do like below. Otherwise, iomap itself will get 787 * into an endless loop on post EOF. 788 * 789 * Calculate the effective offset by subtracting extent start 790 * (map.m_la) from the requested offset, and add it to length. 791 * (NB: offset >= map.m_la always) 792 */ 793 if (iomap->offset >= inode->i_size) 794 iomap->length = length + offset - map.m_la; 795 } 796 iomap->flags = 0; 797 return 0; 798 } 799 800 const struct iomap_ops z_erofs_iomap_report_ops = { 801 .iomap_begin = z_erofs_iomap_begin_report, 802 }; 803