1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018-2019 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 */ 6 #include "internal.h" 7 #include <asm/unaligned.h> 8 #include <trace/events/erofs.h> 9 10 static int z_erofs_do_map_blocks(struct inode *inode, 11 struct erofs_map_blocks *map, 12 int flags); 13 14 int z_erofs_fill_inode(struct inode *inode) 15 { 16 struct erofs_inode *const vi = EROFS_I(inode); 17 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); 18 19 if (!erofs_sb_has_big_pcluster(sbi) && 20 !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) && 21 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { 22 vi->z_advise = 0; 23 vi->z_algorithmtype[0] = 0; 24 vi->z_algorithmtype[1] = 0; 25 vi->z_logical_clusterbits = LOG_BLOCK_SIZE; 26 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 27 } 28 inode->i_mapping->a_ops = &z_erofs_aops; 29 return 0; 30 } 31 32 static int z_erofs_fill_inode_lazy(struct inode *inode) 33 { 34 struct erofs_inode *const vi = EROFS_I(inode); 35 struct super_block *const sb = inode->i_sb; 36 int err, headnr; 37 erofs_off_t pos; 38 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 39 void *kaddr; 40 struct z_erofs_map_header *h; 41 42 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { 43 /* 44 * paired with smp_mb() at the end of the function to ensure 45 * fields will only be observed after the bit is set. 46 */ 47 smp_mb(); 48 return 0; 49 } 50 51 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) 52 return -ERESTARTSYS; 53 54 err = 0; 55 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) 56 goto out_unlock; 57 58 pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize + 59 vi->xattr_isize, 8); 60 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos), 61 EROFS_KMAP_ATOMIC); 62 if (IS_ERR(kaddr)) { 63 err = PTR_ERR(kaddr); 64 goto out_unlock; 65 } 66 67 h = kaddr + erofs_blkoff(pos); 68 /* 69 * if the highest bit of the 8-byte map header is set, the whole file 70 * is stored in the packed inode. The rest bits keeps z_fragmentoff. 71 */ 72 if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { 73 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 74 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); 75 vi->z_tailextent_headlcn = 0; 76 goto unmap_done; 77 } 78 vi->z_advise = le16_to_cpu(h->h_advise); 79 vi->z_algorithmtype[0] = h->h_algorithmtype & 15; 80 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; 81 82 headnr = 0; 83 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX || 84 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) { 85 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", 86 headnr + 1, vi->z_algorithmtype[headnr], vi->nid); 87 err = -EOPNOTSUPP; 88 goto unmap_done; 89 } 90 91 vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7); 92 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && 93 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 94 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 95 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", 96 vi->nid); 97 err = -EFSCORRUPTED; 98 goto unmap_done; 99 } 100 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION && 101 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ 102 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 103 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", 104 vi->nid); 105 err = -EFSCORRUPTED; 106 goto unmap_done; 107 } 108 unmap_done: 109 erofs_put_metabuf(&buf); 110 if (err) 111 goto out_unlock; 112 113 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) { 114 struct erofs_map_blocks map = { 115 .buf = __EROFS_BUF_INITIALIZER 116 }; 117 118 vi->z_idata_size = le16_to_cpu(h->h_idata_size); 119 err = z_erofs_do_map_blocks(inode, &map, 120 EROFS_GET_BLOCKS_FINDTAIL); 121 erofs_put_metabuf(&map.buf); 122 123 if (!map.m_plen || 124 erofs_blkoff(map.m_pa) + map.m_plen > EROFS_BLKSIZ) { 125 erofs_err(sb, "invalid tail-packing pclustersize %llu", 126 map.m_plen); 127 err = -EFSCORRUPTED; 128 } 129 if (err < 0) 130 goto out_unlock; 131 } 132 133 if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER && 134 !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) { 135 struct erofs_map_blocks map = { 136 .buf = __EROFS_BUF_INITIALIZER 137 }; 138 139 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); 140 err = z_erofs_do_map_blocks(inode, &map, 141 EROFS_GET_BLOCKS_FINDTAIL); 142 erofs_put_metabuf(&map.buf); 143 if (err < 0) 144 goto out_unlock; 145 } 146 /* paired with smp_mb() at the beginning of the function */ 147 smp_mb(); 148 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 149 out_unlock: 150 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); 151 return err; 152 } 153 154 struct z_erofs_maprecorder { 155 struct inode *inode; 156 struct erofs_map_blocks *map; 157 void *kaddr; 158 159 unsigned long lcn; 160 /* compression extent information gathered */ 161 u8 type, headtype; 162 u16 clusterofs; 163 u16 delta[2]; 164 erofs_blk_t pblk, compressedblks; 165 erofs_off_t nextpackoff; 166 }; 167 168 static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, 169 erofs_blk_t eblk) 170 { 171 struct super_block *const sb = m->inode->i_sb; 172 173 m->kaddr = erofs_read_metabuf(&m->map->buf, sb, eblk, 174 EROFS_KMAP_ATOMIC); 175 if (IS_ERR(m->kaddr)) 176 return PTR_ERR(m->kaddr); 177 return 0; 178 } 179 180 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, 181 unsigned long lcn) 182 { 183 struct inode *const inode = m->inode; 184 struct erofs_inode *const vi = EROFS_I(inode); 185 const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid); 186 const erofs_off_t pos = 187 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize + 188 vi->xattr_isize) + 189 lcn * sizeof(struct z_erofs_vle_decompressed_index); 190 struct z_erofs_vle_decompressed_index *di; 191 unsigned int advise, type; 192 int err; 193 194 err = z_erofs_reload_indexes(m, erofs_blknr(pos)); 195 if (err) 196 return err; 197 198 m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index); 199 m->lcn = lcn; 200 di = m->kaddr + erofs_blkoff(pos); 201 202 advise = le16_to_cpu(di->di_advise); 203 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) & 204 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1); 205 switch (type) { 206 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 207 m->clusterofs = 1 << vi->z_logical_clusterbits; 208 m->delta[0] = le16_to_cpu(di->di_u.delta[0]); 209 if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) { 210 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 211 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 212 DBG_BUGON(1); 213 return -EFSCORRUPTED; 214 } 215 m->compressedblks = m->delta[0] & 216 ~Z_EROFS_VLE_DI_D0_CBLKCNT; 217 m->delta[0] = 1; 218 } 219 m->delta[1] = le16_to_cpu(di->di_u.delta[1]); 220 break; 221 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 222 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 223 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 224 m->clusterofs = le16_to_cpu(di->di_clusterofs); 225 m->pblk = le32_to_cpu(di->di_u.blkaddr); 226 break; 227 default: 228 DBG_BUGON(1); 229 return -EOPNOTSUPP; 230 } 231 m->type = type; 232 return 0; 233 } 234 235 static unsigned int decode_compactedbits(unsigned int lobits, 236 unsigned int lomask, 237 u8 *in, unsigned int pos, u8 *type) 238 { 239 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); 240 const unsigned int lo = v & lomask; 241 242 *type = (v >> lobits) & 3; 243 return lo; 244 } 245 246 static int get_compacted_la_distance(unsigned int lclusterbits, 247 unsigned int encodebits, 248 unsigned int vcnt, u8 *in, int i) 249 { 250 const unsigned int lomask = (1 << lclusterbits) - 1; 251 unsigned int lo, d1 = 0; 252 u8 type; 253 254 DBG_BUGON(i >= vcnt); 255 256 do { 257 lo = decode_compactedbits(lclusterbits, lomask, 258 in, encodebits * i, &type); 259 260 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 261 return d1; 262 ++d1; 263 } while (++i < vcnt); 264 265 /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */ 266 if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT)) 267 d1 += lo - 1; 268 return d1; 269 } 270 271 static int unpack_compacted_index(struct z_erofs_maprecorder *m, 272 unsigned int amortizedshift, 273 erofs_off_t pos, bool lookahead) 274 { 275 struct erofs_inode *const vi = EROFS_I(m->inode); 276 const unsigned int lclusterbits = vi->z_logical_clusterbits; 277 const unsigned int lomask = (1 << lclusterbits) - 1; 278 unsigned int vcnt, base, lo, encodebits, nblk, eofs; 279 int i; 280 u8 *in, type; 281 bool big_pcluster; 282 283 if (1 << amortizedshift == 4) 284 vcnt = 2; 285 else if (1 << amortizedshift == 2 && lclusterbits == 12) 286 vcnt = 16; 287 else 288 return -EOPNOTSUPP; 289 290 /* it doesn't equal to round_up(..) */ 291 m->nextpackoff = round_down(pos, vcnt << amortizedshift) + 292 (vcnt << amortizedshift); 293 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; 294 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; 295 eofs = erofs_blkoff(pos); 296 base = round_down(eofs, vcnt << amortizedshift); 297 in = m->kaddr + base; 298 299 i = (eofs - base) >> amortizedshift; 300 301 lo = decode_compactedbits(lclusterbits, lomask, 302 in, encodebits * i, &type); 303 m->type = type; 304 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 305 m->clusterofs = 1 << lclusterbits; 306 307 /* figure out lookahead_distance: delta[1] if needed */ 308 if (lookahead) 309 m->delta[1] = get_compacted_la_distance(lclusterbits, 310 encodebits, vcnt, in, i); 311 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { 312 if (!big_pcluster) { 313 DBG_BUGON(1); 314 return -EFSCORRUPTED; 315 } 316 m->compressedblks = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; 317 m->delta[0] = 1; 318 return 0; 319 } else if (i + 1 != (int)vcnt) { 320 m->delta[0] = lo; 321 return 0; 322 } 323 /* 324 * since the last lcluster in the pack is special, 325 * of which lo saves delta[1] rather than delta[0]. 326 * Hence, get delta[0] by the previous lcluster indirectly. 327 */ 328 lo = decode_compactedbits(lclusterbits, lomask, 329 in, encodebits * (i - 1), &type); 330 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 331 lo = 0; 332 else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) 333 lo = 1; 334 m->delta[0] = lo + 1; 335 return 0; 336 } 337 m->clusterofs = lo; 338 m->delta[0] = 0; 339 /* figout out blkaddr (pblk) for HEAD lclusters */ 340 if (!big_pcluster) { 341 nblk = 1; 342 while (i > 0) { 343 --i; 344 lo = decode_compactedbits(lclusterbits, lomask, 345 in, encodebits * i, &type); 346 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 347 i -= lo; 348 349 if (i >= 0) 350 ++nblk; 351 } 352 } else { 353 nblk = 0; 354 while (i > 0) { 355 --i; 356 lo = decode_compactedbits(lclusterbits, lomask, 357 in, encodebits * i, &type); 358 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 359 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { 360 --i; 361 nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; 362 continue; 363 } 364 /* bigpcluster shouldn't have plain d0 == 1 */ 365 if (lo <= 1) { 366 DBG_BUGON(1); 367 return -EFSCORRUPTED; 368 } 369 i -= lo - 2; 370 continue; 371 } 372 ++nblk; 373 } 374 } 375 in += (vcnt << amortizedshift) - sizeof(__le32); 376 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; 377 return 0; 378 } 379 380 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, 381 unsigned long lcn, bool lookahead) 382 { 383 struct inode *const inode = m->inode; 384 struct erofs_inode *const vi = EROFS_I(inode); 385 const unsigned int lclusterbits = vi->z_logical_clusterbits; 386 const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) + 387 vi->inode_isize + vi->xattr_isize, 8) + 388 sizeof(struct z_erofs_map_header); 389 const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); 390 unsigned int compacted_4b_initial, compacted_2b; 391 unsigned int amortizedshift; 392 erofs_off_t pos; 393 int err; 394 395 if (lclusterbits != 12) 396 return -EOPNOTSUPP; 397 398 if (lcn >= totalidx) 399 return -EINVAL; 400 401 m->lcn = lcn; 402 /* used to align to 32-byte (compacted_2b) alignment */ 403 compacted_4b_initial = (32 - ebase % 32) / 4; 404 if (compacted_4b_initial == 32 / 4) 405 compacted_4b_initial = 0; 406 407 if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && 408 compacted_4b_initial < totalidx) 409 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); 410 else 411 compacted_2b = 0; 412 413 pos = ebase; 414 if (lcn < compacted_4b_initial) { 415 amortizedshift = 2; 416 goto out; 417 } 418 pos += compacted_4b_initial * 4; 419 lcn -= compacted_4b_initial; 420 421 if (lcn < compacted_2b) { 422 amortizedshift = 1; 423 goto out; 424 } 425 pos += compacted_2b * 2; 426 lcn -= compacted_2b; 427 amortizedshift = 2; 428 out: 429 pos += lcn * (1 << amortizedshift); 430 err = z_erofs_reload_indexes(m, erofs_blknr(pos)); 431 if (err) 432 return err; 433 return unpack_compacted_index(m, amortizedshift, pos, lookahead); 434 } 435 436 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, 437 unsigned int lcn, bool lookahead) 438 { 439 const unsigned int datamode = EROFS_I(m->inode)->datalayout; 440 441 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY) 442 return legacy_load_cluster_from_disk(m, lcn); 443 444 if (datamode == EROFS_INODE_FLAT_COMPRESSION) 445 return compacted_load_cluster_from_disk(m, lcn, lookahead); 446 447 return -EINVAL; 448 } 449 450 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, 451 unsigned int lookback_distance) 452 { 453 struct erofs_inode *const vi = EROFS_I(m->inode); 454 const unsigned int lclusterbits = vi->z_logical_clusterbits; 455 456 while (m->lcn >= lookback_distance) { 457 unsigned long lcn = m->lcn - lookback_distance; 458 int err; 459 460 /* load extent head logical cluster if needed */ 461 err = z_erofs_load_cluster_from_disk(m, lcn, false); 462 if (err) 463 return err; 464 465 switch (m->type) { 466 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 467 if (!m->delta[0]) { 468 erofs_err(m->inode->i_sb, 469 "invalid lookback distance 0 @ nid %llu", 470 vi->nid); 471 DBG_BUGON(1); 472 return -EFSCORRUPTED; 473 } 474 lookback_distance = m->delta[0]; 475 continue; 476 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 477 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 478 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 479 m->headtype = m->type; 480 m->map->m_la = (lcn << lclusterbits) | m->clusterofs; 481 return 0; 482 default: 483 erofs_err(m->inode->i_sb, 484 "unknown type %u @ lcn %lu of nid %llu", 485 m->type, lcn, vi->nid); 486 DBG_BUGON(1); 487 return -EOPNOTSUPP; 488 } 489 } 490 491 erofs_err(m->inode->i_sb, "bogus lookback distance @ nid %llu", 492 vi->nid); 493 DBG_BUGON(1); 494 return -EFSCORRUPTED; 495 } 496 497 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, 498 unsigned int initial_lcn) 499 { 500 struct erofs_inode *const vi = EROFS_I(m->inode); 501 struct erofs_map_blocks *const map = m->map; 502 const unsigned int lclusterbits = vi->z_logical_clusterbits; 503 unsigned long lcn; 504 int err; 505 506 DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN && 507 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 && 508 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD2); 509 DBG_BUGON(m->type != m->headtype); 510 511 if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || 512 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1) && 513 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) || 514 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) && 515 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 516 map->m_plen = 1ULL << lclusterbits; 517 return 0; 518 } 519 lcn = m->lcn + 1; 520 if (m->compressedblks) 521 goto out; 522 523 err = z_erofs_load_cluster_from_disk(m, lcn, false); 524 if (err) 525 return err; 526 527 /* 528 * If the 1st NONHEAD lcluster has already been handled initially w/o 529 * valid compressedblks, which means at least it mustn't be CBLKCNT, or 530 * an internal implemenatation error is detected. 531 * 532 * The following code can also handle it properly anyway, but let's 533 * BUG_ON in the debugging mode only for developers to notice that. 534 */ 535 DBG_BUGON(lcn == initial_lcn && 536 m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD); 537 538 switch (m->type) { 539 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 540 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 541 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 542 /* 543 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type 544 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. 545 */ 546 m->compressedblks = 1 << (lclusterbits - LOG_BLOCK_SIZE); 547 break; 548 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 549 if (m->delta[0] != 1) 550 goto err_bonus_cblkcnt; 551 if (m->compressedblks) 552 break; 553 fallthrough; 554 default: 555 erofs_err(m->inode->i_sb, 556 "cannot found CBLKCNT @ lcn %lu of nid %llu", 557 lcn, vi->nid); 558 DBG_BUGON(1); 559 return -EFSCORRUPTED; 560 } 561 out: 562 map->m_plen = (u64)m->compressedblks << LOG_BLOCK_SIZE; 563 return 0; 564 err_bonus_cblkcnt: 565 erofs_err(m->inode->i_sb, 566 "bogus CBLKCNT @ lcn %lu of nid %llu", 567 lcn, vi->nid); 568 DBG_BUGON(1); 569 return -EFSCORRUPTED; 570 } 571 572 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) 573 { 574 struct inode *inode = m->inode; 575 struct erofs_inode *vi = EROFS_I(inode); 576 struct erofs_map_blocks *map = m->map; 577 unsigned int lclusterbits = vi->z_logical_clusterbits; 578 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits; 579 int err; 580 581 do { 582 /* handle the last EOF pcluster (no next HEAD lcluster) */ 583 if ((lcn << lclusterbits) >= inode->i_size) { 584 map->m_llen = inode->i_size - map->m_la; 585 return 0; 586 } 587 588 err = z_erofs_load_cluster_from_disk(m, lcn, true); 589 if (err) 590 return err; 591 592 if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 593 DBG_BUGON(!m->delta[1] && 594 m->clusterofs != 1 << lclusterbits); 595 } else if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || 596 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 || 597 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) { 598 /* go on until the next HEAD lcluster */ 599 if (lcn != headlcn) 600 break; 601 m->delta[1] = 1; 602 } else { 603 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", 604 m->type, lcn, vi->nid); 605 DBG_BUGON(1); 606 return -EOPNOTSUPP; 607 } 608 lcn += m->delta[1]; 609 } while (m->delta[1]); 610 611 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la; 612 return 0; 613 } 614 615 static int z_erofs_do_map_blocks(struct inode *inode, 616 struct erofs_map_blocks *map, 617 int flags) 618 { 619 struct erofs_inode *const vi = EROFS_I(inode); 620 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; 621 bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 622 struct z_erofs_maprecorder m = { 623 .inode = inode, 624 .map = map, 625 }; 626 int err = 0; 627 unsigned int lclusterbits, endoff; 628 unsigned long initial_lcn; 629 unsigned long long ofs, end; 630 631 lclusterbits = vi->z_logical_clusterbits; 632 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la; 633 initial_lcn = ofs >> lclusterbits; 634 endoff = ofs & ((1 << lclusterbits) - 1); 635 636 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false); 637 if (err) 638 goto unmap_out; 639 640 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL)) 641 vi->z_idataoff = m.nextpackoff; 642 643 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; 644 end = (m.lcn + 1ULL) << lclusterbits; 645 646 switch (m.type) { 647 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 648 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 649 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 650 if (endoff >= m.clusterofs) { 651 m.headtype = m.type; 652 map->m_la = (m.lcn << lclusterbits) | m.clusterofs; 653 /* 654 * For ztailpacking files, in order to inline data more 655 * effectively, special EOF lclusters are now supported 656 * which can have three parts at most. 657 */ 658 if (ztailpacking && end > inode->i_size) 659 end = inode->i_size; 660 break; 661 } 662 /* m.lcn should be >= 1 if endoff < m.clusterofs */ 663 if (!m.lcn) { 664 erofs_err(inode->i_sb, 665 "invalid logical cluster 0 at nid %llu", 666 vi->nid); 667 err = -EFSCORRUPTED; 668 goto unmap_out; 669 } 670 end = (m.lcn << lclusterbits) | m.clusterofs; 671 map->m_flags |= EROFS_MAP_FULL_MAPPED; 672 m.delta[0] = 1; 673 fallthrough; 674 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 675 /* get the corresponding first chunk */ 676 err = z_erofs_extent_lookback(&m, m.delta[0]); 677 if (err) 678 goto unmap_out; 679 break; 680 default: 681 erofs_err(inode->i_sb, 682 "unknown type %u @ offset %llu of nid %llu", 683 m.type, ofs, vi->nid); 684 err = -EOPNOTSUPP; 685 goto unmap_out; 686 } 687 688 map->m_llen = end - map->m_la; 689 690 if (flags & EROFS_GET_BLOCKS_FINDTAIL) { 691 vi->z_tailextent_headlcn = m.lcn; 692 /* for non-compact indexes, fragmentoff is 64 bits */ 693 if (fragment && 694 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) 695 vi->z_fragmentoff |= (u64)m.pblk << 32; 696 } 697 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { 698 map->m_flags |= EROFS_MAP_META; 699 map->m_pa = vi->z_idataoff; 700 map->m_plen = vi->z_idata_size; 701 } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { 702 map->m_flags |= EROFS_MAP_FRAGMENT; 703 } else { 704 map->m_pa = blknr_to_addr(m.pblk); 705 err = z_erofs_get_extent_compressedlen(&m, initial_lcn); 706 if (err) 707 goto out; 708 } 709 710 if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) { 711 if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) 712 map->m_algorithmformat = 713 Z_EROFS_COMPRESSION_INTERLACED; 714 else 715 map->m_algorithmformat = 716 Z_EROFS_COMPRESSION_SHIFTED; 717 } else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) { 718 map->m_algorithmformat = vi->z_algorithmtype[1]; 719 } else { 720 map->m_algorithmformat = vi->z_algorithmtype[0]; 721 } 722 723 if ((flags & EROFS_GET_BLOCKS_FIEMAP) || 724 ((flags & EROFS_GET_BLOCKS_READMORE) && 725 map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA && 726 map->m_llen >= EROFS_BLKSIZ)) { 727 err = z_erofs_get_extent_decompressedlen(&m); 728 if (!err) 729 map->m_flags |= EROFS_MAP_FULL_MAPPED; 730 } 731 unmap_out: 732 erofs_unmap_metabuf(&m.map->buf); 733 734 out: 735 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o", 736 __func__, map->m_la, map->m_pa, 737 map->m_llen, map->m_plen, map->m_flags); 738 739 return err; 740 } 741 742 int z_erofs_map_blocks_iter(struct inode *inode, 743 struct erofs_map_blocks *map, 744 int flags) 745 { 746 struct erofs_inode *const vi = EROFS_I(inode); 747 int err = 0; 748 749 trace_z_erofs_map_blocks_iter_enter(inode, map, flags); 750 751 /* when trying to read beyond EOF, leave it unmapped */ 752 if (map->m_la >= inode->i_size) { 753 map->m_llen = map->m_la + 1 - inode->i_size; 754 map->m_la = inode->i_size; 755 map->m_flags = 0; 756 goto out; 757 } 758 759 err = z_erofs_fill_inode_lazy(inode); 760 if (err) 761 goto out; 762 763 if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) && 764 !vi->z_tailextent_headlcn) { 765 map->m_la = 0; 766 map->m_llen = inode->i_size; 767 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | 768 EROFS_MAP_FRAGMENT; 769 goto out; 770 } 771 772 err = z_erofs_do_map_blocks(inode, map, flags); 773 out: 774 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); 775 776 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */ 777 DBG_BUGON(err < 0 && err != -ENOMEM); 778 return err; 779 } 780 781 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, 782 loff_t length, unsigned int flags, 783 struct iomap *iomap, struct iomap *srcmap) 784 { 785 int ret; 786 struct erofs_map_blocks map = { .m_la = offset }; 787 788 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP); 789 erofs_put_metabuf(&map.buf); 790 if (ret < 0) 791 return ret; 792 793 iomap->bdev = inode->i_sb->s_bdev; 794 iomap->offset = map.m_la; 795 iomap->length = map.m_llen; 796 if (map.m_flags & EROFS_MAP_MAPPED) { 797 iomap->type = IOMAP_MAPPED; 798 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ? 799 IOMAP_NULL_ADDR : map.m_pa; 800 } else { 801 iomap->type = IOMAP_HOLE; 802 iomap->addr = IOMAP_NULL_ADDR; 803 /* 804 * No strict rule how to describe extents for post EOF, yet 805 * we need do like below. Otherwise, iomap itself will get 806 * into an endless loop on post EOF. 807 */ 808 if (iomap->offset >= inode->i_size) 809 iomap->length = length + map.m_la - offset; 810 } 811 iomap->flags = 0; 812 return 0; 813 } 814 815 const struct iomap_ops z_erofs_iomap_report_ops = { 816 .iomap_begin = z_erofs_iomap_begin_report, 817 }; 818