1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018-2019 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 */ 6 #include "internal.h" 7 #include <asm/unaligned.h> 8 #include <trace/events/erofs.h> 9 10 static int z_erofs_do_map_blocks(struct inode *inode, 11 struct erofs_map_blocks *map, 12 int flags); 13 14 int z_erofs_fill_inode(struct inode *inode) 15 { 16 struct erofs_inode *const vi = EROFS_I(inode); 17 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); 18 19 if (!erofs_sb_has_big_pcluster(sbi) && 20 !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) && 21 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { 22 vi->z_advise = 0; 23 vi->z_algorithmtype[0] = 0; 24 vi->z_algorithmtype[1] = 0; 25 vi->z_logical_clusterbits = LOG_BLOCK_SIZE; 26 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 27 } 28 inode->i_mapping->a_ops = &z_erofs_aops; 29 return 0; 30 } 31 32 static int z_erofs_fill_inode_lazy(struct inode *inode) 33 { 34 struct erofs_inode *const vi = EROFS_I(inode); 35 struct super_block *const sb = inode->i_sb; 36 int err, headnr; 37 erofs_off_t pos; 38 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 39 void *kaddr; 40 struct z_erofs_map_header *h; 41 42 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { 43 /* 44 * paired with smp_mb() at the end of the function to ensure 45 * fields will only be observed after the bit is set. 46 */ 47 smp_mb(); 48 return 0; 49 } 50 51 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) 52 return -ERESTARTSYS; 53 54 err = 0; 55 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) 56 goto out_unlock; 57 58 pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); 59 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos), EROFS_KMAP); 60 if (IS_ERR(kaddr)) { 61 err = PTR_ERR(kaddr); 62 goto out_unlock; 63 } 64 65 h = kaddr + erofs_blkoff(pos); 66 /* 67 * if the highest bit of the 8-byte map header is set, the whole file 68 * is stored in the packed inode. The rest bits keeps z_fragmentoff. 69 */ 70 if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { 71 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 72 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); 73 vi->z_tailextent_headlcn = 0; 74 goto done; 75 } 76 vi->z_advise = le16_to_cpu(h->h_advise); 77 vi->z_algorithmtype[0] = h->h_algorithmtype & 15; 78 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; 79 80 headnr = 0; 81 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX || 82 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) { 83 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", 84 headnr + 1, vi->z_algorithmtype[headnr], vi->nid); 85 err = -EOPNOTSUPP; 86 goto out_put_metabuf; 87 } 88 89 vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7); 90 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && 91 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 92 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 93 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", 94 vi->nid); 95 err = -EFSCORRUPTED; 96 goto out_put_metabuf; 97 } 98 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION && 99 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ 100 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 101 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", 102 vi->nid); 103 err = -EFSCORRUPTED; 104 goto out_put_metabuf; 105 } 106 107 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) { 108 struct erofs_map_blocks map = { 109 .buf = __EROFS_BUF_INITIALIZER 110 }; 111 112 vi->z_idata_size = le16_to_cpu(h->h_idata_size); 113 err = z_erofs_do_map_blocks(inode, &map, 114 EROFS_GET_BLOCKS_FINDTAIL); 115 erofs_put_metabuf(&map.buf); 116 117 if (!map.m_plen || 118 erofs_blkoff(map.m_pa) + map.m_plen > EROFS_BLKSIZ) { 119 erofs_err(sb, "invalid tail-packing pclustersize %llu", 120 map.m_plen); 121 err = -EFSCORRUPTED; 122 } 123 if (err < 0) 124 goto out_put_metabuf; 125 } 126 127 if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER && 128 !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) { 129 struct erofs_map_blocks map = { 130 .buf = __EROFS_BUF_INITIALIZER 131 }; 132 133 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); 134 err = z_erofs_do_map_blocks(inode, &map, 135 EROFS_GET_BLOCKS_FINDTAIL); 136 erofs_put_metabuf(&map.buf); 137 if (err < 0) 138 goto out_put_metabuf; 139 } 140 done: 141 /* paired with smp_mb() at the beginning of the function */ 142 smp_mb(); 143 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 144 out_put_metabuf: 145 erofs_put_metabuf(&buf); 146 out_unlock: 147 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); 148 return err; 149 } 150 151 struct z_erofs_maprecorder { 152 struct inode *inode; 153 struct erofs_map_blocks *map; 154 void *kaddr; 155 156 unsigned long lcn; 157 /* compression extent information gathered */ 158 u8 type, headtype; 159 u16 clusterofs; 160 u16 delta[2]; 161 erofs_blk_t pblk, compressedblks; 162 erofs_off_t nextpackoff; 163 bool partialref; 164 }; 165 166 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, 167 unsigned long lcn) 168 { 169 struct inode *const inode = m->inode; 170 struct erofs_inode *const vi = EROFS_I(inode); 171 const erofs_off_t pos = 172 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(erofs_iloc(inode) + 173 vi->inode_isize + vi->xattr_isize) + 174 lcn * sizeof(struct z_erofs_vle_decompressed_index); 175 struct z_erofs_vle_decompressed_index *di; 176 unsigned int advise, type; 177 178 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, 179 erofs_blknr(pos), EROFS_KMAP); 180 if (IS_ERR(m->kaddr)) 181 return PTR_ERR(m->kaddr); 182 183 m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index); 184 m->lcn = lcn; 185 di = m->kaddr + erofs_blkoff(pos); 186 187 advise = le16_to_cpu(di->di_advise); 188 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) & 189 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1); 190 switch (type) { 191 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 192 m->clusterofs = 1 << vi->z_logical_clusterbits; 193 m->delta[0] = le16_to_cpu(di->di_u.delta[0]); 194 if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) { 195 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 196 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 197 DBG_BUGON(1); 198 return -EFSCORRUPTED; 199 } 200 m->compressedblks = m->delta[0] & 201 ~Z_EROFS_VLE_DI_D0_CBLKCNT; 202 m->delta[0] = 1; 203 } 204 m->delta[1] = le16_to_cpu(di->di_u.delta[1]); 205 break; 206 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 207 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 208 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 209 if (advise & Z_EROFS_VLE_DI_PARTIAL_REF) 210 m->partialref = true; 211 m->clusterofs = le16_to_cpu(di->di_clusterofs); 212 m->pblk = le32_to_cpu(di->di_u.blkaddr); 213 break; 214 default: 215 DBG_BUGON(1); 216 return -EOPNOTSUPP; 217 } 218 m->type = type; 219 return 0; 220 } 221 222 static unsigned int decode_compactedbits(unsigned int lobits, 223 unsigned int lomask, 224 u8 *in, unsigned int pos, u8 *type) 225 { 226 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); 227 const unsigned int lo = v & lomask; 228 229 *type = (v >> lobits) & 3; 230 return lo; 231 } 232 233 static int get_compacted_la_distance(unsigned int lclusterbits, 234 unsigned int encodebits, 235 unsigned int vcnt, u8 *in, int i) 236 { 237 const unsigned int lomask = (1 << lclusterbits) - 1; 238 unsigned int lo, d1 = 0; 239 u8 type; 240 241 DBG_BUGON(i >= vcnt); 242 243 do { 244 lo = decode_compactedbits(lclusterbits, lomask, 245 in, encodebits * i, &type); 246 247 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 248 return d1; 249 ++d1; 250 } while (++i < vcnt); 251 252 /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */ 253 if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT)) 254 d1 += lo - 1; 255 return d1; 256 } 257 258 static int unpack_compacted_index(struct z_erofs_maprecorder *m, 259 unsigned int amortizedshift, 260 erofs_off_t pos, bool lookahead) 261 { 262 struct erofs_inode *const vi = EROFS_I(m->inode); 263 const unsigned int lclusterbits = vi->z_logical_clusterbits; 264 const unsigned int lomask = (1 << lclusterbits) - 1; 265 unsigned int vcnt, base, lo, encodebits, nblk, eofs; 266 int i; 267 u8 *in, type; 268 bool big_pcluster; 269 270 if (1 << amortizedshift == 4) 271 vcnt = 2; 272 else if (1 << amortizedshift == 2 && lclusterbits == 12) 273 vcnt = 16; 274 else 275 return -EOPNOTSUPP; 276 277 /* it doesn't equal to round_up(..) */ 278 m->nextpackoff = round_down(pos, vcnt << amortizedshift) + 279 (vcnt << amortizedshift); 280 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; 281 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; 282 eofs = erofs_blkoff(pos); 283 base = round_down(eofs, vcnt << amortizedshift); 284 in = m->kaddr + base; 285 286 i = (eofs - base) >> amortizedshift; 287 288 lo = decode_compactedbits(lclusterbits, lomask, 289 in, encodebits * i, &type); 290 m->type = type; 291 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 292 m->clusterofs = 1 << lclusterbits; 293 294 /* figure out lookahead_distance: delta[1] if needed */ 295 if (lookahead) 296 m->delta[1] = get_compacted_la_distance(lclusterbits, 297 encodebits, vcnt, in, i); 298 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { 299 if (!big_pcluster) { 300 DBG_BUGON(1); 301 return -EFSCORRUPTED; 302 } 303 m->compressedblks = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; 304 m->delta[0] = 1; 305 return 0; 306 } else if (i + 1 != (int)vcnt) { 307 m->delta[0] = lo; 308 return 0; 309 } 310 /* 311 * since the last lcluster in the pack is special, 312 * of which lo saves delta[1] rather than delta[0]. 313 * Hence, get delta[0] by the previous lcluster indirectly. 314 */ 315 lo = decode_compactedbits(lclusterbits, lomask, 316 in, encodebits * (i - 1), &type); 317 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 318 lo = 0; 319 else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) 320 lo = 1; 321 m->delta[0] = lo + 1; 322 return 0; 323 } 324 m->clusterofs = lo; 325 m->delta[0] = 0; 326 /* figout out blkaddr (pblk) for HEAD lclusters */ 327 if (!big_pcluster) { 328 nblk = 1; 329 while (i > 0) { 330 --i; 331 lo = decode_compactedbits(lclusterbits, lomask, 332 in, encodebits * i, &type); 333 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 334 i -= lo; 335 336 if (i >= 0) 337 ++nblk; 338 } 339 } else { 340 nblk = 0; 341 while (i > 0) { 342 --i; 343 lo = decode_compactedbits(lclusterbits, lomask, 344 in, encodebits * i, &type); 345 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 346 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { 347 --i; 348 nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; 349 continue; 350 } 351 /* bigpcluster shouldn't have plain d0 == 1 */ 352 if (lo <= 1) { 353 DBG_BUGON(1); 354 return -EFSCORRUPTED; 355 } 356 i -= lo - 2; 357 continue; 358 } 359 ++nblk; 360 } 361 } 362 in += (vcnt << amortizedshift) - sizeof(__le32); 363 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; 364 return 0; 365 } 366 367 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, 368 unsigned long lcn, bool lookahead) 369 { 370 struct inode *const inode = m->inode; 371 struct erofs_inode *const vi = EROFS_I(inode); 372 const unsigned int lclusterbits = vi->z_logical_clusterbits; 373 const erofs_off_t ebase = sizeof(struct z_erofs_map_header) + 374 ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); 375 const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); 376 unsigned int compacted_4b_initial, compacted_2b; 377 unsigned int amortizedshift; 378 erofs_off_t pos; 379 380 if (lclusterbits != 12) 381 return -EOPNOTSUPP; 382 383 if (lcn >= totalidx) 384 return -EINVAL; 385 386 m->lcn = lcn; 387 /* used to align to 32-byte (compacted_2b) alignment */ 388 compacted_4b_initial = (32 - ebase % 32) / 4; 389 if (compacted_4b_initial == 32 / 4) 390 compacted_4b_initial = 0; 391 392 if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && 393 compacted_4b_initial < totalidx) 394 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); 395 else 396 compacted_2b = 0; 397 398 pos = ebase; 399 if (lcn < compacted_4b_initial) { 400 amortizedshift = 2; 401 goto out; 402 } 403 pos += compacted_4b_initial * 4; 404 lcn -= compacted_4b_initial; 405 406 if (lcn < compacted_2b) { 407 amortizedshift = 1; 408 goto out; 409 } 410 pos += compacted_2b * 2; 411 lcn -= compacted_2b; 412 amortizedshift = 2; 413 out: 414 pos += lcn * (1 << amortizedshift); 415 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, 416 erofs_blknr(pos), EROFS_KMAP); 417 if (IS_ERR(m->kaddr)) 418 return PTR_ERR(m->kaddr); 419 return unpack_compacted_index(m, amortizedshift, pos, lookahead); 420 } 421 422 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, 423 unsigned int lcn, bool lookahead) 424 { 425 const unsigned int datamode = EROFS_I(m->inode)->datalayout; 426 427 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY) 428 return legacy_load_cluster_from_disk(m, lcn); 429 430 if (datamode == EROFS_INODE_FLAT_COMPRESSION) 431 return compacted_load_cluster_from_disk(m, lcn, lookahead); 432 433 return -EINVAL; 434 } 435 436 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, 437 unsigned int lookback_distance) 438 { 439 struct erofs_inode *const vi = EROFS_I(m->inode); 440 const unsigned int lclusterbits = vi->z_logical_clusterbits; 441 442 while (m->lcn >= lookback_distance) { 443 unsigned long lcn = m->lcn - lookback_distance; 444 int err; 445 446 /* load extent head logical cluster if needed */ 447 err = z_erofs_load_cluster_from_disk(m, lcn, false); 448 if (err) 449 return err; 450 451 switch (m->type) { 452 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 453 if (!m->delta[0]) { 454 erofs_err(m->inode->i_sb, 455 "invalid lookback distance 0 @ nid %llu", 456 vi->nid); 457 DBG_BUGON(1); 458 return -EFSCORRUPTED; 459 } 460 lookback_distance = m->delta[0]; 461 continue; 462 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 463 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 464 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 465 m->headtype = m->type; 466 m->map->m_la = (lcn << lclusterbits) | m->clusterofs; 467 return 0; 468 default: 469 erofs_err(m->inode->i_sb, 470 "unknown type %u @ lcn %lu of nid %llu", 471 m->type, lcn, vi->nid); 472 DBG_BUGON(1); 473 return -EOPNOTSUPP; 474 } 475 } 476 477 erofs_err(m->inode->i_sb, "bogus lookback distance @ nid %llu", 478 vi->nid); 479 DBG_BUGON(1); 480 return -EFSCORRUPTED; 481 } 482 483 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, 484 unsigned int initial_lcn) 485 { 486 struct erofs_inode *const vi = EROFS_I(m->inode); 487 struct erofs_map_blocks *const map = m->map; 488 const unsigned int lclusterbits = vi->z_logical_clusterbits; 489 unsigned long lcn; 490 int err; 491 492 DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN && 493 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 && 494 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD2); 495 DBG_BUGON(m->type != m->headtype); 496 497 if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || 498 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1) && 499 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) || 500 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) && 501 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 502 map->m_plen = 1ULL << lclusterbits; 503 return 0; 504 } 505 lcn = m->lcn + 1; 506 if (m->compressedblks) 507 goto out; 508 509 err = z_erofs_load_cluster_from_disk(m, lcn, false); 510 if (err) 511 return err; 512 513 /* 514 * If the 1st NONHEAD lcluster has already been handled initially w/o 515 * valid compressedblks, which means at least it mustn't be CBLKCNT, or 516 * an internal implemenatation error is detected. 517 * 518 * The following code can also handle it properly anyway, but let's 519 * BUG_ON in the debugging mode only for developers to notice that. 520 */ 521 DBG_BUGON(lcn == initial_lcn && 522 m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD); 523 524 switch (m->type) { 525 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 526 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 527 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 528 /* 529 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type 530 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. 531 */ 532 m->compressedblks = 1 << (lclusterbits - LOG_BLOCK_SIZE); 533 break; 534 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 535 if (m->delta[0] != 1) 536 goto err_bonus_cblkcnt; 537 if (m->compressedblks) 538 break; 539 fallthrough; 540 default: 541 erofs_err(m->inode->i_sb, 542 "cannot found CBLKCNT @ lcn %lu of nid %llu", 543 lcn, vi->nid); 544 DBG_BUGON(1); 545 return -EFSCORRUPTED; 546 } 547 out: 548 map->m_plen = (u64)m->compressedblks << LOG_BLOCK_SIZE; 549 return 0; 550 err_bonus_cblkcnt: 551 erofs_err(m->inode->i_sb, 552 "bogus CBLKCNT @ lcn %lu of nid %llu", 553 lcn, vi->nid); 554 DBG_BUGON(1); 555 return -EFSCORRUPTED; 556 } 557 558 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) 559 { 560 struct inode *inode = m->inode; 561 struct erofs_inode *vi = EROFS_I(inode); 562 struct erofs_map_blocks *map = m->map; 563 unsigned int lclusterbits = vi->z_logical_clusterbits; 564 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits; 565 int err; 566 567 do { 568 /* handle the last EOF pcluster (no next HEAD lcluster) */ 569 if ((lcn << lclusterbits) >= inode->i_size) { 570 map->m_llen = inode->i_size - map->m_la; 571 return 0; 572 } 573 574 err = z_erofs_load_cluster_from_disk(m, lcn, true); 575 if (err) 576 return err; 577 578 if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 579 DBG_BUGON(!m->delta[1] && 580 m->clusterofs != 1 << lclusterbits); 581 } else if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || 582 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 || 583 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) { 584 /* go on until the next HEAD lcluster */ 585 if (lcn != headlcn) 586 break; 587 m->delta[1] = 1; 588 } else { 589 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", 590 m->type, lcn, vi->nid); 591 DBG_BUGON(1); 592 return -EOPNOTSUPP; 593 } 594 lcn += m->delta[1]; 595 } while (m->delta[1]); 596 597 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la; 598 return 0; 599 } 600 601 static int z_erofs_do_map_blocks(struct inode *inode, 602 struct erofs_map_blocks *map, 603 int flags) 604 { 605 struct erofs_inode *const vi = EROFS_I(inode); 606 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; 607 bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 608 struct z_erofs_maprecorder m = { 609 .inode = inode, 610 .map = map, 611 }; 612 int err = 0; 613 unsigned int lclusterbits, endoff; 614 unsigned long initial_lcn; 615 unsigned long long ofs, end; 616 617 lclusterbits = vi->z_logical_clusterbits; 618 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la; 619 initial_lcn = ofs >> lclusterbits; 620 endoff = ofs & ((1 << lclusterbits) - 1); 621 622 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false); 623 if (err) 624 goto unmap_out; 625 626 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL)) 627 vi->z_idataoff = m.nextpackoff; 628 629 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; 630 end = (m.lcn + 1ULL) << lclusterbits; 631 632 switch (m.type) { 633 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 634 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 635 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 636 if (endoff >= m.clusterofs) { 637 m.headtype = m.type; 638 map->m_la = (m.lcn << lclusterbits) | m.clusterofs; 639 /* 640 * For ztailpacking files, in order to inline data more 641 * effectively, special EOF lclusters are now supported 642 * which can have three parts at most. 643 */ 644 if (ztailpacking && end > inode->i_size) 645 end = inode->i_size; 646 break; 647 } 648 /* m.lcn should be >= 1 if endoff < m.clusterofs */ 649 if (!m.lcn) { 650 erofs_err(inode->i_sb, 651 "invalid logical cluster 0 at nid %llu", 652 vi->nid); 653 err = -EFSCORRUPTED; 654 goto unmap_out; 655 } 656 end = (m.lcn << lclusterbits) | m.clusterofs; 657 map->m_flags |= EROFS_MAP_FULL_MAPPED; 658 m.delta[0] = 1; 659 fallthrough; 660 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 661 /* get the corresponding first chunk */ 662 err = z_erofs_extent_lookback(&m, m.delta[0]); 663 if (err) 664 goto unmap_out; 665 break; 666 default: 667 erofs_err(inode->i_sb, 668 "unknown type %u @ offset %llu of nid %llu", 669 m.type, ofs, vi->nid); 670 err = -EOPNOTSUPP; 671 goto unmap_out; 672 } 673 if (m.partialref) 674 map->m_flags |= EROFS_MAP_PARTIAL_REF; 675 map->m_llen = end - map->m_la; 676 677 if (flags & EROFS_GET_BLOCKS_FINDTAIL) { 678 vi->z_tailextent_headlcn = m.lcn; 679 /* for non-compact indexes, fragmentoff is 64 bits */ 680 if (fragment && 681 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) 682 vi->z_fragmentoff |= (u64)m.pblk << 32; 683 } 684 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { 685 map->m_flags |= EROFS_MAP_META; 686 map->m_pa = vi->z_idataoff; 687 map->m_plen = vi->z_idata_size; 688 } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { 689 map->m_flags |= EROFS_MAP_FRAGMENT; 690 } else { 691 map->m_pa = blknr_to_addr(m.pblk); 692 err = z_erofs_get_extent_compressedlen(&m, initial_lcn); 693 if (err) 694 goto unmap_out; 695 } 696 697 if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) { 698 if (map->m_llen > map->m_plen) { 699 DBG_BUGON(1); 700 err = -EFSCORRUPTED; 701 goto unmap_out; 702 } 703 if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) 704 map->m_algorithmformat = 705 Z_EROFS_COMPRESSION_INTERLACED; 706 else 707 map->m_algorithmformat = 708 Z_EROFS_COMPRESSION_SHIFTED; 709 } else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) { 710 map->m_algorithmformat = vi->z_algorithmtype[1]; 711 } else { 712 map->m_algorithmformat = vi->z_algorithmtype[0]; 713 } 714 715 if ((flags & EROFS_GET_BLOCKS_FIEMAP) || 716 ((flags & EROFS_GET_BLOCKS_READMORE) && 717 map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA && 718 map->m_llen >= EROFS_BLKSIZ)) { 719 err = z_erofs_get_extent_decompressedlen(&m); 720 if (!err) 721 map->m_flags |= EROFS_MAP_FULL_MAPPED; 722 } 723 724 unmap_out: 725 erofs_unmap_metabuf(&m.map->buf); 726 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o", 727 __func__, map->m_la, map->m_pa, 728 map->m_llen, map->m_plen, map->m_flags); 729 return err; 730 } 731 732 int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, 733 int flags) 734 { 735 struct erofs_inode *const vi = EROFS_I(inode); 736 int err = 0; 737 738 trace_z_erofs_map_blocks_iter_enter(inode, map, flags); 739 740 /* when trying to read beyond EOF, leave it unmapped */ 741 if (map->m_la >= inode->i_size) { 742 map->m_llen = map->m_la + 1 - inode->i_size; 743 map->m_la = inode->i_size; 744 map->m_flags = 0; 745 goto out; 746 } 747 748 err = z_erofs_fill_inode_lazy(inode); 749 if (err) 750 goto out; 751 752 if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) && 753 !vi->z_tailextent_headlcn) { 754 map->m_la = 0; 755 map->m_llen = inode->i_size; 756 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | 757 EROFS_MAP_FRAGMENT; 758 goto out; 759 } 760 761 err = z_erofs_do_map_blocks(inode, map, flags); 762 out: 763 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); 764 765 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */ 766 DBG_BUGON(err < 0 && err != -ENOMEM); 767 return err; 768 } 769 770 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, 771 loff_t length, unsigned int flags, 772 struct iomap *iomap, struct iomap *srcmap) 773 { 774 int ret; 775 struct erofs_map_blocks map = { .m_la = offset }; 776 777 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP); 778 erofs_put_metabuf(&map.buf); 779 if (ret < 0) 780 return ret; 781 782 iomap->bdev = inode->i_sb->s_bdev; 783 iomap->offset = map.m_la; 784 iomap->length = map.m_llen; 785 if (map.m_flags & EROFS_MAP_MAPPED) { 786 iomap->type = IOMAP_MAPPED; 787 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ? 788 IOMAP_NULL_ADDR : map.m_pa; 789 } else { 790 iomap->type = IOMAP_HOLE; 791 iomap->addr = IOMAP_NULL_ADDR; 792 /* 793 * No strict rule on how to describe extents for post EOF, yet 794 * we need to do like below. Otherwise, iomap itself will get 795 * into an endless loop on post EOF. 796 * 797 * Calculate the effective offset by subtracting extent start 798 * (map.m_la) from the requested offset, and add it to length. 799 * (NB: offset >= map.m_la always) 800 */ 801 if (iomap->offset >= inode->i_size) 802 iomap->length = length + offset - map.m_la; 803 } 804 iomap->flags = 0; 805 return 0; 806 } 807 808 const struct iomap_ops z_erofs_iomap_report_ops = { 809 .iomap_begin = z_erofs_iomap_begin_report, 810 }; 811