1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018-2019 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 */ 6 #include "internal.h" 7 #include <asm/unaligned.h> 8 #include <trace/events/erofs.h> 9 10 struct z_erofs_maprecorder { 11 struct inode *inode; 12 struct erofs_map_blocks *map; 13 void *kaddr; 14 15 unsigned long lcn; 16 /* compression extent information gathered */ 17 u8 type, headtype; 18 u16 clusterofs; 19 u16 delta[2]; 20 erofs_blk_t pblk, compressedblks; 21 erofs_off_t nextpackoff; 22 bool partialref; 23 }; 24 25 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, 26 unsigned long lcn) 27 { 28 struct inode *const inode = m->inode; 29 struct erofs_inode *const vi = EROFS_I(inode); 30 const erofs_off_t pos = Z_EROFS_FULL_INDEX_ALIGN(erofs_iloc(inode) + 31 vi->inode_isize + vi->xattr_isize) + 32 lcn * sizeof(struct z_erofs_lcluster_index); 33 struct z_erofs_lcluster_index *di; 34 unsigned int advise, type; 35 36 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, 37 erofs_blknr(inode->i_sb, pos), EROFS_KMAP); 38 if (IS_ERR(m->kaddr)) 39 return PTR_ERR(m->kaddr); 40 41 m->nextpackoff = pos + sizeof(struct z_erofs_lcluster_index); 42 m->lcn = lcn; 43 di = m->kaddr + erofs_blkoff(inode->i_sb, pos); 44 45 advise = le16_to_cpu(di->di_advise); 46 type = (advise >> Z_EROFS_LI_LCLUSTER_TYPE_BIT) & 47 ((1 << Z_EROFS_LI_LCLUSTER_TYPE_BITS) - 1); 48 switch (type) { 49 case Z_EROFS_LCLUSTER_TYPE_NONHEAD: 50 m->clusterofs = 1 << vi->z_logical_clusterbits; 51 m->delta[0] = le16_to_cpu(di->di_u.delta[0]); 52 if (m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) { 53 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 54 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 55 DBG_BUGON(1); 56 return -EFSCORRUPTED; 57 } 58 m->compressedblks = m->delta[0] & 59 ~Z_EROFS_LI_D0_CBLKCNT; 60 m->delta[0] = 1; 61 } 62 m->delta[1] = le16_to_cpu(di->di_u.delta[1]); 63 break; 64 case Z_EROFS_LCLUSTER_TYPE_PLAIN: 65 case Z_EROFS_LCLUSTER_TYPE_HEAD1: 66 case Z_EROFS_LCLUSTER_TYPE_HEAD2: 67 if (advise & Z_EROFS_LI_PARTIAL_REF) 68 m->partialref = true; 69 m->clusterofs = le16_to_cpu(di->di_clusterofs); 70 if (m->clusterofs >= 1 << vi->z_logical_clusterbits) { 71 DBG_BUGON(1); 72 return -EFSCORRUPTED; 73 } 74 m->pblk = le32_to_cpu(di->di_u.blkaddr); 75 break; 76 default: 77 DBG_BUGON(1); 78 return -EOPNOTSUPP; 79 } 80 m->type = type; 81 return 0; 82 } 83 84 static unsigned int decode_compactedbits(unsigned int lobits, 85 unsigned int lomask, 86 u8 *in, unsigned int pos, u8 *type) 87 { 88 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); 89 const unsigned int lo = v & lomask; 90 91 *type = (v >> lobits) & 3; 92 return lo; 93 } 94 95 static int get_compacted_la_distance(unsigned int lclusterbits, 96 unsigned int encodebits, 97 unsigned int vcnt, u8 *in, int i) 98 { 99 const unsigned int lomask = (1 << lclusterbits) - 1; 100 unsigned int lo, d1 = 0; 101 u8 type; 102 103 DBG_BUGON(i >= vcnt); 104 105 do { 106 lo = decode_compactedbits(lclusterbits, lomask, 107 in, encodebits * i, &type); 108 109 if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) 110 return d1; 111 ++d1; 112 } while (++i < vcnt); 113 114 /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */ 115 if (!(lo & Z_EROFS_LI_D0_CBLKCNT)) 116 d1 += lo - 1; 117 return d1; 118 } 119 120 static int unpack_compacted_index(struct z_erofs_maprecorder *m, 121 unsigned int amortizedshift, 122 erofs_off_t pos, bool lookahead) 123 { 124 struct erofs_inode *const vi = EROFS_I(m->inode); 125 const unsigned int lclusterbits = vi->z_logical_clusterbits; 126 const unsigned int lomask = (1 << lclusterbits) - 1; 127 unsigned int vcnt, base, lo, encodebits, nblk, eofs; 128 int i; 129 u8 *in, type; 130 bool big_pcluster; 131 132 if (1 << amortizedshift == 4) 133 vcnt = 2; 134 else if (1 << amortizedshift == 2 && lclusterbits == 12) 135 vcnt = 16; 136 else 137 return -EOPNOTSUPP; 138 139 /* it doesn't equal to round_up(..) */ 140 m->nextpackoff = round_down(pos, vcnt << amortizedshift) + 141 (vcnt << amortizedshift); 142 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; 143 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; 144 eofs = erofs_blkoff(m->inode->i_sb, pos); 145 base = round_down(eofs, vcnt << amortizedshift); 146 in = m->kaddr + base; 147 148 i = (eofs - base) >> amortizedshift; 149 150 lo = decode_compactedbits(lclusterbits, lomask, 151 in, encodebits * i, &type); 152 m->type = type; 153 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { 154 m->clusterofs = 1 << lclusterbits; 155 156 /* figure out lookahead_distance: delta[1] if needed */ 157 if (lookahead) 158 m->delta[1] = get_compacted_la_distance(lclusterbits, 159 encodebits, vcnt, in, i); 160 if (lo & Z_EROFS_LI_D0_CBLKCNT) { 161 if (!big_pcluster) { 162 DBG_BUGON(1); 163 return -EFSCORRUPTED; 164 } 165 m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; 166 m->delta[0] = 1; 167 return 0; 168 } else if (i + 1 != (int)vcnt) { 169 m->delta[0] = lo; 170 return 0; 171 } 172 /* 173 * since the last lcluster in the pack is special, 174 * of which lo saves delta[1] rather than delta[0]. 175 * Hence, get delta[0] by the previous lcluster indirectly. 176 */ 177 lo = decode_compactedbits(lclusterbits, lomask, 178 in, encodebits * (i - 1), &type); 179 if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) 180 lo = 0; 181 else if (lo & Z_EROFS_LI_D0_CBLKCNT) 182 lo = 1; 183 m->delta[0] = lo + 1; 184 return 0; 185 } 186 m->clusterofs = lo; 187 m->delta[0] = 0; 188 /* figout out blkaddr (pblk) for HEAD lclusters */ 189 if (!big_pcluster) { 190 nblk = 1; 191 while (i > 0) { 192 --i; 193 lo = decode_compactedbits(lclusterbits, lomask, 194 in, encodebits * i, &type); 195 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) 196 i -= lo; 197 198 if (i >= 0) 199 ++nblk; 200 } 201 } else { 202 nblk = 0; 203 while (i > 0) { 204 --i; 205 lo = decode_compactedbits(lclusterbits, lomask, 206 in, encodebits * i, &type); 207 if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { 208 if (lo & Z_EROFS_LI_D0_CBLKCNT) { 209 --i; 210 nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; 211 continue; 212 } 213 /* bigpcluster shouldn't have plain d0 == 1 */ 214 if (lo <= 1) { 215 DBG_BUGON(1); 216 return -EFSCORRUPTED; 217 } 218 i -= lo - 2; 219 continue; 220 } 221 ++nblk; 222 } 223 } 224 in += (vcnt << amortizedshift) - sizeof(__le32); 225 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; 226 return 0; 227 } 228 229 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, 230 unsigned long lcn, bool lookahead) 231 { 232 struct inode *const inode = m->inode; 233 struct erofs_inode *const vi = EROFS_I(inode); 234 const unsigned int lclusterbits = vi->z_logical_clusterbits; 235 const erofs_off_t ebase = sizeof(struct z_erofs_map_header) + 236 ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); 237 unsigned int totalidx = erofs_iblks(inode); 238 unsigned int compacted_4b_initial, compacted_2b; 239 unsigned int amortizedshift; 240 erofs_off_t pos; 241 242 if (lclusterbits != 12) 243 return -EOPNOTSUPP; 244 245 if (lcn >= totalidx) 246 return -EINVAL; 247 248 m->lcn = lcn; 249 /* used to align to 32-byte (compacted_2b) alignment */ 250 compacted_4b_initial = (32 - ebase % 32) / 4; 251 if (compacted_4b_initial == 32 / 4) 252 compacted_4b_initial = 0; 253 254 if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && 255 compacted_4b_initial < totalidx) 256 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); 257 else 258 compacted_2b = 0; 259 260 pos = ebase; 261 if (lcn < compacted_4b_initial) { 262 amortizedshift = 2; 263 goto out; 264 } 265 pos += compacted_4b_initial * 4; 266 lcn -= compacted_4b_initial; 267 268 if (lcn < compacted_2b) { 269 amortizedshift = 1; 270 goto out; 271 } 272 pos += compacted_2b * 2; 273 lcn -= compacted_2b; 274 amortizedshift = 2; 275 out: 276 pos += lcn * (1 << amortizedshift); 277 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, 278 erofs_blknr(inode->i_sb, pos), EROFS_KMAP); 279 if (IS_ERR(m->kaddr)) 280 return PTR_ERR(m->kaddr); 281 return unpack_compacted_index(m, amortizedshift, pos, lookahead); 282 } 283 284 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, 285 unsigned int lcn, bool lookahead) 286 { 287 const unsigned int datamode = EROFS_I(m->inode)->datalayout; 288 289 if (datamode == EROFS_INODE_COMPRESSED_FULL) 290 return legacy_load_cluster_from_disk(m, lcn); 291 292 if (datamode == EROFS_INODE_COMPRESSED_COMPACT) 293 return compacted_load_cluster_from_disk(m, lcn, lookahead); 294 295 return -EINVAL; 296 } 297 298 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, 299 unsigned int lookback_distance) 300 { 301 struct erofs_inode *const vi = EROFS_I(m->inode); 302 const unsigned int lclusterbits = vi->z_logical_clusterbits; 303 304 while (m->lcn >= lookback_distance) { 305 unsigned long lcn = m->lcn - lookback_distance; 306 int err; 307 308 /* load extent head logical cluster if needed */ 309 err = z_erofs_load_cluster_from_disk(m, lcn, false); 310 if (err) 311 return err; 312 313 switch (m->type) { 314 case Z_EROFS_LCLUSTER_TYPE_NONHEAD: 315 if (!m->delta[0]) { 316 erofs_err(m->inode->i_sb, 317 "invalid lookback distance 0 @ nid %llu", 318 vi->nid); 319 DBG_BUGON(1); 320 return -EFSCORRUPTED; 321 } 322 lookback_distance = m->delta[0]; 323 continue; 324 case Z_EROFS_LCLUSTER_TYPE_PLAIN: 325 case Z_EROFS_LCLUSTER_TYPE_HEAD1: 326 case Z_EROFS_LCLUSTER_TYPE_HEAD2: 327 m->headtype = m->type; 328 m->map->m_la = (lcn << lclusterbits) | m->clusterofs; 329 return 0; 330 default: 331 erofs_err(m->inode->i_sb, 332 "unknown type %u @ lcn %lu of nid %llu", 333 m->type, lcn, vi->nid); 334 DBG_BUGON(1); 335 return -EOPNOTSUPP; 336 } 337 } 338 339 erofs_err(m->inode->i_sb, "bogus lookback distance @ nid %llu", 340 vi->nid); 341 DBG_BUGON(1); 342 return -EFSCORRUPTED; 343 } 344 345 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, 346 unsigned int initial_lcn) 347 { 348 struct super_block *sb = m->inode->i_sb; 349 struct erofs_inode *const vi = EROFS_I(m->inode); 350 struct erofs_map_blocks *const map = m->map; 351 const unsigned int lclusterbits = vi->z_logical_clusterbits; 352 unsigned long lcn; 353 int err; 354 355 DBG_BUGON(m->type != Z_EROFS_LCLUSTER_TYPE_PLAIN && 356 m->type != Z_EROFS_LCLUSTER_TYPE_HEAD1 && 357 m->type != Z_EROFS_LCLUSTER_TYPE_HEAD2); 358 DBG_BUGON(m->type != m->headtype); 359 360 if (m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || 361 ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1) && 362 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) || 363 ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && 364 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 365 map->m_plen = 1ULL << lclusterbits; 366 return 0; 367 } 368 lcn = m->lcn + 1; 369 if (m->compressedblks) 370 goto out; 371 372 err = z_erofs_load_cluster_from_disk(m, lcn, false); 373 if (err) 374 return err; 375 376 /* 377 * If the 1st NONHEAD lcluster has already been handled initially w/o 378 * valid compressedblks, which means at least it mustn't be CBLKCNT, or 379 * an internal implemenatation error is detected. 380 * 381 * The following code can also handle it properly anyway, but let's 382 * BUG_ON in the debugging mode only for developers to notice that. 383 */ 384 DBG_BUGON(lcn == initial_lcn && 385 m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); 386 387 switch (m->type) { 388 case Z_EROFS_LCLUSTER_TYPE_PLAIN: 389 case Z_EROFS_LCLUSTER_TYPE_HEAD1: 390 case Z_EROFS_LCLUSTER_TYPE_HEAD2: 391 /* 392 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type 393 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. 394 */ 395 m->compressedblks = 1 << (lclusterbits - sb->s_blocksize_bits); 396 break; 397 case Z_EROFS_LCLUSTER_TYPE_NONHEAD: 398 if (m->delta[0] != 1) 399 goto err_bonus_cblkcnt; 400 if (m->compressedblks) 401 break; 402 fallthrough; 403 default: 404 erofs_err(m->inode->i_sb, 405 "cannot found CBLKCNT @ lcn %lu of nid %llu", 406 lcn, vi->nid); 407 DBG_BUGON(1); 408 return -EFSCORRUPTED; 409 } 410 out: 411 map->m_plen = erofs_pos(sb, m->compressedblks); 412 return 0; 413 err_bonus_cblkcnt: 414 erofs_err(m->inode->i_sb, 415 "bogus CBLKCNT @ lcn %lu of nid %llu", 416 lcn, vi->nid); 417 DBG_BUGON(1); 418 return -EFSCORRUPTED; 419 } 420 421 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) 422 { 423 struct inode *inode = m->inode; 424 struct erofs_inode *vi = EROFS_I(inode); 425 struct erofs_map_blocks *map = m->map; 426 unsigned int lclusterbits = vi->z_logical_clusterbits; 427 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits; 428 int err; 429 430 do { 431 /* handle the last EOF pcluster (no next HEAD lcluster) */ 432 if ((lcn << lclusterbits) >= inode->i_size) { 433 map->m_llen = inode->i_size - map->m_la; 434 return 0; 435 } 436 437 err = z_erofs_load_cluster_from_disk(m, lcn, true); 438 if (err) 439 return err; 440 441 if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { 442 DBG_BUGON(!m->delta[1] && 443 m->clusterofs != 1 << lclusterbits); 444 } else if (m->type == Z_EROFS_LCLUSTER_TYPE_PLAIN || 445 m->type == Z_EROFS_LCLUSTER_TYPE_HEAD1 || 446 m->type == Z_EROFS_LCLUSTER_TYPE_HEAD2) { 447 /* go on until the next HEAD lcluster */ 448 if (lcn != headlcn) 449 break; 450 m->delta[1] = 1; 451 } else { 452 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", 453 m->type, lcn, vi->nid); 454 DBG_BUGON(1); 455 return -EOPNOTSUPP; 456 } 457 lcn += m->delta[1]; 458 } while (m->delta[1]); 459 460 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la; 461 return 0; 462 } 463 464 static int z_erofs_do_map_blocks(struct inode *inode, 465 struct erofs_map_blocks *map, int flags) 466 { 467 struct erofs_inode *const vi = EROFS_I(inode); 468 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; 469 bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 470 struct z_erofs_maprecorder m = { 471 .inode = inode, 472 .map = map, 473 }; 474 int err = 0; 475 unsigned int lclusterbits, endoff; 476 unsigned long initial_lcn; 477 unsigned long long ofs, end; 478 479 lclusterbits = vi->z_logical_clusterbits; 480 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la; 481 initial_lcn = ofs >> lclusterbits; 482 endoff = ofs & ((1 << lclusterbits) - 1); 483 484 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false); 485 if (err) 486 goto unmap_out; 487 488 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL)) 489 vi->z_idataoff = m.nextpackoff; 490 491 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; 492 end = (m.lcn + 1ULL) << lclusterbits; 493 494 switch (m.type) { 495 case Z_EROFS_LCLUSTER_TYPE_PLAIN: 496 case Z_EROFS_LCLUSTER_TYPE_HEAD1: 497 case Z_EROFS_LCLUSTER_TYPE_HEAD2: 498 if (endoff >= m.clusterofs) { 499 m.headtype = m.type; 500 map->m_la = (m.lcn << lclusterbits) | m.clusterofs; 501 /* 502 * For ztailpacking files, in order to inline data more 503 * effectively, special EOF lclusters are now supported 504 * which can have three parts at most. 505 */ 506 if (ztailpacking && end > inode->i_size) 507 end = inode->i_size; 508 break; 509 } 510 /* m.lcn should be >= 1 if endoff < m.clusterofs */ 511 if (!m.lcn) { 512 erofs_err(inode->i_sb, 513 "invalid logical cluster 0 at nid %llu", 514 vi->nid); 515 err = -EFSCORRUPTED; 516 goto unmap_out; 517 } 518 end = (m.lcn << lclusterbits) | m.clusterofs; 519 map->m_flags |= EROFS_MAP_FULL_MAPPED; 520 m.delta[0] = 1; 521 fallthrough; 522 case Z_EROFS_LCLUSTER_TYPE_NONHEAD: 523 /* get the corresponding first chunk */ 524 err = z_erofs_extent_lookback(&m, m.delta[0]); 525 if (err) 526 goto unmap_out; 527 break; 528 default: 529 erofs_err(inode->i_sb, 530 "unknown type %u @ offset %llu of nid %llu", 531 m.type, ofs, vi->nid); 532 err = -EOPNOTSUPP; 533 goto unmap_out; 534 } 535 if (m.partialref) 536 map->m_flags |= EROFS_MAP_PARTIAL_REF; 537 map->m_llen = end - map->m_la; 538 539 if (flags & EROFS_GET_BLOCKS_FINDTAIL) { 540 vi->z_tailextent_headlcn = m.lcn; 541 /* for non-compact indexes, fragmentoff is 64 bits */ 542 if (fragment && 543 vi->datalayout == EROFS_INODE_COMPRESSED_FULL) 544 vi->z_fragmentoff |= (u64)m.pblk << 32; 545 } 546 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { 547 map->m_flags |= EROFS_MAP_META; 548 map->m_pa = vi->z_idataoff; 549 map->m_plen = vi->z_idata_size; 550 } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { 551 map->m_flags |= EROFS_MAP_FRAGMENT; 552 } else { 553 map->m_pa = erofs_pos(inode->i_sb, m.pblk); 554 err = z_erofs_get_extent_compressedlen(&m, initial_lcn); 555 if (err) 556 goto unmap_out; 557 } 558 559 if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { 560 if (map->m_llen > map->m_plen) { 561 DBG_BUGON(1); 562 err = -EFSCORRUPTED; 563 goto unmap_out; 564 } 565 if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) 566 map->m_algorithmformat = 567 Z_EROFS_COMPRESSION_INTERLACED; 568 else 569 map->m_algorithmformat = 570 Z_EROFS_COMPRESSION_SHIFTED; 571 } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { 572 map->m_algorithmformat = vi->z_algorithmtype[1]; 573 } else { 574 map->m_algorithmformat = vi->z_algorithmtype[0]; 575 } 576 577 if ((flags & EROFS_GET_BLOCKS_FIEMAP) || 578 ((flags & EROFS_GET_BLOCKS_READMORE) && 579 map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA && 580 map->m_llen >= i_blocksize(inode))) { 581 err = z_erofs_get_extent_decompressedlen(&m); 582 if (!err) 583 map->m_flags |= EROFS_MAP_FULL_MAPPED; 584 } 585 586 unmap_out: 587 erofs_unmap_metabuf(&m.map->buf); 588 return err; 589 } 590 591 static int z_erofs_fill_inode_lazy(struct inode *inode) 592 { 593 struct erofs_inode *const vi = EROFS_I(inode); 594 struct super_block *const sb = inode->i_sb; 595 int err, headnr; 596 erofs_off_t pos; 597 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 598 void *kaddr; 599 struct z_erofs_map_header *h; 600 601 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { 602 /* 603 * paired with smp_mb() at the end of the function to ensure 604 * fields will only be observed after the bit is set. 605 */ 606 smp_mb(); 607 return 0; 608 } 609 610 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) 611 return -ERESTARTSYS; 612 613 err = 0; 614 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) 615 goto out_unlock; 616 617 pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); 618 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(sb, pos), EROFS_KMAP); 619 if (IS_ERR(kaddr)) { 620 err = PTR_ERR(kaddr); 621 goto out_unlock; 622 } 623 624 h = kaddr + erofs_blkoff(sb, pos); 625 /* 626 * if the highest bit of the 8-byte map header is set, the whole file 627 * is stored in the packed inode. The rest bits keeps z_fragmentoff. 628 */ 629 if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { 630 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 631 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); 632 vi->z_tailextent_headlcn = 0; 633 goto done; 634 } 635 vi->z_advise = le16_to_cpu(h->h_advise); 636 vi->z_algorithmtype[0] = h->h_algorithmtype & 15; 637 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; 638 639 headnr = 0; 640 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX || 641 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) { 642 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", 643 headnr + 1, vi->z_algorithmtype[headnr], vi->nid); 644 err = -EOPNOTSUPP; 645 goto out_put_metabuf; 646 } 647 648 vi->z_logical_clusterbits = sb->s_blocksize_bits + (h->h_clusterbits & 7); 649 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && 650 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 651 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 652 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", 653 vi->nid); 654 err = -EFSCORRUPTED; 655 goto out_put_metabuf; 656 } 657 if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && 658 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ 659 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 660 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", 661 vi->nid); 662 err = -EFSCORRUPTED; 663 goto out_put_metabuf; 664 } 665 666 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) { 667 struct erofs_map_blocks map = { 668 .buf = __EROFS_BUF_INITIALIZER 669 }; 670 671 vi->z_idata_size = le16_to_cpu(h->h_idata_size); 672 err = z_erofs_do_map_blocks(inode, &map, 673 EROFS_GET_BLOCKS_FINDTAIL); 674 erofs_put_metabuf(&map.buf); 675 676 if (!map.m_plen || 677 erofs_blkoff(sb, map.m_pa) + map.m_plen > sb->s_blocksize) { 678 erofs_err(sb, "invalid tail-packing pclustersize %llu", 679 map.m_plen); 680 err = -EFSCORRUPTED; 681 } 682 if (err < 0) 683 goto out_put_metabuf; 684 } 685 686 if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER && 687 !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) { 688 struct erofs_map_blocks map = { 689 .buf = __EROFS_BUF_INITIALIZER 690 }; 691 692 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); 693 err = z_erofs_do_map_blocks(inode, &map, 694 EROFS_GET_BLOCKS_FINDTAIL); 695 erofs_put_metabuf(&map.buf); 696 if (err < 0) 697 goto out_put_metabuf; 698 } 699 done: 700 /* paired with smp_mb() at the beginning of the function */ 701 smp_mb(); 702 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 703 out_put_metabuf: 704 erofs_put_metabuf(&buf); 705 out_unlock: 706 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); 707 return err; 708 } 709 710 int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, 711 int flags) 712 { 713 struct erofs_inode *const vi = EROFS_I(inode); 714 int err = 0; 715 716 trace_z_erofs_map_blocks_iter_enter(inode, map, flags); 717 718 /* when trying to read beyond EOF, leave it unmapped */ 719 if (map->m_la >= inode->i_size) { 720 map->m_llen = map->m_la + 1 - inode->i_size; 721 map->m_la = inode->i_size; 722 map->m_flags = 0; 723 goto out; 724 } 725 726 err = z_erofs_fill_inode_lazy(inode); 727 if (err) 728 goto out; 729 730 if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) && 731 !vi->z_tailextent_headlcn) { 732 map->m_la = 0; 733 map->m_llen = inode->i_size; 734 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | 735 EROFS_MAP_FRAGMENT; 736 goto out; 737 } 738 739 err = z_erofs_do_map_blocks(inode, map, flags); 740 out: 741 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); 742 return err; 743 } 744 745 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, 746 loff_t length, unsigned int flags, 747 struct iomap *iomap, struct iomap *srcmap) 748 { 749 int ret; 750 struct erofs_map_blocks map = { .m_la = offset }; 751 752 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP); 753 erofs_put_metabuf(&map.buf); 754 if (ret < 0) 755 return ret; 756 757 iomap->bdev = inode->i_sb->s_bdev; 758 iomap->offset = map.m_la; 759 iomap->length = map.m_llen; 760 if (map.m_flags & EROFS_MAP_MAPPED) { 761 iomap->type = IOMAP_MAPPED; 762 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ? 763 IOMAP_NULL_ADDR : map.m_pa; 764 } else { 765 iomap->type = IOMAP_HOLE; 766 iomap->addr = IOMAP_NULL_ADDR; 767 /* 768 * No strict rule on how to describe extents for post EOF, yet 769 * we need to do like below. Otherwise, iomap itself will get 770 * into an endless loop on post EOF. 771 * 772 * Calculate the effective offset by subtracting extent start 773 * (map.m_la) from the requested offset, and add it to length. 774 * (NB: offset >= map.m_la always) 775 */ 776 if (iomap->offset >= inode->i_size) 777 iomap->length = length + offset - map.m_la; 778 } 779 iomap->flags = 0; 780 return 0; 781 } 782 783 const struct iomap_ops z_erofs_iomap_report_ops = { 784 .iomap_begin = z_erofs_iomap_begin_report, 785 }; 786