1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2018-2019 HUAWEI, Inc. 4 * https://www.huawei.com/ 5 */ 6 #include "internal.h" 7 #include <asm/unaligned.h> 8 #include <trace/events/erofs.h> 9 10 int z_erofs_fill_inode(struct inode *inode) 11 { 12 struct erofs_inode *const vi = EROFS_I(inode); 13 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); 14 15 if (!erofs_sb_has_big_pcluster(sbi) && 16 !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) && 17 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { 18 vi->z_advise = 0; 19 vi->z_algorithmtype[0] = 0; 20 vi->z_algorithmtype[1] = 0; 21 vi->z_logical_clusterbits = LOG_BLOCK_SIZE; 22 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 23 } 24 inode->i_mapping->a_ops = &z_erofs_aops; 25 return 0; 26 } 27 28 struct z_erofs_maprecorder { 29 struct inode *inode; 30 struct erofs_map_blocks *map; 31 void *kaddr; 32 33 unsigned long lcn; 34 /* compression extent information gathered */ 35 u8 type, headtype; 36 u16 clusterofs; 37 u16 delta[2]; 38 erofs_blk_t pblk, compressedblks; 39 erofs_off_t nextpackoff; 40 bool partialref; 41 }; 42 43 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, 44 unsigned long lcn) 45 { 46 struct inode *const inode = m->inode; 47 struct erofs_inode *const vi = EROFS_I(inode); 48 const erofs_off_t pos = 49 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(erofs_iloc(inode) + 50 vi->inode_isize + vi->xattr_isize) + 51 lcn * sizeof(struct z_erofs_vle_decompressed_index); 52 struct z_erofs_vle_decompressed_index *di; 53 unsigned int advise, type; 54 55 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, 56 erofs_blknr(pos), EROFS_KMAP); 57 if (IS_ERR(m->kaddr)) 58 return PTR_ERR(m->kaddr); 59 60 m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index); 61 m->lcn = lcn; 62 di = m->kaddr + erofs_blkoff(pos); 63 64 advise = le16_to_cpu(di->di_advise); 65 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) & 66 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1); 67 switch (type) { 68 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 69 m->clusterofs = 1 << vi->z_logical_clusterbits; 70 m->delta[0] = le16_to_cpu(di->di_u.delta[0]); 71 if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) { 72 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 73 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 74 DBG_BUGON(1); 75 return -EFSCORRUPTED; 76 } 77 m->compressedblks = m->delta[0] & 78 ~Z_EROFS_VLE_DI_D0_CBLKCNT; 79 m->delta[0] = 1; 80 } 81 m->delta[1] = le16_to_cpu(di->di_u.delta[1]); 82 break; 83 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 84 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 85 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 86 if (advise & Z_EROFS_VLE_DI_PARTIAL_REF) 87 m->partialref = true; 88 m->clusterofs = le16_to_cpu(di->di_clusterofs); 89 m->pblk = le32_to_cpu(di->di_u.blkaddr); 90 break; 91 default: 92 DBG_BUGON(1); 93 return -EOPNOTSUPP; 94 } 95 m->type = type; 96 return 0; 97 } 98 99 static unsigned int decode_compactedbits(unsigned int lobits, 100 unsigned int lomask, 101 u8 *in, unsigned int pos, u8 *type) 102 { 103 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); 104 const unsigned int lo = v & lomask; 105 106 *type = (v >> lobits) & 3; 107 return lo; 108 } 109 110 static int get_compacted_la_distance(unsigned int lclusterbits, 111 unsigned int encodebits, 112 unsigned int vcnt, u8 *in, int i) 113 { 114 const unsigned int lomask = (1 << lclusterbits) - 1; 115 unsigned int lo, d1 = 0; 116 u8 type; 117 118 DBG_BUGON(i >= vcnt); 119 120 do { 121 lo = decode_compactedbits(lclusterbits, lomask, 122 in, encodebits * i, &type); 123 124 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 125 return d1; 126 ++d1; 127 } while (++i < vcnt); 128 129 /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */ 130 if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT)) 131 d1 += lo - 1; 132 return d1; 133 } 134 135 static int unpack_compacted_index(struct z_erofs_maprecorder *m, 136 unsigned int amortizedshift, 137 erofs_off_t pos, bool lookahead) 138 { 139 struct erofs_inode *const vi = EROFS_I(m->inode); 140 const unsigned int lclusterbits = vi->z_logical_clusterbits; 141 const unsigned int lomask = (1 << lclusterbits) - 1; 142 unsigned int vcnt, base, lo, encodebits, nblk, eofs; 143 int i; 144 u8 *in, type; 145 bool big_pcluster; 146 147 if (1 << amortizedshift == 4) 148 vcnt = 2; 149 else if (1 << amortizedshift == 2 && lclusterbits == 12) 150 vcnt = 16; 151 else 152 return -EOPNOTSUPP; 153 154 /* it doesn't equal to round_up(..) */ 155 m->nextpackoff = round_down(pos, vcnt << amortizedshift) + 156 (vcnt << amortizedshift); 157 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; 158 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; 159 eofs = erofs_blkoff(pos); 160 base = round_down(eofs, vcnt << amortizedshift); 161 in = m->kaddr + base; 162 163 i = (eofs - base) >> amortizedshift; 164 165 lo = decode_compactedbits(lclusterbits, lomask, 166 in, encodebits * i, &type); 167 m->type = type; 168 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 169 m->clusterofs = 1 << lclusterbits; 170 171 /* figure out lookahead_distance: delta[1] if needed */ 172 if (lookahead) 173 m->delta[1] = get_compacted_la_distance(lclusterbits, 174 encodebits, vcnt, in, i); 175 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { 176 if (!big_pcluster) { 177 DBG_BUGON(1); 178 return -EFSCORRUPTED; 179 } 180 m->compressedblks = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; 181 m->delta[0] = 1; 182 return 0; 183 } else if (i + 1 != (int)vcnt) { 184 m->delta[0] = lo; 185 return 0; 186 } 187 /* 188 * since the last lcluster in the pack is special, 189 * of which lo saves delta[1] rather than delta[0]. 190 * Hence, get delta[0] by the previous lcluster indirectly. 191 */ 192 lo = decode_compactedbits(lclusterbits, lomask, 193 in, encodebits * (i - 1), &type); 194 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 195 lo = 0; 196 else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) 197 lo = 1; 198 m->delta[0] = lo + 1; 199 return 0; 200 } 201 m->clusterofs = lo; 202 m->delta[0] = 0; 203 /* figout out blkaddr (pblk) for HEAD lclusters */ 204 if (!big_pcluster) { 205 nblk = 1; 206 while (i > 0) { 207 --i; 208 lo = decode_compactedbits(lclusterbits, lomask, 209 in, encodebits * i, &type); 210 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) 211 i -= lo; 212 213 if (i >= 0) 214 ++nblk; 215 } 216 } else { 217 nblk = 0; 218 while (i > 0) { 219 --i; 220 lo = decode_compactedbits(lclusterbits, lomask, 221 in, encodebits * i, &type); 222 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 223 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) { 224 --i; 225 nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT; 226 continue; 227 } 228 /* bigpcluster shouldn't have plain d0 == 1 */ 229 if (lo <= 1) { 230 DBG_BUGON(1); 231 return -EFSCORRUPTED; 232 } 233 i -= lo - 2; 234 continue; 235 } 236 ++nblk; 237 } 238 } 239 in += (vcnt << amortizedshift) - sizeof(__le32); 240 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; 241 return 0; 242 } 243 244 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, 245 unsigned long lcn, bool lookahead) 246 { 247 struct inode *const inode = m->inode; 248 struct erofs_inode *const vi = EROFS_I(inode); 249 const unsigned int lclusterbits = vi->z_logical_clusterbits; 250 const erofs_off_t ebase = sizeof(struct z_erofs_map_header) + 251 ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); 252 const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); 253 unsigned int compacted_4b_initial, compacted_2b; 254 unsigned int amortizedshift; 255 erofs_off_t pos; 256 257 if (lclusterbits != 12) 258 return -EOPNOTSUPP; 259 260 if (lcn >= totalidx) 261 return -EINVAL; 262 263 m->lcn = lcn; 264 /* used to align to 32-byte (compacted_2b) alignment */ 265 compacted_4b_initial = (32 - ebase % 32) / 4; 266 if (compacted_4b_initial == 32 / 4) 267 compacted_4b_initial = 0; 268 269 if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && 270 compacted_4b_initial < totalidx) 271 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); 272 else 273 compacted_2b = 0; 274 275 pos = ebase; 276 if (lcn < compacted_4b_initial) { 277 amortizedshift = 2; 278 goto out; 279 } 280 pos += compacted_4b_initial * 4; 281 lcn -= compacted_4b_initial; 282 283 if (lcn < compacted_2b) { 284 amortizedshift = 1; 285 goto out; 286 } 287 pos += compacted_2b * 2; 288 lcn -= compacted_2b; 289 amortizedshift = 2; 290 out: 291 pos += lcn * (1 << amortizedshift); 292 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb, 293 erofs_blknr(pos), EROFS_KMAP); 294 if (IS_ERR(m->kaddr)) 295 return PTR_ERR(m->kaddr); 296 return unpack_compacted_index(m, amortizedshift, pos, lookahead); 297 } 298 299 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, 300 unsigned int lcn, bool lookahead) 301 { 302 const unsigned int datamode = EROFS_I(m->inode)->datalayout; 303 304 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY) 305 return legacy_load_cluster_from_disk(m, lcn); 306 307 if (datamode == EROFS_INODE_FLAT_COMPRESSION) 308 return compacted_load_cluster_from_disk(m, lcn, lookahead); 309 310 return -EINVAL; 311 } 312 313 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, 314 unsigned int lookback_distance) 315 { 316 struct erofs_inode *const vi = EROFS_I(m->inode); 317 const unsigned int lclusterbits = vi->z_logical_clusterbits; 318 319 while (m->lcn >= lookback_distance) { 320 unsigned long lcn = m->lcn - lookback_distance; 321 int err; 322 323 /* load extent head logical cluster if needed */ 324 err = z_erofs_load_cluster_from_disk(m, lcn, false); 325 if (err) 326 return err; 327 328 switch (m->type) { 329 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 330 if (!m->delta[0]) { 331 erofs_err(m->inode->i_sb, 332 "invalid lookback distance 0 @ nid %llu", 333 vi->nid); 334 DBG_BUGON(1); 335 return -EFSCORRUPTED; 336 } 337 lookback_distance = m->delta[0]; 338 continue; 339 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 340 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 341 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 342 m->headtype = m->type; 343 m->map->m_la = (lcn << lclusterbits) | m->clusterofs; 344 return 0; 345 default: 346 erofs_err(m->inode->i_sb, 347 "unknown type %u @ lcn %lu of nid %llu", 348 m->type, lcn, vi->nid); 349 DBG_BUGON(1); 350 return -EOPNOTSUPP; 351 } 352 } 353 354 erofs_err(m->inode->i_sb, "bogus lookback distance @ nid %llu", 355 vi->nid); 356 DBG_BUGON(1); 357 return -EFSCORRUPTED; 358 } 359 360 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, 361 unsigned int initial_lcn) 362 { 363 struct erofs_inode *const vi = EROFS_I(m->inode); 364 struct erofs_map_blocks *const map = m->map; 365 const unsigned int lclusterbits = vi->z_logical_clusterbits; 366 unsigned long lcn; 367 int err; 368 369 DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN && 370 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 && 371 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD2); 372 DBG_BUGON(m->type != m->headtype); 373 374 if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || 375 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1) && 376 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) || 377 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) && 378 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { 379 map->m_plen = 1ULL << lclusterbits; 380 return 0; 381 } 382 lcn = m->lcn + 1; 383 if (m->compressedblks) 384 goto out; 385 386 err = z_erofs_load_cluster_from_disk(m, lcn, false); 387 if (err) 388 return err; 389 390 /* 391 * If the 1st NONHEAD lcluster has already been handled initially w/o 392 * valid compressedblks, which means at least it mustn't be CBLKCNT, or 393 * an internal implemenatation error is detected. 394 * 395 * The following code can also handle it properly anyway, but let's 396 * BUG_ON in the debugging mode only for developers to notice that. 397 */ 398 DBG_BUGON(lcn == initial_lcn && 399 m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD); 400 401 switch (m->type) { 402 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 403 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 404 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 405 /* 406 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type 407 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster. 408 */ 409 m->compressedblks = 1 << (lclusterbits - LOG_BLOCK_SIZE); 410 break; 411 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 412 if (m->delta[0] != 1) 413 goto err_bonus_cblkcnt; 414 if (m->compressedblks) 415 break; 416 fallthrough; 417 default: 418 erofs_err(m->inode->i_sb, 419 "cannot found CBLKCNT @ lcn %lu of nid %llu", 420 lcn, vi->nid); 421 DBG_BUGON(1); 422 return -EFSCORRUPTED; 423 } 424 out: 425 map->m_plen = (u64)m->compressedblks << LOG_BLOCK_SIZE; 426 return 0; 427 err_bonus_cblkcnt: 428 erofs_err(m->inode->i_sb, 429 "bogus CBLKCNT @ lcn %lu of nid %llu", 430 lcn, vi->nid); 431 DBG_BUGON(1); 432 return -EFSCORRUPTED; 433 } 434 435 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) 436 { 437 struct inode *inode = m->inode; 438 struct erofs_inode *vi = EROFS_I(inode); 439 struct erofs_map_blocks *map = m->map; 440 unsigned int lclusterbits = vi->z_logical_clusterbits; 441 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits; 442 int err; 443 444 do { 445 /* handle the last EOF pcluster (no next HEAD lcluster) */ 446 if ((lcn << lclusterbits) >= inode->i_size) { 447 map->m_llen = inode->i_size - map->m_la; 448 return 0; 449 } 450 451 err = z_erofs_load_cluster_from_disk(m, lcn, true); 452 if (err) 453 return err; 454 455 if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) { 456 DBG_BUGON(!m->delta[1] && 457 m->clusterofs != 1 << lclusterbits); 458 } else if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || 459 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 || 460 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) { 461 /* go on until the next HEAD lcluster */ 462 if (lcn != headlcn) 463 break; 464 m->delta[1] = 1; 465 } else { 466 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", 467 m->type, lcn, vi->nid); 468 DBG_BUGON(1); 469 return -EOPNOTSUPP; 470 } 471 lcn += m->delta[1]; 472 } while (m->delta[1]); 473 474 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la; 475 return 0; 476 } 477 478 static int z_erofs_do_map_blocks(struct inode *inode, 479 struct erofs_map_blocks *map, 480 int flags) 481 { 482 struct erofs_inode *const vi = EROFS_I(inode); 483 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; 484 bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 485 struct z_erofs_maprecorder m = { 486 .inode = inode, 487 .map = map, 488 }; 489 int err = 0; 490 unsigned int lclusterbits, endoff; 491 unsigned long initial_lcn; 492 unsigned long long ofs, end; 493 494 lclusterbits = vi->z_logical_clusterbits; 495 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la; 496 initial_lcn = ofs >> lclusterbits; 497 endoff = ofs & ((1 << lclusterbits) - 1); 498 499 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false); 500 if (err) 501 goto unmap_out; 502 503 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL)) 504 vi->z_idataoff = m.nextpackoff; 505 506 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; 507 end = (m.lcn + 1ULL) << lclusterbits; 508 509 switch (m.type) { 510 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN: 511 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1: 512 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2: 513 if (endoff >= m.clusterofs) { 514 m.headtype = m.type; 515 map->m_la = (m.lcn << lclusterbits) | m.clusterofs; 516 /* 517 * For ztailpacking files, in order to inline data more 518 * effectively, special EOF lclusters are now supported 519 * which can have three parts at most. 520 */ 521 if (ztailpacking && end > inode->i_size) 522 end = inode->i_size; 523 break; 524 } 525 /* m.lcn should be >= 1 if endoff < m.clusterofs */ 526 if (!m.lcn) { 527 erofs_err(inode->i_sb, 528 "invalid logical cluster 0 at nid %llu", 529 vi->nid); 530 err = -EFSCORRUPTED; 531 goto unmap_out; 532 } 533 end = (m.lcn << lclusterbits) | m.clusterofs; 534 map->m_flags |= EROFS_MAP_FULL_MAPPED; 535 m.delta[0] = 1; 536 fallthrough; 537 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD: 538 /* get the corresponding first chunk */ 539 err = z_erofs_extent_lookback(&m, m.delta[0]); 540 if (err) 541 goto unmap_out; 542 break; 543 default: 544 erofs_err(inode->i_sb, 545 "unknown type %u @ offset %llu of nid %llu", 546 m.type, ofs, vi->nid); 547 err = -EOPNOTSUPP; 548 goto unmap_out; 549 } 550 if (m.partialref) 551 map->m_flags |= EROFS_MAP_PARTIAL_REF; 552 map->m_llen = end - map->m_la; 553 554 if (flags & EROFS_GET_BLOCKS_FINDTAIL) { 555 vi->z_tailextent_headlcn = m.lcn; 556 /* for non-compact indexes, fragmentoff is 64 bits */ 557 if (fragment && 558 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) 559 vi->z_fragmentoff |= (u64)m.pblk << 32; 560 } 561 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { 562 map->m_flags |= EROFS_MAP_META; 563 map->m_pa = vi->z_idataoff; 564 map->m_plen = vi->z_idata_size; 565 } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { 566 map->m_flags |= EROFS_MAP_FRAGMENT; 567 } else { 568 map->m_pa = blknr_to_addr(m.pblk); 569 err = z_erofs_get_extent_compressedlen(&m, initial_lcn); 570 if (err) 571 goto unmap_out; 572 } 573 574 if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) { 575 if (map->m_llen > map->m_plen) { 576 DBG_BUGON(1); 577 err = -EFSCORRUPTED; 578 goto unmap_out; 579 } 580 if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) 581 map->m_algorithmformat = 582 Z_EROFS_COMPRESSION_INTERLACED; 583 else 584 map->m_algorithmformat = 585 Z_EROFS_COMPRESSION_SHIFTED; 586 } else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) { 587 map->m_algorithmformat = vi->z_algorithmtype[1]; 588 } else { 589 map->m_algorithmformat = vi->z_algorithmtype[0]; 590 } 591 592 if ((flags & EROFS_GET_BLOCKS_FIEMAP) || 593 ((flags & EROFS_GET_BLOCKS_READMORE) && 594 map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA && 595 map->m_llen >= EROFS_BLKSIZ)) { 596 err = z_erofs_get_extent_decompressedlen(&m); 597 if (!err) 598 map->m_flags |= EROFS_MAP_FULL_MAPPED; 599 } 600 601 unmap_out: 602 erofs_unmap_metabuf(&m.map->buf); 603 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o", 604 __func__, map->m_la, map->m_pa, 605 map->m_llen, map->m_plen, map->m_flags); 606 return err; 607 } 608 609 static int z_erofs_fill_inode_lazy(struct inode *inode) 610 { 611 struct erofs_inode *const vi = EROFS_I(inode); 612 struct super_block *const sb = inode->i_sb; 613 int err, headnr; 614 erofs_off_t pos; 615 struct erofs_buf buf = __EROFS_BUF_INITIALIZER; 616 void *kaddr; 617 struct z_erofs_map_header *h; 618 619 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { 620 /* 621 * paired with smp_mb() at the end of the function to ensure 622 * fields will only be observed after the bit is set. 623 */ 624 smp_mb(); 625 return 0; 626 } 627 628 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) 629 return -ERESTARTSYS; 630 631 err = 0; 632 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) 633 goto out_unlock; 634 635 pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); 636 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos), EROFS_KMAP); 637 if (IS_ERR(kaddr)) { 638 err = PTR_ERR(kaddr); 639 goto out_unlock; 640 } 641 642 h = kaddr + erofs_blkoff(pos); 643 /* 644 * if the highest bit of the 8-byte map header is set, the whole file 645 * is stored in the packed inode. The rest bits keeps z_fragmentoff. 646 */ 647 if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { 648 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; 649 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); 650 vi->z_tailextent_headlcn = 0; 651 goto done; 652 } 653 vi->z_advise = le16_to_cpu(h->h_advise); 654 vi->z_algorithmtype[0] = h->h_algorithmtype & 15; 655 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; 656 657 headnr = 0; 658 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX || 659 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) { 660 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", 661 headnr + 1, vi->z_algorithmtype[headnr], vi->nid); 662 err = -EOPNOTSUPP; 663 goto out_put_metabuf; 664 } 665 666 vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7); 667 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && 668 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | 669 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 670 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", 671 vi->nid); 672 err = -EFSCORRUPTED; 673 goto out_put_metabuf; 674 } 675 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION && 676 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ 677 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { 678 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", 679 vi->nid); 680 err = -EFSCORRUPTED; 681 goto out_put_metabuf; 682 } 683 684 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) { 685 struct erofs_map_blocks map = { 686 .buf = __EROFS_BUF_INITIALIZER 687 }; 688 689 vi->z_idata_size = le16_to_cpu(h->h_idata_size); 690 err = z_erofs_do_map_blocks(inode, &map, 691 EROFS_GET_BLOCKS_FINDTAIL); 692 erofs_put_metabuf(&map.buf); 693 694 if (!map.m_plen || 695 erofs_blkoff(map.m_pa) + map.m_plen > EROFS_BLKSIZ) { 696 erofs_err(sb, "invalid tail-packing pclustersize %llu", 697 map.m_plen); 698 err = -EFSCORRUPTED; 699 } 700 if (err < 0) 701 goto out_put_metabuf; 702 } 703 704 if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER && 705 !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) { 706 struct erofs_map_blocks map = { 707 .buf = __EROFS_BUF_INITIALIZER 708 }; 709 710 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); 711 err = z_erofs_do_map_blocks(inode, &map, 712 EROFS_GET_BLOCKS_FINDTAIL); 713 erofs_put_metabuf(&map.buf); 714 if (err < 0) 715 goto out_put_metabuf; 716 } 717 done: 718 /* paired with smp_mb() at the beginning of the function */ 719 smp_mb(); 720 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); 721 out_put_metabuf: 722 erofs_put_metabuf(&buf); 723 out_unlock: 724 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); 725 return err; 726 } 727 728 int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, 729 int flags) 730 { 731 struct erofs_inode *const vi = EROFS_I(inode); 732 int err = 0; 733 734 trace_z_erofs_map_blocks_iter_enter(inode, map, flags); 735 736 /* when trying to read beyond EOF, leave it unmapped */ 737 if (map->m_la >= inode->i_size) { 738 map->m_llen = map->m_la + 1 - inode->i_size; 739 map->m_la = inode->i_size; 740 map->m_flags = 0; 741 goto out; 742 } 743 744 err = z_erofs_fill_inode_lazy(inode); 745 if (err) 746 goto out; 747 748 if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) && 749 !vi->z_tailextent_headlcn) { 750 map->m_la = 0; 751 map->m_llen = inode->i_size; 752 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED | 753 EROFS_MAP_FRAGMENT; 754 goto out; 755 } 756 757 err = z_erofs_do_map_blocks(inode, map, flags); 758 out: 759 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err); 760 761 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */ 762 DBG_BUGON(err < 0 && err != -ENOMEM); 763 return err; 764 } 765 766 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, 767 loff_t length, unsigned int flags, 768 struct iomap *iomap, struct iomap *srcmap) 769 { 770 int ret; 771 struct erofs_map_blocks map = { .m_la = offset }; 772 773 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP); 774 erofs_put_metabuf(&map.buf); 775 if (ret < 0) 776 return ret; 777 778 iomap->bdev = inode->i_sb->s_bdev; 779 iomap->offset = map.m_la; 780 iomap->length = map.m_llen; 781 if (map.m_flags & EROFS_MAP_MAPPED) { 782 iomap->type = IOMAP_MAPPED; 783 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ? 784 IOMAP_NULL_ADDR : map.m_pa; 785 } else { 786 iomap->type = IOMAP_HOLE; 787 iomap->addr = IOMAP_NULL_ADDR; 788 /* 789 * No strict rule on how to describe extents for post EOF, yet 790 * we need to do like below. Otherwise, iomap itself will get 791 * into an endless loop on post EOF. 792 * 793 * Calculate the effective offset by subtracting extent start 794 * (map.m_la) from the requested offset, and add it to length. 795 * (NB: offset >= map.m_la always) 796 */ 797 if (iomap->offset >= inode->i_size) 798 iomap->length = length + offset - map.m_la; 799 } 800 iomap->flags = 0; 801 return 0; 802 } 803 804 const struct iomap_ops z_erofs_iomap_report_ops = { 805 .iomap_begin = z_erofs_iomap_begin_report, 806 }; 807