147e4937aSGao Xiang // SPDX-License-Identifier: GPL-2.0-only 247e4937aSGao Xiang /* 347e4937aSGao Xiang * Copyright (C) 2019 HUAWEI, Inc. 4592e7cd0SAlexander A. Klimov * https://www.huawei.com/ 547e4937aSGao Xiang * Created by Gao Xiang <gaoxiang25@huawei.com> 647e4937aSGao Xiang */ 747e4937aSGao Xiang #include "compress.h" 847e4937aSGao Xiang #include <linux/module.h> 947e4937aSGao Xiang #include <linux/lz4.h> 1047e4937aSGao Xiang 1147e4937aSGao Xiang #ifndef LZ4_DISTANCE_MAX /* history window size */ 1247e4937aSGao Xiang #define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ 1347e4937aSGao Xiang #endif 1447e4937aSGao Xiang 1547e4937aSGao Xiang #define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1) 1647e4937aSGao Xiang #ifndef LZ4_DECOMPRESS_INPLACE_MARGIN 1747e4937aSGao Xiang #define LZ4_DECOMPRESS_INPLACE_MARGIN(srcsize) (((srcsize) >> 8) + 32) 1847e4937aSGao Xiang #endif 1947e4937aSGao Xiang 2047e4937aSGao Xiang struct z_erofs_decompressor { 2147e4937aSGao Xiang /* 2247e4937aSGao Xiang * if destpages have sparsed pages, fill them with bounce pages. 2347e4937aSGao Xiang * it also check whether destpages indicate continuous physical memory. 2447e4937aSGao Xiang */ 2547e4937aSGao Xiang int (*prepare_destpages)(struct z_erofs_decompress_req *rq, 2647e4937aSGao Xiang struct list_head *pagepool); 2747e4937aSGao Xiang int (*decompress)(struct z_erofs_decompress_req *rq, u8 *out); 2847e4937aSGao Xiang char *name; 2947e4937aSGao Xiang }; 3047e4937aSGao Xiang 315d50538fSHuang Jianan int z_erofs_load_lz4_config(struct super_block *sb, 3246249cdeSGao Xiang struct erofs_super_block *dsb, 3346249cdeSGao Xiang struct z_erofs_lz4_cfgs *lz4, int size) 345d50538fSHuang Jianan { 354fea63f7SGao Xiang struct erofs_sb_info *sbi = EROFS_SB(sb); 3646249cdeSGao Xiang u16 distance; 3746249cdeSGao Xiang 3846249cdeSGao Xiang if (lz4) { 3946249cdeSGao Xiang if (size < sizeof(struct z_erofs_lz4_cfgs)) { 4046249cdeSGao Xiang erofs_err(sb, "invalid lz4 cfgs, size=%u", size); 4146249cdeSGao Xiang return -EINVAL; 4246249cdeSGao Xiang } 4346249cdeSGao Xiang distance = le16_to_cpu(lz4->max_distance); 444fea63f7SGao Xiang 454fea63f7SGao Xiang sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks); 464fea63f7SGao Xiang if (!sbi->lz4.max_pclusterblks) { 474fea63f7SGao Xiang sbi->lz4.max_pclusterblks = 1; /* reserved case */ 484fea63f7SGao Xiang } else if (sbi->lz4.max_pclusterblks > 494fea63f7SGao Xiang Z_EROFS_PCLUSTER_MAX_SIZE / EROFS_BLKSIZ) { 504fea63f7SGao Xiang erofs_err(sb, "too large lz4 pclusterblks %u", 514fea63f7SGao Xiang sbi->lz4.max_pclusterblks); 524fea63f7SGao Xiang return -EINVAL; 534fea63f7SGao Xiang } else if (sbi->lz4.max_pclusterblks >= 2) { 544fea63f7SGao Xiang erofs_info(sb, "EXPERIMENTAL big pcluster feature in use. Use at your own risk!"); 554fea63f7SGao Xiang } 5646249cdeSGao Xiang } else { 5714373711SGao Xiang distance = le16_to_cpu(dsb->u1.lz4_max_distance); 584fea63f7SGao Xiang sbi->lz4.max_pclusterblks = 1; 5946249cdeSGao Xiang } 605d50538fSHuang Jianan 614fea63f7SGao Xiang sbi->lz4.max_distance_pages = distance ? 625d50538fSHuang Jianan DIV_ROUND_UP(distance, PAGE_SIZE) + 1 : 635d50538fSHuang Jianan LZ4_MAX_DISTANCE_PAGES; 644fea63f7SGao Xiang return erofs_pcpubuf_growsize(sbi->lz4.max_pclusterblks); 655d50538fSHuang Jianan } 665d50538fSHuang Jianan 6799634bf3SGao Xiang static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq, 6847e4937aSGao Xiang struct list_head *pagepool) 6947e4937aSGao Xiang { 7047e4937aSGao Xiang const unsigned int nr = 7147e4937aSGao Xiang PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; 7247e4937aSGao Xiang struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL }; 7347e4937aSGao Xiang unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES, 7447e4937aSGao Xiang BITS_PER_LONG)] = { 0 }; 755d50538fSHuang Jianan unsigned int lz4_max_distance_pages = 765d50538fSHuang Jianan EROFS_SB(rq->sb)->lz4.max_distance_pages; 7747e4937aSGao Xiang void *kaddr = NULL; 7847e4937aSGao Xiang unsigned int i, j, top; 7947e4937aSGao Xiang 8047e4937aSGao Xiang top = 0; 8147e4937aSGao Xiang for (i = j = 0; i < nr; ++i, ++j) { 8247e4937aSGao Xiang struct page *const page = rq->out[i]; 8347e4937aSGao Xiang struct page *victim; 8447e4937aSGao Xiang 855d50538fSHuang Jianan if (j >= lz4_max_distance_pages) 8647e4937aSGao Xiang j = 0; 8747e4937aSGao Xiang 8847e4937aSGao Xiang /* 'valid' bounced can only be tested after a complete round */ 8947e4937aSGao Xiang if (test_bit(j, bounced)) { 905d50538fSHuang Jianan DBG_BUGON(i < lz4_max_distance_pages); 915d50538fSHuang Jianan DBG_BUGON(top >= lz4_max_distance_pages); 925d50538fSHuang Jianan availables[top++] = rq->out[i - lz4_max_distance_pages]; 9347e4937aSGao Xiang } 9447e4937aSGao Xiang 9547e4937aSGao Xiang if (page) { 9647e4937aSGao Xiang __clear_bit(j, bounced); 9747e4937aSGao Xiang if (kaddr) { 9847e4937aSGao Xiang if (kaddr + PAGE_SIZE == page_address(page)) 9947e4937aSGao Xiang kaddr += PAGE_SIZE; 10047e4937aSGao Xiang else 10147e4937aSGao Xiang kaddr = NULL; 10247e4937aSGao Xiang } else if (!i) { 10347e4937aSGao Xiang kaddr = page_address(page); 10447e4937aSGao Xiang } 10547e4937aSGao Xiang continue; 10647e4937aSGao Xiang } 10747e4937aSGao Xiang kaddr = NULL; 10847e4937aSGao Xiang __set_bit(j, bounced); 10947e4937aSGao Xiang 11047e4937aSGao Xiang if (top) { 11147e4937aSGao Xiang victim = availables[--top]; 11247e4937aSGao Xiang get_page(victim); 11347e4937aSGao Xiang } else { 114b4892fa3SHuang Jianan victim = erofs_allocpage(pagepool, 115b4892fa3SHuang Jianan GFP_KERNEL | __GFP_NOFAIL); 1166aaa7b06SGao Xiang set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); 11747e4937aSGao Xiang } 11847e4937aSGao Xiang rq->out[i] = victim; 11947e4937aSGao Xiang } 12047e4937aSGao Xiang return kaddr ? 1 : 0; 12147e4937aSGao Xiang } 12247e4937aSGao Xiang 123*598162d0SGao Xiang static void *z_erofs_handle_inplace_io(struct z_erofs_decompress_req *rq, 124*598162d0SGao Xiang void *inpage, unsigned int *inputmargin, int *maptype, 125*598162d0SGao Xiang bool support_0padding) 12647e4937aSGao Xiang { 127*598162d0SGao Xiang unsigned int nrpages_in, nrpages_out; 128*598162d0SGao Xiang unsigned int ofull, oend, inputsize, total, i, j; 129*598162d0SGao Xiang struct page **in; 130*598162d0SGao Xiang void *src, *tmp; 13147e4937aSGao Xiang 132*598162d0SGao Xiang inputsize = rq->inputsize; 133*598162d0SGao Xiang nrpages_in = PAGE_ALIGN(inputsize) >> PAGE_SHIFT; 134*598162d0SGao Xiang oend = rq->pageofs_out + rq->outputsize; 135*598162d0SGao Xiang ofull = PAGE_ALIGN(oend); 136*598162d0SGao Xiang nrpages_out = ofull >> PAGE_SHIFT; 137*598162d0SGao Xiang 138*598162d0SGao Xiang if (rq->inplace_io) { 139*598162d0SGao Xiang if (rq->partial_decoding || !support_0padding || 140*598162d0SGao Xiang ofull - oend < LZ4_DECOMPRESS_INPLACE_MARGIN(inputsize)) 141*598162d0SGao Xiang goto docopy; 142*598162d0SGao Xiang 143*598162d0SGao Xiang for (i = 0; i < nrpages_in; ++i) { 144*598162d0SGao Xiang DBG_BUGON(rq->in[i] == NULL); 145*598162d0SGao Xiang for (j = 0; j < nrpages_out - nrpages_in + i; ++j) 146*598162d0SGao Xiang if (rq->out[j] == rq->in[i]) 147*598162d0SGao Xiang goto docopy; 14847e4937aSGao Xiang } 149*598162d0SGao Xiang } 150*598162d0SGao Xiang 151*598162d0SGao Xiang if (nrpages_in <= 1) { 152*598162d0SGao Xiang *maptype = 0; 153*598162d0SGao Xiang return inpage; 154*598162d0SGao Xiang } 155*598162d0SGao Xiang kunmap_atomic(inpage); 156*598162d0SGao Xiang might_sleep(); 157*598162d0SGao Xiang src = erofs_vm_map_ram(rq->in, nrpages_in); 158*598162d0SGao Xiang if (!src) 159*598162d0SGao Xiang return ERR_PTR(-ENOMEM); 160*598162d0SGao Xiang *maptype = 1; 161*598162d0SGao Xiang return src; 162*598162d0SGao Xiang 163*598162d0SGao Xiang docopy: 164*598162d0SGao Xiang /* Or copy compressed data which can be overlapped to per-CPU buffer */ 165*598162d0SGao Xiang in = rq->in; 166*598162d0SGao Xiang src = erofs_get_pcpubuf(nrpages_in); 167*598162d0SGao Xiang if (!src) { 168*598162d0SGao Xiang DBG_BUGON(1); 169*598162d0SGao Xiang kunmap_atomic(inpage); 170*598162d0SGao Xiang return ERR_PTR(-EFAULT); 171*598162d0SGao Xiang } 172*598162d0SGao Xiang 173*598162d0SGao Xiang tmp = src; 174*598162d0SGao Xiang total = rq->inputsize; 175*598162d0SGao Xiang while (total) { 176*598162d0SGao Xiang unsigned int page_copycnt = 177*598162d0SGao Xiang min_t(unsigned int, total, PAGE_SIZE - *inputmargin); 178*598162d0SGao Xiang 179*598162d0SGao Xiang if (!inpage) 180*598162d0SGao Xiang inpage = kmap_atomic(*in); 181*598162d0SGao Xiang memcpy(tmp, inpage + *inputmargin, page_copycnt); 182*598162d0SGao Xiang kunmap_atomic(inpage); 183*598162d0SGao Xiang inpage = NULL; 184*598162d0SGao Xiang tmp += page_copycnt; 185*598162d0SGao Xiang total -= page_copycnt; 186*598162d0SGao Xiang ++in; 187*598162d0SGao Xiang *inputmargin = 0; 188*598162d0SGao Xiang } 189*598162d0SGao Xiang *maptype = 2; 190*598162d0SGao Xiang return src; 19147e4937aSGao Xiang } 19247e4937aSGao Xiang 19399634bf3SGao Xiang static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out) 19447e4937aSGao Xiang { 195*598162d0SGao Xiang unsigned int inputmargin; 196*598162d0SGao Xiang u8 *headpage, *src; 197*598162d0SGao Xiang bool support_0padding; 198*598162d0SGao Xiang int ret, maptype; 19947e4937aSGao Xiang 200*598162d0SGao Xiang DBG_BUGON(*rq->in == NULL); 201*598162d0SGao Xiang headpage = kmap_atomic(*rq->in); 20247e4937aSGao Xiang inputmargin = 0; 20347e4937aSGao Xiang support_0padding = false; 20447e4937aSGao Xiang 20547e4937aSGao Xiang /* decompression inplace is only safe when 0padding is enabled */ 206de06a6a3SGao Xiang if (erofs_sb_has_lz4_0padding(EROFS_SB(rq->sb))) { 20747e4937aSGao Xiang support_0padding = true; 20847e4937aSGao Xiang 209*598162d0SGao Xiang while (!headpage[inputmargin & ~PAGE_MASK]) 21047e4937aSGao Xiang if (!(++inputmargin & ~PAGE_MASK)) 21147e4937aSGao Xiang break; 21247e4937aSGao Xiang 21347e4937aSGao Xiang if (inputmargin >= rq->inputsize) { 214*598162d0SGao Xiang kunmap_atomic(headpage); 21547e4937aSGao Xiang return -EIO; 21647e4937aSGao Xiang } 21747e4937aSGao Xiang } 21847e4937aSGao Xiang 219*598162d0SGao Xiang rq->inputsize -= inputmargin; 220*598162d0SGao Xiang src = z_erofs_handle_inplace_io(rq, headpage, &inputmargin, &maptype, 221*598162d0SGao Xiang support_0padding); 222*598162d0SGao Xiang if (IS_ERR(src)) 223*598162d0SGao Xiang return PTR_ERR(src); 22447e4937aSGao Xiang 225af1038abSGao Xiang /* legacy format could compress extra data in a pcluster. */ 226af1038abSGao Xiang if (rq->partial_decoding || !support_0padding) 22747e4937aSGao Xiang ret = LZ4_decompress_safe_partial(src + inputmargin, out, 228*598162d0SGao Xiang rq->inputsize, rq->outputsize, rq->outputsize); 229af1038abSGao Xiang else 230af1038abSGao Xiang ret = LZ4_decompress_safe(src + inputmargin, out, 231*598162d0SGao Xiang rq->inputsize, rq->outputsize); 232af1038abSGao Xiang 233aa99a76bSGao Xiang if (ret != rq->outputsize) { 234aa99a76bSGao Xiang erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]", 235*598162d0SGao Xiang ret, rq->inputsize, inputmargin, rq->outputsize); 236aa99a76bSGao Xiang 23747e4937aSGao Xiang WARN_ON(1); 23847e4937aSGao Xiang print_hex_dump(KERN_DEBUG, "[ in]: ", DUMP_PREFIX_OFFSET, 239*598162d0SGao Xiang 16, 1, src + inputmargin, rq->inputsize, true); 24047e4937aSGao Xiang print_hex_dump(KERN_DEBUG, "[out]: ", DUMP_PREFIX_OFFSET, 24147e4937aSGao Xiang 16, 1, out, rq->outputsize, true); 242aa99a76bSGao Xiang 243aa99a76bSGao Xiang if (ret >= 0) 244aa99a76bSGao Xiang memset(out + ret, 0, rq->outputsize - ret); 24547e4937aSGao Xiang ret = -EIO; 24647e4937aSGao Xiang } 24747e4937aSGao Xiang 248*598162d0SGao Xiang if (maptype == 0) { 24947e4937aSGao Xiang kunmap_atomic(src); 250*598162d0SGao Xiang } else if (maptype == 1) { 251*598162d0SGao Xiang vm_unmap_ram(src, PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT); 252*598162d0SGao Xiang } else if (maptype == 2) { 253*598162d0SGao Xiang erofs_put_pcpubuf(src); 254*598162d0SGao Xiang } else { 255*598162d0SGao Xiang DBG_BUGON(1); 256*598162d0SGao Xiang return -EFAULT; 257*598162d0SGao Xiang } 25847e4937aSGao Xiang return ret; 25947e4937aSGao Xiang } 26047e4937aSGao Xiang 26147e4937aSGao Xiang static struct z_erofs_decompressor decompressors[] = { 26247e4937aSGao Xiang [Z_EROFS_COMPRESSION_SHIFTED] = { 26347e4937aSGao Xiang .name = "shifted" 26447e4937aSGao Xiang }, 26547e4937aSGao Xiang [Z_EROFS_COMPRESSION_LZ4] = { 26699634bf3SGao Xiang .prepare_destpages = z_erofs_lz4_prepare_destpages, 26799634bf3SGao Xiang .decompress = z_erofs_lz4_decompress, 26847e4937aSGao Xiang .name = "lz4" 26947e4937aSGao Xiang }, 27047e4937aSGao Xiang }; 27147e4937aSGao Xiang 27247e4937aSGao Xiang static void copy_from_pcpubuf(struct page **out, const char *dst, 27347e4937aSGao Xiang unsigned short pageofs_out, 27447e4937aSGao Xiang unsigned int outputsize) 27547e4937aSGao Xiang { 27647e4937aSGao Xiang const char *end = dst + outputsize; 27747e4937aSGao Xiang const unsigned int righthalf = PAGE_SIZE - pageofs_out; 27847e4937aSGao Xiang const char *cur = dst - pageofs_out; 27947e4937aSGao Xiang 28047e4937aSGao Xiang while (cur < end) { 28147e4937aSGao Xiang struct page *const page = *out++; 28247e4937aSGao Xiang 28347e4937aSGao Xiang if (page) { 28447e4937aSGao Xiang char *buf = kmap_atomic(page); 28547e4937aSGao Xiang 28647e4937aSGao Xiang if (cur >= dst) { 28747e4937aSGao Xiang memcpy(buf, cur, min_t(uint, PAGE_SIZE, 28847e4937aSGao Xiang end - cur)); 28947e4937aSGao Xiang } else { 29047e4937aSGao Xiang memcpy(buf + pageofs_out, cur + pageofs_out, 29147e4937aSGao Xiang min_t(uint, righthalf, end - cur)); 29247e4937aSGao Xiang } 29347e4937aSGao Xiang kunmap_atomic(buf); 29447e4937aSGao Xiang } 29547e4937aSGao Xiang cur += PAGE_SIZE; 29647e4937aSGao Xiang } 29747e4937aSGao Xiang } 29847e4937aSGao Xiang 29999634bf3SGao Xiang static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq, 30047e4937aSGao Xiang struct list_head *pagepool) 30147e4937aSGao Xiang { 30247e4937aSGao Xiang const unsigned int nrpages_out = 30347e4937aSGao Xiang PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; 30447e4937aSGao Xiang const struct z_erofs_decompressor *alg = decompressors + rq->alg; 30547e4937aSGao Xiang unsigned int dst_maptype; 30647e4937aSGao Xiang void *dst; 307*598162d0SGao Xiang int ret; 30847e4937aSGao Xiang 309*598162d0SGao Xiang /* two optimized fast paths only for non bigpcluster cases yet */ 310*598162d0SGao Xiang if (rq->inputsize <= PAGE_SIZE) { 31147e4937aSGao Xiang if (nrpages_out == 1 && !rq->inplace_io) { 31247e4937aSGao Xiang DBG_BUGON(!*rq->out); 31347e4937aSGao Xiang dst = kmap_atomic(*rq->out); 31447e4937aSGao Xiang dst_maptype = 0; 31547e4937aSGao Xiang goto dstmap_out; 31647e4937aSGao Xiang } 31747e4937aSGao Xiang 31847e4937aSGao Xiang /* 31947e4937aSGao Xiang * For the case of small output size (especially much less 32047e4937aSGao Xiang * than PAGE_SIZE), memcpy the decompressed data rather than 32147e4937aSGao Xiang * compressed data is preferred. 32247e4937aSGao Xiang */ 32347e4937aSGao Xiang if (rq->outputsize <= PAGE_SIZE * 7 / 8) { 32452488734SGao Xiang dst = erofs_get_pcpubuf(1); 32547e4937aSGao Xiang if (IS_ERR(dst)) 32647e4937aSGao Xiang return PTR_ERR(dst); 32747e4937aSGao Xiang 32847e4937aSGao Xiang rq->inplace_io = false; 32947e4937aSGao Xiang ret = alg->decompress(rq, dst); 33047e4937aSGao Xiang if (!ret) 33147e4937aSGao Xiang copy_from_pcpubuf(rq->out, dst, rq->pageofs_out, 33247e4937aSGao Xiang rq->outputsize); 33347e4937aSGao Xiang 33447e4937aSGao Xiang erofs_put_pcpubuf(dst); 33547e4937aSGao Xiang return ret; 33647e4937aSGao Xiang } 337*598162d0SGao Xiang } 33847e4937aSGao Xiang 339*598162d0SGao Xiang /* general decoding path which can be used for all cases */ 34047e4937aSGao Xiang ret = alg->prepare_destpages(rq, pagepool); 341*598162d0SGao Xiang if (ret < 0) 34247e4937aSGao Xiang return ret; 343*598162d0SGao Xiang if (ret) { 34447e4937aSGao Xiang dst = page_address(*rq->out); 34547e4937aSGao Xiang dst_maptype = 1; 34647e4937aSGao Xiang goto dstmap_out; 34747e4937aSGao Xiang } 34847e4937aSGao Xiang 349*598162d0SGao Xiang dst = erofs_vm_map_ram(rq->out, nrpages_out); 35047e4937aSGao Xiang if (!dst) 35147e4937aSGao Xiang return -ENOMEM; 35247e4937aSGao Xiang dst_maptype = 2; 35347e4937aSGao Xiang 35447e4937aSGao Xiang dstmap_out: 35547e4937aSGao Xiang ret = alg->decompress(rq, dst + rq->pageofs_out); 35647e4937aSGao Xiang 35747e4937aSGao Xiang if (!dst_maptype) 35847e4937aSGao Xiang kunmap_atomic(dst); 35947e4937aSGao Xiang else if (dst_maptype == 2) 36073d03931SGao Xiang vm_unmap_ram(dst, nrpages_out); 36147e4937aSGao Xiang return ret; 36247e4937aSGao Xiang } 36347e4937aSGao Xiang 36499634bf3SGao Xiang static int z_erofs_shifted_transform(const struct z_erofs_decompress_req *rq, 36547e4937aSGao Xiang struct list_head *pagepool) 36647e4937aSGao Xiang { 36747e4937aSGao Xiang const unsigned int nrpages_out = 36847e4937aSGao Xiang PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; 36947e4937aSGao Xiang const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out; 37047e4937aSGao Xiang unsigned char *src, *dst; 37147e4937aSGao Xiang 37247e4937aSGao Xiang if (nrpages_out > 2) { 37347e4937aSGao Xiang DBG_BUGON(1); 37447e4937aSGao Xiang return -EIO; 37547e4937aSGao Xiang } 37647e4937aSGao Xiang 37747e4937aSGao Xiang if (rq->out[0] == *rq->in) { 37847e4937aSGao Xiang DBG_BUGON(nrpages_out != 1); 37947e4937aSGao Xiang return 0; 38047e4937aSGao Xiang } 38147e4937aSGao Xiang 38247e4937aSGao Xiang src = kmap_atomic(*rq->in); 3834d202437SGao Xiang if (rq->out[0]) { 38447e4937aSGao Xiang dst = kmap_atomic(rq->out[0]); 38547e4937aSGao Xiang memcpy(dst + rq->pageofs_out, src, righthalf); 3864d202437SGao Xiang kunmap_atomic(dst); 38747e4937aSGao Xiang } 38847e4937aSGao Xiang 3894d202437SGao Xiang if (nrpages_out == 2) { 3904d202437SGao Xiang DBG_BUGON(!rq->out[1]); 39147e4937aSGao Xiang if (rq->out[1] == *rq->in) { 39247e4937aSGao Xiang memmove(src, src + righthalf, rq->pageofs_out); 3934d202437SGao Xiang } else { 39447e4937aSGao Xiang dst = kmap_atomic(rq->out[1]); 39547e4937aSGao Xiang memcpy(dst, src + righthalf, rq->pageofs_out); 39647e4937aSGao Xiang kunmap_atomic(dst); 3974d202437SGao Xiang } 3984d202437SGao Xiang } 39947e4937aSGao Xiang kunmap_atomic(src); 40047e4937aSGao Xiang return 0; 40147e4937aSGao Xiang } 40247e4937aSGao Xiang 40347e4937aSGao Xiang int z_erofs_decompress(struct z_erofs_decompress_req *rq, 40447e4937aSGao Xiang struct list_head *pagepool) 40547e4937aSGao Xiang { 40647e4937aSGao Xiang if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) 40799634bf3SGao Xiang return z_erofs_shifted_transform(rq, pagepool); 40899634bf3SGao Xiang return z_erofs_decompress_generic(rq, pagepool); 40947e4937aSGao Xiang } 41047e4937aSGao Xiang 411