1622ceaddSGao Xiang // SPDX-License-Identifier: GPL-2.0-or-later 2622ceaddSGao Xiang #include <linux/xz.h> 3622ceaddSGao Xiang #include <linux/module.h> 4622ceaddSGao Xiang #include "compress.h" 5622ceaddSGao Xiang 6622ceaddSGao Xiang struct z_erofs_lzma { 7622ceaddSGao Xiang struct z_erofs_lzma *next; 8622ceaddSGao Xiang struct xz_dec_microlzma *state; 9622ceaddSGao Xiang struct xz_buf buf; 10622ceaddSGao Xiang u8 bounce[PAGE_SIZE]; 11622ceaddSGao Xiang }; 12622ceaddSGao Xiang 13622ceaddSGao Xiang /* considering the LZMA performance, no need to use a lockless list for now */ 14622ceaddSGao Xiang static DEFINE_SPINLOCK(z_erofs_lzma_lock); 15622ceaddSGao Xiang static unsigned int z_erofs_lzma_max_dictsize; 16622ceaddSGao Xiang static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; 17622ceaddSGao Xiang static struct z_erofs_lzma *z_erofs_lzma_head; 18622ceaddSGao Xiang static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); 19622ceaddSGao Xiang 20622ceaddSGao Xiang module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); 21622ceaddSGao Xiang 22622ceaddSGao Xiang void z_erofs_lzma_exit(void) 23622ceaddSGao Xiang { 24622ceaddSGao Xiang /* there should be no running fs instance */ 25622ceaddSGao Xiang while (z_erofs_lzma_avail_strms) { 26622ceaddSGao Xiang struct z_erofs_lzma *strm; 27622ceaddSGao Xiang 28622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock); 29622ceaddSGao Xiang strm = z_erofs_lzma_head; 30622ceaddSGao Xiang if (!strm) { 31622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 32622ceaddSGao Xiang DBG_BUGON(1); 33622ceaddSGao Xiang return; 34622ceaddSGao Xiang } 35622ceaddSGao Xiang z_erofs_lzma_head = NULL; 36622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 37622ceaddSGao Xiang 38622ceaddSGao Xiang while (strm) { 39622ceaddSGao Xiang struct z_erofs_lzma *n = strm->next; 40622ceaddSGao Xiang 41622ceaddSGao Xiang if (strm->state) 42622ceaddSGao Xiang xz_dec_microlzma_end(strm->state); 43622ceaddSGao Xiang kfree(strm); 44622ceaddSGao Xiang --z_erofs_lzma_avail_strms; 45622ceaddSGao Xiang strm = n; 46622ceaddSGao Xiang } 47622ceaddSGao Xiang } 48622ceaddSGao Xiang } 49622ceaddSGao Xiang 50*a279adedSYangtao Li int __init z_erofs_lzma_init(void) 51622ceaddSGao Xiang { 52622ceaddSGao Xiang unsigned int i; 53622ceaddSGao Xiang 54622ceaddSGao Xiang /* by default, use # of possible CPUs instead */ 55622ceaddSGao Xiang if (!z_erofs_lzma_nstrms) 56622ceaddSGao Xiang z_erofs_lzma_nstrms = num_possible_cpus(); 57622ceaddSGao Xiang 58622ceaddSGao Xiang for (i = 0; i < z_erofs_lzma_nstrms; ++i) { 59622ceaddSGao Xiang struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL); 60622ceaddSGao Xiang 61622ceaddSGao Xiang if (!strm) { 62622ceaddSGao Xiang z_erofs_lzma_exit(); 63622ceaddSGao Xiang return -ENOMEM; 64622ceaddSGao Xiang } 65622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock); 66622ceaddSGao Xiang strm->next = z_erofs_lzma_head; 67622ceaddSGao Xiang z_erofs_lzma_head = strm; 68622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 69622ceaddSGao Xiang ++z_erofs_lzma_avail_strms; 70622ceaddSGao Xiang } 71622ceaddSGao Xiang return 0; 72622ceaddSGao Xiang } 73622ceaddSGao Xiang 74622ceaddSGao Xiang int z_erofs_load_lzma_config(struct super_block *sb, 75622ceaddSGao Xiang struct erofs_super_block *dsb, 76622ceaddSGao Xiang struct z_erofs_lzma_cfgs *lzma, int size) 77622ceaddSGao Xiang { 78622ceaddSGao Xiang static DEFINE_MUTEX(lzma_resize_mutex); 79622ceaddSGao Xiang unsigned int dict_size, i; 80622ceaddSGao Xiang struct z_erofs_lzma *strm, *head = NULL; 81622ceaddSGao Xiang int err; 82622ceaddSGao Xiang 83622ceaddSGao Xiang if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { 84622ceaddSGao Xiang erofs_err(sb, "invalid lzma cfgs, size=%u", size); 85622ceaddSGao Xiang return -EINVAL; 86622ceaddSGao Xiang } 87622ceaddSGao Xiang if (lzma->format) { 88622ceaddSGao Xiang erofs_err(sb, "unidentified lzma format %x, please check kernel version", 89622ceaddSGao Xiang le16_to_cpu(lzma->format)); 90622ceaddSGao Xiang return -EINVAL; 91622ceaddSGao Xiang } 92622ceaddSGao Xiang dict_size = le32_to_cpu(lzma->dict_size); 93622ceaddSGao Xiang if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { 94622ceaddSGao Xiang erofs_err(sb, "unsupported lzma dictionary size %u", 95622ceaddSGao Xiang dict_size); 96622ceaddSGao Xiang return -EINVAL; 97622ceaddSGao Xiang } 98622ceaddSGao Xiang 99622ceaddSGao Xiang erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!"); 100622ceaddSGao Xiang 101622ceaddSGao Xiang /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ 102622ceaddSGao Xiang mutex_lock(&lzma_resize_mutex); 103622ceaddSGao Xiang 104622ceaddSGao Xiang if (z_erofs_lzma_max_dictsize >= dict_size) { 105622ceaddSGao Xiang mutex_unlock(&lzma_resize_mutex); 106622ceaddSGao Xiang return 0; 107622ceaddSGao Xiang } 108622ceaddSGao Xiang 109622ceaddSGao Xiang /* 1. collect/isolate all streams for the following check */ 110622ceaddSGao Xiang for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { 111622ceaddSGao Xiang struct z_erofs_lzma *last; 112622ceaddSGao Xiang 113622ceaddSGao Xiang again: 114622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock); 115622ceaddSGao Xiang strm = z_erofs_lzma_head; 116622ceaddSGao Xiang if (!strm) { 117622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 118622ceaddSGao Xiang wait_event(z_erofs_lzma_wq, 119622ceaddSGao Xiang READ_ONCE(z_erofs_lzma_head)); 120622ceaddSGao Xiang goto again; 121622ceaddSGao Xiang } 122622ceaddSGao Xiang z_erofs_lzma_head = NULL; 123622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 124622ceaddSGao Xiang 125622ceaddSGao Xiang for (last = strm; last->next; last = last->next) 126622ceaddSGao Xiang ++i; 127622ceaddSGao Xiang last->next = head; 128622ceaddSGao Xiang head = strm; 129622ceaddSGao Xiang } 130622ceaddSGao Xiang 131622ceaddSGao Xiang err = 0; 132622ceaddSGao Xiang /* 2. walk each isolated stream and grow max dict_size if needed */ 133622ceaddSGao Xiang for (strm = head; strm; strm = strm->next) { 134622ceaddSGao Xiang if (strm->state) 135622ceaddSGao Xiang xz_dec_microlzma_end(strm->state); 136622ceaddSGao Xiang strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); 137622ceaddSGao Xiang if (!strm->state) 138622ceaddSGao Xiang err = -ENOMEM; 139622ceaddSGao Xiang } 140622ceaddSGao Xiang 141622ceaddSGao Xiang /* 3. push back all to the global list and update max dict_size */ 142622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock); 143622ceaddSGao Xiang DBG_BUGON(z_erofs_lzma_head); 144622ceaddSGao Xiang z_erofs_lzma_head = head; 145622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 1462df7c4bdSYuwen Chen wake_up_all(&z_erofs_lzma_wq); 147622ceaddSGao Xiang 148622ceaddSGao Xiang z_erofs_lzma_max_dictsize = dict_size; 149622ceaddSGao Xiang mutex_unlock(&lzma_resize_mutex); 150622ceaddSGao Xiang return err; 151622ceaddSGao Xiang } 152622ceaddSGao Xiang 153622ceaddSGao Xiang int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, 154eaa9172aSGao Xiang struct page **pagepool) 155622ceaddSGao Xiang { 156622ceaddSGao Xiang const unsigned int nrpages_out = 157622ceaddSGao Xiang PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; 158622ceaddSGao Xiang const unsigned int nrpages_in = 159622ceaddSGao Xiang PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; 16010e5f6e4SGao Xiang unsigned int inlen, outlen, pageofs; 161622ceaddSGao Xiang struct z_erofs_lzma *strm; 162622ceaddSGao Xiang u8 *kin; 163622ceaddSGao Xiang bool bounced = false; 164622ceaddSGao Xiang int no, ni, j, err = 0; 165622ceaddSGao Xiang 166622ceaddSGao Xiang /* 1. get the exact LZMA compressed size */ 167622ceaddSGao Xiang kin = kmap(*rq->in); 16810e5f6e4SGao Xiang err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in, 16910e5f6e4SGao Xiang min_t(unsigned int, rq->inputsize, 17010e5f6e4SGao Xiang EROFS_BLKSIZ - rq->pageofs_in)); 17110e5f6e4SGao Xiang if (err) { 172622ceaddSGao Xiang kunmap(*rq->in); 17310e5f6e4SGao Xiang return err; 174622ceaddSGao Xiang } 175622ceaddSGao Xiang 176622ceaddSGao Xiang /* 2. get an available lzma context */ 177622ceaddSGao Xiang again: 178622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock); 179622ceaddSGao Xiang strm = z_erofs_lzma_head; 180622ceaddSGao Xiang if (!strm) { 181622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 182622ceaddSGao Xiang wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); 183622ceaddSGao Xiang goto again; 184622ceaddSGao Xiang } 185622ceaddSGao Xiang z_erofs_lzma_head = strm->next; 186622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 187622ceaddSGao Xiang 188622ceaddSGao Xiang /* 3. multi-call decompress */ 189622ceaddSGao Xiang inlen = rq->inputsize; 190622ceaddSGao Xiang outlen = rq->outputsize; 191622ceaddSGao Xiang xz_dec_microlzma_reset(strm->state, inlen, outlen, 192622ceaddSGao Xiang !rq->partial_decoding); 193622ceaddSGao Xiang pageofs = rq->pageofs_out; 19410e5f6e4SGao Xiang strm->buf.in = kin + rq->pageofs_in; 195622ceaddSGao Xiang strm->buf.in_pos = 0; 19610e5f6e4SGao Xiang strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in); 197622ceaddSGao Xiang inlen -= strm->buf.in_size; 198622ceaddSGao Xiang strm->buf.out = NULL; 199622ceaddSGao Xiang strm->buf.out_pos = 0; 200622ceaddSGao Xiang strm->buf.out_size = 0; 201622ceaddSGao Xiang 202622ceaddSGao Xiang for (ni = 0, no = -1;;) { 203622ceaddSGao Xiang enum xz_ret xz_err; 204622ceaddSGao Xiang 205622ceaddSGao Xiang if (strm->buf.out_pos == strm->buf.out_size) { 206622ceaddSGao Xiang if (strm->buf.out) { 207622ceaddSGao Xiang kunmap(rq->out[no]); 208622ceaddSGao Xiang strm->buf.out = NULL; 209622ceaddSGao Xiang } 210622ceaddSGao Xiang 211622ceaddSGao Xiang if (++no >= nrpages_out || !outlen) { 212622ceaddSGao Xiang erofs_err(rq->sb, "decompressed buf out of bound"); 213622ceaddSGao Xiang err = -EFSCORRUPTED; 214622ceaddSGao Xiang break; 215622ceaddSGao Xiang } 216622ceaddSGao Xiang strm->buf.out_pos = 0; 217622ceaddSGao Xiang strm->buf.out_size = min_t(u32, outlen, 218622ceaddSGao Xiang PAGE_SIZE - pageofs); 219622ceaddSGao Xiang outlen -= strm->buf.out_size; 2205c2a6425SGao Xiang if (!rq->out[no] && rq->fillgaps) /* deduped */ 2215c2a6425SGao Xiang rq->out[no] = erofs_allocpage(pagepool, 2225c2a6425SGao Xiang GFP_KERNEL | __GFP_NOFAIL); 223622ceaddSGao Xiang if (rq->out[no]) 224622ceaddSGao Xiang strm->buf.out = kmap(rq->out[no]) + pageofs; 225622ceaddSGao Xiang pageofs = 0; 226622ceaddSGao Xiang } else if (strm->buf.in_pos == strm->buf.in_size) { 227622ceaddSGao Xiang kunmap(rq->in[ni]); 228622ceaddSGao Xiang 229622ceaddSGao Xiang if (++ni >= nrpages_in || !inlen) { 230622ceaddSGao Xiang erofs_err(rq->sb, "compressed buf out of bound"); 231622ceaddSGao Xiang err = -EFSCORRUPTED; 232622ceaddSGao Xiang break; 233622ceaddSGao Xiang } 234622ceaddSGao Xiang strm->buf.in_pos = 0; 235622ceaddSGao Xiang strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE); 236622ceaddSGao Xiang inlen -= strm->buf.in_size; 237622ceaddSGao Xiang kin = kmap(rq->in[ni]); 238622ceaddSGao Xiang strm->buf.in = kin; 239622ceaddSGao Xiang bounced = false; 240622ceaddSGao Xiang } 241622ceaddSGao Xiang 242622ceaddSGao Xiang /* 243622ceaddSGao Xiang * Handle overlapping: Use bounced buffer if the compressed 244622ceaddSGao Xiang * data is under processing; Otherwise, Use short-lived pages 245622ceaddSGao Xiang * from the on-stack pagepool where pages share with the same 246622ceaddSGao Xiang * request. 247622ceaddSGao Xiang */ 248622ceaddSGao Xiang if (!bounced && rq->out[no] == rq->in[ni]) { 249622ceaddSGao Xiang memcpy(strm->bounce, strm->buf.in, strm->buf.in_size); 250622ceaddSGao Xiang strm->buf.in = strm->bounce; 251622ceaddSGao Xiang bounced = true; 252622ceaddSGao Xiang } 253622ceaddSGao Xiang for (j = ni + 1; j < nrpages_in; ++j) { 254622ceaddSGao Xiang struct page *tmppage; 255622ceaddSGao Xiang 256622ceaddSGao Xiang if (rq->out[no] != rq->in[j]) 257622ceaddSGao Xiang continue; 258622ceaddSGao Xiang 259622ceaddSGao Xiang DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb), 260622ceaddSGao Xiang rq->in[j])); 261622ceaddSGao Xiang tmppage = erofs_allocpage(pagepool, 262622ceaddSGao Xiang GFP_KERNEL | __GFP_NOFAIL); 263622ceaddSGao Xiang set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); 264622ceaddSGao Xiang copy_highpage(tmppage, rq->in[j]); 265622ceaddSGao Xiang rq->in[j] = tmppage; 266622ceaddSGao Xiang } 267622ceaddSGao Xiang xz_err = xz_dec_microlzma_run(strm->state, &strm->buf); 268622ceaddSGao Xiang DBG_BUGON(strm->buf.out_pos > strm->buf.out_size); 269622ceaddSGao Xiang DBG_BUGON(strm->buf.in_pos > strm->buf.in_size); 270622ceaddSGao Xiang 271622ceaddSGao Xiang if (xz_err != XZ_OK) { 272622ceaddSGao Xiang if (xz_err == XZ_STREAM_END && !outlen) 273622ceaddSGao Xiang break; 274622ceaddSGao Xiang erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]", 275622ceaddSGao Xiang xz_err, rq->inputsize, rq->outputsize); 276622ceaddSGao Xiang err = -EFSCORRUPTED; 277622ceaddSGao Xiang break; 278622ceaddSGao Xiang } 279622ceaddSGao Xiang } 280622ceaddSGao Xiang if (no < nrpages_out && strm->buf.out) 281622ceaddSGao Xiang kunmap(rq->in[no]); 282622ceaddSGao Xiang if (ni < nrpages_in) 283622ceaddSGao Xiang kunmap(rq->in[ni]); 284622ceaddSGao Xiang /* 4. push back LZMA stream context to the global list */ 285622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock); 286622ceaddSGao Xiang strm->next = z_erofs_lzma_head; 287622ceaddSGao Xiang z_erofs_lzma_head = strm; 288622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock); 289622ceaddSGao Xiang wake_up(&z_erofs_lzma_wq); 290622ceaddSGao Xiang return err; 291622ceaddSGao Xiang } 292