1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/xz.h> 3 #include <linux/module.h> 4 #include "compress.h" 5 6 struct z_erofs_lzma { 7 struct z_erofs_lzma *next; 8 struct xz_dec_microlzma *state; 9 struct xz_buf buf; 10 u8 bounce[PAGE_SIZE]; 11 }; 12 13 /* considering the LZMA performance, no need to use a lockless list for now */ 14 static DEFINE_SPINLOCK(z_erofs_lzma_lock); 15 static unsigned int z_erofs_lzma_max_dictsize; 16 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; 17 static struct z_erofs_lzma *z_erofs_lzma_head; 18 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); 19 20 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); 21 22 void z_erofs_lzma_exit(void) 23 { 24 /* there should be no running fs instance */ 25 while (z_erofs_lzma_avail_strms) { 26 struct z_erofs_lzma *strm; 27 28 spin_lock(&z_erofs_lzma_lock); 29 strm = z_erofs_lzma_head; 30 if (!strm) { 31 spin_unlock(&z_erofs_lzma_lock); 32 DBG_BUGON(1); 33 return; 34 } 35 z_erofs_lzma_head = NULL; 36 spin_unlock(&z_erofs_lzma_lock); 37 38 while (strm) { 39 struct z_erofs_lzma *n = strm->next; 40 41 if (strm->state) 42 xz_dec_microlzma_end(strm->state); 43 kfree(strm); 44 --z_erofs_lzma_avail_strms; 45 strm = n; 46 } 47 } 48 } 49 50 int z_erofs_lzma_init(void) 51 { 52 unsigned int i; 53 54 /* by default, use # of possible CPUs instead */ 55 if (!z_erofs_lzma_nstrms) 56 z_erofs_lzma_nstrms = num_possible_cpus(); 57 58 for (i = 0; i < z_erofs_lzma_nstrms; ++i) { 59 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL); 60 61 if (!strm) { 62 z_erofs_lzma_exit(); 63 return -ENOMEM; 64 } 65 spin_lock(&z_erofs_lzma_lock); 66 strm->next = z_erofs_lzma_head; 67 z_erofs_lzma_head = strm; 68 spin_unlock(&z_erofs_lzma_lock); 69 ++z_erofs_lzma_avail_strms; 70 } 71 return 0; 72 } 73 74 int z_erofs_load_lzma_config(struct super_block *sb, 75 struct erofs_super_block *dsb, 76 struct z_erofs_lzma_cfgs *lzma, int size) 77 { 78 static DEFINE_MUTEX(lzma_resize_mutex); 79 unsigned int dict_size, i; 80 struct z_erofs_lzma *strm, *head = NULL; 81 int err; 82 83 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { 84 erofs_err(sb, "invalid lzma cfgs, size=%u", size); 85 return -EINVAL; 86 } 87 if (lzma->format) { 88 erofs_err(sb, "unidentified lzma format %x, please check kernel version", 89 le16_to_cpu(lzma->format)); 90 return -EINVAL; 91 } 92 dict_size = le32_to_cpu(lzma->dict_size); 93 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { 94 erofs_err(sb, "unsupported lzma dictionary size %u", 95 dict_size); 96 return -EINVAL; 97 } 98 99 erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!"); 100 101 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ 102 mutex_lock(&lzma_resize_mutex); 103 104 if (z_erofs_lzma_max_dictsize >= dict_size) { 105 mutex_unlock(&lzma_resize_mutex); 106 return 0; 107 } 108 109 /* 1. collect/isolate all streams for the following check */ 110 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { 111 struct z_erofs_lzma *last; 112 113 again: 114 spin_lock(&z_erofs_lzma_lock); 115 strm = z_erofs_lzma_head; 116 if (!strm) { 117 spin_unlock(&z_erofs_lzma_lock); 118 wait_event(z_erofs_lzma_wq, 119 READ_ONCE(z_erofs_lzma_head)); 120 goto again; 121 } 122 z_erofs_lzma_head = NULL; 123 spin_unlock(&z_erofs_lzma_lock); 124 125 for (last = strm; last->next; last = last->next) 126 ++i; 127 last->next = head; 128 head = strm; 129 } 130 131 err = 0; 132 /* 2. walk each isolated stream and grow max dict_size if needed */ 133 for (strm = head; strm; strm = strm->next) { 134 if (strm->state) 135 xz_dec_microlzma_end(strm->state); 136 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); 137 if (!strm->state) 138 err = -ENOMEM; 139 } 140 141 /* 3. push back all to the global list and update max dict_size */ 142 spin_lock(&z_erofs_lzma_lock); 143 DBG_BUGON(z_erofs_lzma_head); 144 z_erofs_lzma_head = head; 145 spin_unlock(&z_erofs_lzma_lock); 146 wake_up_all(&z_erofs_lzma_wq); 147 148 z_erofs_lzma_max_dictsize = dict_size; 149 mutex_unlock(&lzma_resize_mutex); 150 return err; 151 } 152 153 int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, 154 struct page **pagepool) 155 { 156 const unsigned int nrpages_out = 157 PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT; 158 const unsigned int nrpages_in = 159 PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT; 160 unsigned int inlen, outlen, pageofs; 161 struct z_erofs_lzma *strm; 162 u8 *kin; 163 bool bounced = false; 164 int no, ni, j, err = 0; 165 166 /* 1. get the exact LZMA compressed size */ 167 kin = kmap(*rq->in); 168 err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in, 169 min_t(unsigned int, rq->inputsize, 170 EROFS_BLKSIZ - rq->pageofs_in)); 171 if (err) { 172 kunmap(*rq->in); 173 return err; 174 } 175 176 /* 2. get an available lzma context */ 177 again: 178 spin_lock(&z_erofs_lzma_lock); 179 strm = z_erofs_lzma_head; 180 if (!strm) { 181 spin_unlock(&z_erofs_lzma_lock); 182 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); 183 goto again; 184 } 185 z_erofs_lzma_head = strm->next; 186 spin_unlock(&z_erofs_lzma_lock); 187 188 /* 3. multi-call decompress */ 189 inlen = rq->inputsize; 190 outlen = rq->outputsize; 191 xz_dec_microlzma_reset(strm->state, inlen, outlen, 192 !rq->partial_decoding); 193 pageofs = rq->pageofs_out; 194 strm->buf.in = kin + rq->pageofs_in; 195 strm->buf.in_pos = 0; 196 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in); 197 inlen -= strm->buf.in_size; 198 strm->buf.out = NULL; 199 strm->buf.out_pos = 0; 200 strm->buf.out_size = 0; 201 202 for (ni = 0, no = -1;;) { 203 enum xz_ret xz_err; 204 205 if (strm->buf.out_pos == strm->buf.out_size) { 206 if (strm->buf.out) { 207 kunmap(rq->out[no]); 208 strm->buf.out = NULL; 209 } 210 211 if (++no >= nrpages_out || !outlen) { 212 erofs_err(rq->sb, "decompressed buf out of bound"); 213 err = -EFSCORRUPTED; 214 break; 215 } 216 strm->buf.out_pos = 0; 217 strm->buf.out_size = min_t(u32, outlen, 218 PAGE_SIZE - pageofs); 219 outlen -= strm->buf.out_size; 220 if (!rq->out[no] && rq->fillgaps) /* deduped */ 221 rq->out[no] = erofs_allocpage(pagepool, 222 GFP_KERNEL | __GFP_NOFAIL); 223 if (rq->out[no]) 224 strm->buf.out = kmap(rq->out[no]) + pageofs; 225 pageofs = 0; 226 } else if (strm->buf.in_pos == strm->buf.in_size) { 227 kunmap(rq->in[ni]); 228 229 if (++ni >= nrpages_in || !inlen) { 230 erofs_err(rq->sb, "compressed buf out of bound"); 231 err = -EFSCORRUPTED; 232 break; 233 } 234 strm->buf.in_pos = 0; 235 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE); 236 inlen -= strm->buf.in_size; 237 kin = kmap(rq->in[ni]); 238 strm->buf.in = kin; 239 bounced = false; 240 } 241 242 /* 243 * Handle overlapping: Use bounced buffer if the compressed 244 * data is under processing; Otherwise, Use short-lived pages 245 * from the on-stack pagepool where pages share with the same 246 * request. 247 */ 248 if (!bounced && rq->out[no] == rq->in[ni]) { 249 memcpy(strm->bounce, strm->buf.in, strm->buf.in_size); 250 strm->buf.in = strm->bounce; 251 bounced = true; 252 } 253 for (j = ni + 1; j < nrpages_in; ++j) { 254 struct page *tmppage; 255 256 if (rq->out[no] != rq->in[j]) 257 continue; 258 259 DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb), 260 rq->in[j])); 261 tmppage = erofs_allocpage(pagepool, 262 GFP_KERNEL | __GFP_NOFAIL); 263 set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); 264 copy_highpage(tmppage, rq->in[j]); 265 rq->in[j] = tmppage; 266 } 267 xz_err = xz_dec_microlzma_run(strm->state, &strm->buf); 268 DBG_BUGON(strm->buf.out_pos > strm->buf.out_size); 269 DBG_BUGON(strm->buf.in_pos > strm->buf.in_size); 270 271 if (xz_err != XZ_OK) { 272 if (xz_err == XZ_STREAM_END && !outlen) 273 break; 274 erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]", 275 xz_err, rq->inputsize, rq->outputsize); 276 err = -EFSCORRUPTED; 277 break; 278 } 279 } 280 if (no < nrpages_out && strm->buf.out) 281 kunmap(rq->in[no]); 282 if (ni < nrpages_in) 283 kunmap(rq->in[ni]); 284 /* 4. push back LZMA stream context to the global list */ 285 spin_lock(&z_erofs_lzma_lock); 286 strm->next = z_erofs_lzma_head; 287 z_erofs_lzma_head = strm; 288 spin_unlock(&z_erofs_lzma_lock); 289 wake_up(&z_erofs_lzma_wq); 290 return err; 291 } 292