1622ceaddSGao Xiang // SPDX-License-Identifier: GPL-2.0-or-later
2622ceaddSGao Xiang #include <linux/xz.h>
3622ceaddSGao Xiang #include <linux/module.h>
4622ceaddSGao Xiang #include "compress.h"
5622ceaddSGao Xiang
6622ceaddSGao Xiang struct z_erofs_lzma {
7622ceaddSGao Xiang struct z_erofs_lzma *next;
8622ceaddSGao Xiang struct xz_dec_microlzma *state;
9622ceaddSGao Xiang struct xz_buf buf;
10622ceaddSGao Xiang u8 bounce[PAGE_SIZE];
11622ceaddSGao Xiang };
12622ceaddSGao Xiang
13622ceaddSGao Xiang /* considering the LZMA performance, no need to use a lockless list for now */
14622ceaddSGao Xiang static DEFINE_SPINLOCK(z_erofs_lzma_lock);
15622ceaddSGao Xiang static unsigned int z_erofs_lzma_max_dictsize;
16622ceaddSGao Xiang static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
17622ceaddSGao Xiang static struct z_erofs_lzma *z_erofs_lzma_head;
18622ceaddSGao Xiang static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
19622ceaddSGao Xiang
20622ceaddSGao Xiang module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
21622ceaddSGao Xiang
z_erofs_lzma_exit(void)22622ceaddSGao Xiang void z_erofs_lzma_exit(void)
23622ceaddSGao Xiang {
24622ceaddSGao Xiang /* there should be no running fs instance */
25622ceaddSGao Xiang while (z_erofs_lzma_avail_strms) {
26622ceaddSGao Xiang struct z_erofs_lzma *strm;
27622ceaddSGao Xiang
28622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock);
29622ceaddSGao Xiang strm = z_erofs_lzma_head;
30622ceaddSGao Xiang if (!strm) {
31622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
32622ceaddSGao Xiang DBG_BUGON(1);
33622ceaddSGao Xiang return;
34622ceaddSGao Xiang }
35622ceaddSGao Xiang z_erofs_lzma_head = NULL;
36622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
37622ceaddSGao Xiang
38622ceaddSGao Xiang while (strm) {
39622ceaddSGao Xiang struct z_erofs_lzma *n = strm->next;
40622ceaddSGao Xiang
41622ceaddSGao Xiang if (strm->state)
42622ceaddSGao Xiang xz_dec_microlzma_end(strm->state);
43622ceaddSGao Xiang kfree(strm);
44622ceaddSGao Xiang --z_erofs_lzma_avail_strms;
45622ceaddSGao Xiang strm = n;
46622ceaddSGao Xiang }
47622ceaddSGao Xiang }
48622ceaddSGao Xiang }
49622ceaddSGao Xiang
z_erofs_lzma_init(void)50a279adedSYangtao Li int __init z_erofs_lzma_init(void)
51622ceaddSGao Xiang {
52622ceaddSGao Xiang unsigned int i;
53622ceaddSGao Xiang
54622ceaddSGao Xiang /* by default, use # of possible CPUs instead */
55622ceaddSGao Xiang if (!z_erofs_lzma_nstrms)
56622ceaddSGao Xiang z_erofs_lzma_nstrms = num_possible_cpus();
57622ceaddSGao Xiang
58622ceaddSGao Xiang for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
59622ceaddSGao Xiang struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
60622ceaddSGao Xiang
61622ceaddSGao Xiang if (!strm) {
62622ceaddSGao Xiang z_erofs_lzma_exit();
63622ceaddSGao Xiang return -ENOMEM;
64622ceaddSGao Xiang }
65622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock);
66622ceaddSGao Xiang strm->next = z_erofs_lzma_head;
67622ceaddSGao Xiang z_erofs_lzma_head = strm;
68622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
69622ceaddSGao Xiang ++z_erofs_lzma_avail_strms;
70622ceaddSGao Xiang }
71622ceaddSGao Xiang return 0;
72622ceaddSGao Xiang }
73622ceaddSGao Xiang
z_erofs_load_lzma_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)74622ceaddSGao Xiang int z_erofs_load_lzma_config(struct super_block *sb,
75*586814edSGao Xiang struct erofs_super_block *dsb, void *data, int size)
76622ceaddSGao Xiang {
77622ceaddSGao Xiang static DEFINE_MUTEX(lzma_resize_mutex);
78*586814edSGao Xiang struct z_erofs_lzma_cfgs *lzma = data;
79622ceaddSGao Xiang unsigned int dict_size, i;
80622ceaddSGao Xiang struct z_erofs_lzma *strm, *head = NULL;
81622ceaddSGao Xiang int err;
82622ceaddSGao Xiang
83622ceaddSGao Xiang if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
84622ceaddSGao Xiang erofs_err(sb, "invalid lzma cfgs, size=%u", size);
85622ceaddSGao Xiang return -EINVAL;
86622ceaddSGao Xiang }
87622ceaddSGao Xiang if (lzma->format) {
88622ceaddSGao Xiang erofs_err(sb, "unidentified lzma format %x, please check kernel version",
89622ceaddSGao Xiang le16_to_cpu(lzma->format));
90622ceaddSGao Xiang return -EINVAL;
91622ceaddSGao Xiang }
92622ceaddSGao Xiang dict_size = le32_to_cpu(lzma->dict_size);
93622ceaddSGao Xiang if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
94622ceaddSGao Xiang erofs_err(sb, "unsupported lzma dictionary size %u",
95622ceaddSGao Xiang dict_size);
96622ceaddSGao Xiang return -EINVAL;
97622ceaddSGao Xiang }
98622ceaddSGao Xiang
99622ceaddSGao Xiang erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!");
100622ceaddSGao Xiang
101622ceaddSGao Xiang /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
102622ceaddSGao Xiang mutex_lock(&lzma_resize_mutex);
103622ceaddSGao Xiang
104622ceaddSGao Xiang if (z_erofs_lzma_max_dictsize >= dict_size) {
105622ceaddSGao Xiang mutex_unlock(&lzma_resize_mutex);
106622ceaddSGao Xiang return 0;
107622ceaddSGao Xiang }
108622ceaddSGao Xiang
109622ceaddSGao Xiang /* 1. collect/isolate all streams for the following check */
110622ceaddSGao Xiang for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
111622ceaddSGao Xiang struct z_erofs_lzma *last;
112622ceaddSGao Xiang
113622ceaddSGao Xiang again:
114622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock);
115622ceaddSGao Xiang strm = z_erofs_lzma_head;
116622ceaddSGao Xiang if (!strm) {
117622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
118622ceaddSGao Xiang wait_event(z_erofs_lzma_wq,
119622ceaddSGao Xiang READ_ONCE(z_erofs_lzma_head));
120622ceaddSGao Xiang goto again;
121622ceaddSGao Xiang }
122622ceaddSGao Xiang z_erofs_lzma_head = NULL;
123622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
124622ceaddSGao Xiang
125622ceaddSGao Xiang for (last = strm; last->next; last = last->next)
126622ceaddSGao Xiang ++i;
127622ceaddSGao Xiang last->next = head;
128622ceaddSGao Xiang head = strm;
129622ceaddSGao Xiang }
130622ceaddSGao Xiang
131622ceaddSGao Xiang err = 0;
132622ceaddSGao Xiang /* 2. walk each isolated stream and grow max dict_size if needed */
133622ceaddSGao Xiang for (strm = head; strm; strm = strm->next) {
134622ceaddSGao Xiang if (strm->state)
135622ceaddSGao Xiang xz_dec_microlzma_end(strm->state);
136622ceaddSGao Xiang strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
137622ceaddSGao Xiang if (!strm->state)
138622ceaddSGao Xiang err = -ENOMEM;
139622ceaddSGao Xiang }
140622ceaddSGao Xiang
141622ceaddSGao Xiang /* 3. push back all to the global list and update max dict_size */
142622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock);
143622ceaddSGao Xiang DBG_BUGON(z_erofs_lzma_head);
144622ceaddSGao Xiang z_erofs_lzma_head = head;
145622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
1462df7c4bdSYuwen Chen wake_up_all(&z_erofs_lzma_wq);
147622ceaddSGao Xiang
148622ceaddSGao Xiang z_erofs_lzma_max_dictsize = dict_size;
149622ceaddSGao Xiang mutex_unlock(&lzma_resize_mutex);
150622ceaddSGao Xiang return err;
151622ceaddSGao Xiang }
152622ceaddSGao Xiang
z_erofs_lzma_decompress(struct z_erofs_decompress_req * rq,struct page ** pagepool)153622ceaddSGao Xiang int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
154eaa9172aSGao Xiang struct page **pagepool)
155622ceaddSGao Xiang {
156622ceaddSGao Xiang const unsigned int nrpages_out =
157622ceaddSGao Xiang PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
158622ceaddSGao Xiang const unsigned int nrpages_in =
159622ceaddSGao Xiang PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
16010e5f6e4SGao Xiang unsigned int inlen, outlen, pageofs;
161622ceaddSGao Xiang struct z_erofs_lzma *strm;
162622ceaddSGao Xiang u8 *kin;
163622ceaddSGao Xiang bool bounced = false;
164622ceaddSGao Xiang int no, ni, j, err = 0;
165622ceaddSGao Xiang
166622ceaddSGao Xiang /* 1. get the exact LZMA compressed size */
167622ceaddSGao Xiang kin = kmap(*rq->in);
16810e5f6e4SGao Xiang err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
16910e5f6e4SGao Xiang min_t(unsigned int, rq->inputsize,
1703acea5fcSJingbo Xu rq->sb->s_blocksize - rq->pageofs_in));
17110e5f6e4SGao Xiang if (err) {
172622ceaddSGao Xiang kunmap(*rq->in);
17310e5f6e4SGao Xiang return err;
174622ceaddSGao Xiang }
175622ceaddSGao Xiang
176622ceaddSGao Xiang /* 2. get an available lzma context */
177622ceaddSGao Xiang again:
178622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock);
179622ceaddSGao Xiang strm = z_erofs_lzma_head;
180622ceaddSGao Xiang if (!strm) {
181622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
182622ceaddSGao Xiang wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
183622ceaddSGao Xiang goto again;
184622ceaddSGao Xiang }
185622ceaddSGao Xiang z_erofs_lzma_head = strm->next;
186622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
187622ceaddSGao Xiang
188622ceaddSGao Xiang /* 3. multi-call decompress */
189622ceaddSGao Xiang inlen = rq->inputsize;
190622ceaddSGao Xiang outlen = rq->outputsize;
191622ceaddSGao Xiang xz_dec_microlzma_reset(strm->state, inlen, outlen,
192622ceaddSGao Xiang !rq->partial_decoding);
193622ceaddSGao Xiang pageofs = rq->pageofs_out;
19410e5f6e4SGao Xiang strm->buf.in = kin + rq->pageofs_in;
195622ceaddSGao Xiang strm->buf.in_pos = 0;
19610e5f6e4SGao Xiang strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
197622ceaddSGao Xiang inlen -= strm->buf.in_size;
198622ceaddSGao Xiang strm->buf.out = NULL;
199622ceaddSGao Xiang strm->buf.out_pos = 0;
200622ceaddSGao Xiang strm->buf.out_size = 0;
201622ceaddSGao Xiang
202622ceaddSGao Xiang for (ni = 0, no = -1;;) {
203622ceaddSGao Xiang enum xz_ret xz_err;
204622ceaddSGao Xiang
205622ceaddSGao Xiang if (strm->buf.out_pos == strm->buf.out_size) {
206622ceaddSGao Xiang if (strm->buf.out) {
207622ceaddSGao Xiang kunmap(rq->out[no]);
208622ceaddSGao Xiang strm->buf.out = NULL;
209622ceaddSGao Xiang }
210622ceaddSGao Xiang
211622ceaddSGao Xiang if (++no >= nrpages_out || !outlen) {
212622ceaddSGao Xiang erofs_err(rq->sb, "decompressed buf out of bound");
213622ceaddSGao Xiang err = -EFSCORRUPTED;
214622ceaddSGao Xiang break;
215622ceaddSGao Xiang }
216622ceaddSGao Xiang strm->buf.out_pos = 0;
217622ceaddSGao Xiang strm->buf.out_size = min_t(u32, outlen,
218622ceaddSGao Xiang PAGE_SIZE - pageofs);
219622ceaddSGao Xiang outlen -= strm->buf.out_size;
22075a52216SGao Xiang if (!rq->out[no] && rq->fillgaps) { /* deduped */
2215c2a6425SGao Xiang rq->out[no] = erofs_allocpage(pagepool,
2225c2a6425SGao Xiang GFP_KERNEL | __GFP_NOFAIL);
22375a52216SGao Xiang set_page_private(rq->out[no],
22475a52216SGao Xiang Z_EROFS_SHORTLIVED_PAGE);
22575a52216SGao Xiang }
226622ceaddSGao Xiang if (rq->out[no])
227622ceaddSGao Xiang strm->buf.out = kmap(rq->out[no]) + pageofs;
228622ceaddSGao Xiang pageofs = 0;
229622ceaddSGao Xiang } else if (strm->buf.in_pos == strm->buf.in_size) {
230622ceaddSGao Xiang kunmap(rq->in[ni]);
231622ceaddSGao Xiang
232622ceaddSGao Xiang if (++ni >= nrpages_in || !inlen) {
233622ceaddSGao Xiang erofs_err(rq->sb, "compressed buf out of bound");
234622ceaddSGao Xiang err = -EFSCORRUPTED;
235622ceaddSGao Xiang break;
236622ceaddSGao Xiang }
237622ceaddSGao Xiang strm->buf.in_pos = 0;
238622ceaddSGao Xiang strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
239622ceaddSGao Xiang inlen -= strm->buf.in_size;
240622ceaddSGao Xiang kin = kmap(rq->in[ni]);
241622ceaddSGao Xiang strm->buf.in = kin;
242622ceaddSGao Xiang bounced = false;
243622ceaddSGao Xiang }
244622ceaddSGao Xiang
245622ceaddSGao Xiang /*
246622ceaddSGao Xiang * Handle overlapping: Use bounced buffer if the compressed
247622ceaddSGao Xiang * data is under processing; Otherwise, Use short-lived pages
248622ceaddSGao Xiang * from the on-stack pagepool where pages share with the same
249622ceaddSGao Xiang * request.
250622ceaddSGao Xiang */
251622ceaddSGao Xiang if (!bounced && rq->out[no] == rq->in[ni]) {
252622ceaddSGao Xiang memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
253622ceaddSGao Xiang strm->buf.in = strm->bounce;
254622ceaddSGao Xiang bounced = true;
255622ceaddSGao Xiang }
256622ceaddSGao Xiang for (j = ni + 1; j < nrpages_in; ++j) {
257622ceaddSGao Xiang struct page *tmppage;
258622ceaddSGao Xiang
259622ceaddSGao Xiang if (rq->out[no] != rq->in[j])
260622ceaddSGao Xiang continue;
261622ceaddSGao Xiang
262622ceaddSGao Xiang DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
263622ceaddSGao Xiang rq->in[j]));
264622ceaddSGao Xiang tmppage = erofs_allocpage(pagepool,
265622ceaddSGao Xiang GFP_KERNEL | __GFP_NOFAIL);
266622ceaddSGao Xiang set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
267622ceaddSGao Xiang copy_highpage(tmppage, rq->in[j]);
268622ceaddSGao Xiang rq->in[j] = tmppage;
269622ceaddSGao Xiang }
270622ceaddSGao Xiang xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
271622ceaddSGao Xiang DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
272622ceaddSGao Xiang DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
273622ceaddSGao Xiang
274622ceaddSGao Xiang if (xz_err != XZ_OK) {
275622ceaddSGao Xiang if (xz_err == XZ_STREAM_END && !outlen)
276622ceaddSGao Xiang break;
277622ceaddSGao Xiang erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
278622ceaddSGao Xiang xz_err, rq->inputsize, rq->outputsize);
279622ceaddSGao Xiang err = -EFSCORRUPTED;
280622ceaddSGao Xiang break;
281622ceaddSGao Xiang }
282622ceaddSGao Xiang }
283622ceaddSGao Xiang if (no < nrpages_out && strm->buf.out)
2848f121dfbSGao Xiang kunmap(rq->out[no]);
285622ceaddSGao Xiang if (ni < nrpages_in)
286622ceaddSGao Xiang kunmap(rq->in[ni]);
287622ceaddSGao Xiang /* 4. push back LZMA stream context to the global list */
288622ceaddSGao Xiang spin_lock(&z_erofs_lzma_lock);
289622ceaddSGao Xiang strm->next = z_erofs_lzma_head;
290622ceaddSGao Xiang z_erofs_lzma_head = strm;
291622ceaddSGao Xiang spin_unlock(&z_erofs_lzma_lock);
292622ceaddSGao Xiang wake_up(&z_erofs_lzma_wq);
293622ceaddSGao Xiang return err;
294622ceaddSGao Xiang }
295