1ffa09b3bSGao Xiang // SPDX-License-Identifier: GPL-2.0-or-later
2ffa09b3bSGao Xiang #include <linux/module.h>
3ffa09b3bSGao Xiang #include <linux/zlib.h>
4ffa09b3bSGao Xiang #include "compress.h"
5ffa09b3bSGao Xiang
6ffa09b3bSGao Xiang struct z_erofs_deflate {
7ffa09b3bSGao Xiang struct z_erofs_deflate *next;
8ffa09b3bSGao Xiang struct z_stream_s z;
9ffa09b3bSGao Xiang u8 bounce[PAGE_SIZE];
10ffa09b3bSGao Xiang };
11ffa09b3bSGao Xiang
12ffa09b3bSGao Xiang static DEFINE_SPINLOCK(z_erofs_deflate_lock);
13ffa09b3bSGao Xiang static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms;
14ffa09b3bSGao Xiang static struct z_erofs_deflate *z_erofs_deflate_head;
15ffa09b3bSGao Xiang static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq);
16ffa09b3bSGao Xiang
17ffa09b3bSGao Xiang module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444);
18ffa09b3bSGao Xiang
z_erofs_deflate_exit(void)19ffa09b3bSGao Xiang void z_erofs_deflate_exit(void)
20ffa09b3bSGao Xiang {
21ffa09b3bSGao Xiang /* there should be no running fs instance */
22ffa09b3bSGao Xiang while (z_erofs_deflate_avail_strms) {
23ffa09b3bSGao Xiang struct z_erofs_deflate *strm;
24ffa09b3bSGao Xiang
25ffa09b3bSGao Xiang spin_lock(&z_erofs_deflate_lock);
26ffa09b3bSGao Xiang strm = z_erofs_deflate_head;
27ffa09b3bSGao Xiang if (!strm) {
28ffa09b3bSGao Xiang spin_unlock(&z_erofs_deflate_lock);
29ffa09b3bSGao Xiang continue;
30ffa09b3bSGao Xiang }
31ffa09b3bSGao Xiang z_erofs_deflate_head = NULL;
32ffa09b3bSGao Xiang spin_unlock(&z_erofs_deflate_lock);
33ffa09b3bSGao Xiang
34ffa09b3bSGao Xiang while (strm) {
35ffa09b3bSGao Xiang struct z_erofs_deflate *n = strm->next;
36ffa09b3bSGao Xiang
37ffa09b3bSGao Xiang vfree(strm->z.workspace);
38ffa09b3bSGao Xiang kfree(strm);
39ffa09b3bSGao Xiang --z_erofs_deflate_avail_strms;
40ffa09b3bSGao Xiang strm = n;
41ffa09b3bSGao Xiang }
42ffa09b3bSGao Xiang }
43ffa09b3bSGao Xiang }
44ffa09b3bSGao Xiang
z_erofs_deflate_init(void)45ffa09b3bSGao Xiang int __init z_erofs_deflate_init(void)
46ffa09b3bSGao Xiang {
47ffa09b3bSGao Xiang /* by default, use # of possible CPUs instead */
48ffa09b3bSGao Xiang if (!z_erofs_deflate_nstrms)
49ffa09b3bSGao Xiang z_erofs_deflate_nstrms = num_possible_cpus();
50ffa09b3bSGao Xiang return 0;
51ffa09b3bSGao Xiang }
52ffa09b3bSGao Xiang
z_erofs_load_deflate_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)53ffa09b3bSGao Xiang int z_erofs_load_deflate_config(struct super_block *sb,
54586814edSGao Xiang struct erofs_super_block *dsb, void *data, int size)
55ffa09b3bSGao Xiang {
56586814edSGao Xiang struct z_erofs_deflate_cfgs *dfl = data;
5715cc248eSGao Xiang static DEFINE_MUTEX(deflate_resize_mutex);
5815cc248eSGao Xiang static bool inited;
59586814edSGao Xiang
60ffa09b3bSGao Xiang if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) {
61ffa09b3bSGao Xiang erofs_err(sb, "invalid deflate cfgs, size=%u", size);
62ffa09b3bSGao Xiang return -EINVAL;
63ffa09b3bSGao Xiang }
64ffa09b3bSGao Xiang
65ffa09b3bSGao Xiang if (dfl->windowbits > MAX_WBITS) {
66ffa09b3bSGao Xiang erofs_err(sb, "unsupported windowbits %u", dfl->windowbits);
67ffa09b3bSGao Xiang return -EOPNOTSUPP;
68ffa09b3bSGao Xiang }
6915cc248eSGao Xiang mutex_lock(&deflate_resize_mutex);
7015cc248eSGao Xiang if (!inited) {
7115cc248eSGao Xiang for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms;
7215cc248eSGao Xiang ++z_erofs_deflate_avail_strms) {
7315cc248eSGao Xiang struct z_erofs_deflate *strm;
74ffa09b3bSGao Xiang
7515cc248eSGao Xiang strm = kzalloc(sizeof(*strm), GFP_KERNEL);
7615cc248eSGao Xiang if (!strm)
7715cc248eSGao Xiang goto failed;
7815cc248eSGao Xiang /* XXX: in-kernel zlib cannot customize windowbits */
7915cc248eSGao Xiang strm->z.workspace = vmalloc(zlib_inflate_workspacesize());
8015cc248eSGao Xiang if (!strm->z.workspace) {
8115cc248eSGao Xiang kfree(strm);
8215cc248eSGao Xiang goto failed;
8315cc248eSGao Xiang }
8415cc248eSGao Xiang
8515cc248eSGao Xiang spin_lock(&z_erofs_deflate_lock);
8615cc248eSGao Xiang strm->next = z_erofs_deflate_head;
8715cc248eSGao Xiang z_erofs_deflate_head = strm;
8815cc248eSGao Xiang spin_unlock(&z_erofs_deflate_lock);
8915cc248eSGao Xiang }
9015cc248eSGao Xiang inited = true;
9115cc248eSGao Xiang }
9215cc248eSGao Xiang mutex_unlock(&deflate_resize_mutex);
93ffa09b3bSGao Xiang erofs_info(sb, "EXPERIMENTAL DEFLATE feature in use. Use at your own risk!");
94ffa09b3bSGao Xiang return 0;
9515cc248eSGao Xiang failed:
9615cc248eSGao Xiang mutex_unlock(&deflate_resize_mutex);
9715cc248eSGao Xiang z_erofs_deflate_exit();
9815cc248eSGao Xiang return -ENOMEM;
99ffa09b3bSGao Xiang }
100ffa09b3bSGao Xiang
z_erofs_deflate_decompress(struct z_erofs_decompress_req * rq,struct page ** pagepool)101ffa09b3bSGao Xiang int z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq,
102ffa09b3bSGao Xiang struct page **pagepool)
103ffa09b3bSGao Xiang {
104ffa09b3bSGao Xiang const unsigned int nrpages_out =
105ffa09b3bSGao Xiang PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
106ffa09b3bSGao Xiang const unsigned int nrpages_in =
107ffa09b3bSGao Xiang PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
108ffa09b3bSGao Xiang struct super_block *sb = rq->sb;
109ffa09b3bSGao Xiang unsigned int insz, outsz, pofs;
110ffa09b3bSGao Xiang struct z_erofs_deflate *strm;
111ffa09b3bSGao Xiang u8 *kin, *kout = NULL;
112ffa09b3bSGao Xiang bool bounced = false;
113ffa09b3bSGao Xiang int no = -1, ni = 0, j = 0, zerr, err;
114ffa09b3bSGao Xiang
115ffa09b3bSGao Xiang /* 1. get the exact DEFLATE compressed size */
116ffa09b3bSGao Xiang kin = kmap_local_page(*rq->in);
117ffa09b3bSGao Xiang err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
118ffa09b3bSGao Xiang min_t(unsigned int, rq->inputsize,
119ffa09b3bSGao Xiang sb->s_blocksize - rq->pageofs_in));
120ffa09b3bSGao Xiang if (err) {
121ffa09b3bSGao Xiang kunmap_local(kin);
122ffa09b3bSGao Xiang return err;
123ffa09b3bSGao Xiang }
124ffa09b3bSGao Xiang
125ffa09b3bSGao Xiang /* 2. get an available DEFLATE context */
126ffa09b3bSGao Xiang again:
127ffa09b3bSGao Xiang spin_lock(&z_erofs_deflate_lock);
128ffa09b3bSGao Xiang strm = z_erofs_deflate_head;
129ffa09b3bSGao Xiang if (!strm) {
130ffa09b3bSGao Xiang spin_unlock(&z_erofs_deflate_lock);
131ffa09b3bSGao Xiang wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head));
132ffa09b3bSGao Xiang goto again;
133ffa09b3bSGao Xiang }
134ffa09b3bSGao Xiang z_erofs_deflate_head = strm->next;
135ffa09b3bSGao Xiang spin_unlock(&z_erofs_deflate_lock);
136ffa09b3bSGao Xiang
137ffa09b3bSGao Xiang /* 3. multi-call decompress */
138ffa09b3bSGao Xiang insz = rq->inputsize;
139ffa09b3bSGao Xiang outsz = rq->outputsize;
140ffa09b3bSGao Xiang zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS);
141ffa09b3bSGao Xiang if (zerr != Z_OK) {
142ffa09b3bSGao Xiang err = -EIO;
143ffa09b3bSGao Xiang goto failed_zinit;
144ffa09b3bSGao Xiang }
145ffa09b3bSGao Xiang
146ffa09b3bSGao Xiang pofs = rq->pageofs_out;
147ffa09b3bSGao Xiang strm->z.avail_in = min_t(u32, insz, PAGE_SIZE - rq->pageofs_in);
148ffa09b3bSGao Xiang insz -= strm->z.avail_in;
149ffa09b3bSGao Xiang strm->z.next_in = kin + rq->pageofs_in;
150ffa09b3bSGao Xiang strm->z.avail_out = 0;
151ffa09b3bSGao Xiang
152ffa09b3bSGao Xiang while (1) {
153ffa09b3bSGao Xiang if (!strm->z.avail_out) {
154ffa09b3bSGao Xiang if (++no >= nrpages_out || !outsz) {
155ffa09b3bSGao Xiang erofs_err(sb, "insufficient space for decompressed data");
156ffa09b3bSGao Xiang err = -EFSCORRUPTED;
157ffa09b3bSGao Xiang break;
158ffa09b3bSGao Xiang }
159ffa09b3bSGao Xiang
160ffa09b3bSGao Xiang if (kout)
161ffa09b3bSGao Xiang kunmap_local(kout);
162ffa09b3bSGao Xiang strm->z.avail_out = min_t(u32, outsz, PAGE_SIZE - pofs);
163ffa09b3bSGao Xiang outsz -= strm->z.avail_out;
164ffa09b3bSGao Xiang if (!rq->out[no]) {
165ffa09b3bSGao Xiang rq->out[no] = erofs_allocpage(pagepool,
166ffa09b3bSGao Xiang GFP_KERNEL | __GFP_NOFAIL);
167ffa09b3bSGao Xiang set_page_private(rq->out[no],
168ffa09b3bSGao Xiang Z_EROFS_SHORTLIVED_PAGE);
169ffa09b3bSGao Xiang }
170ffa09b3bSGao Xiang kout = kmap_local_page(rq->out[no]);
171ffa09b3bSGao Xiang strm->z.next_out = kout + pofs;
172ffa09b3bSGao Xiang pofs = 0;
173ffa09b3bSGao Xiang }
174ffa09b3bSGao Xiang
175ffa09b3bSGao Xiang if (!strm->z.avail_in && insz) {
176ffa09b3bSGao Xiang if (++ni >= nrpages_in) {
177ffa09b3bSGao Xiang erofs_err(sb, "invalid compressed data");
178ffa09b3bSGao Xiang err = -EFSCORRUPTED;
179ffa09b3bSGao Xiang break;
180ffa09b3bSGao Xiang }
181ffa09b3bSGao Xiang
182ffa09b3bSGao Xiang if (kout) { /* unlike kmap(), take care of the orders */
183ffa09b3bSGao Xiang j = strm->z.next_out - kout;
184ffa09b3bSGao Xiang kunmap_local(kout);
185ffa09b3bSGao Xiang }
186ffa09b3bSGao Xiang kunmap_local(kin);
187ffa09b3bSGao Xiang strm->z.avail_in = min_t(u32, insz, PAGE_SIZE);
188ffa09b3bSGao Xiang insz -= strm->z.avail_in;
189ffa09b3bSGao Xiang kin = kmap_local_page(rq->in[ni]);
190ffa09b3bSGao Xiang strm->z.next_in = kin;
191ffa09b3bSGao Xiang bounced = false;
192ffa09b3bSGao Xiang if (kout) {
193ffa09b3bSGao Xiang kout = kmap_local_page(rq->out[no]);
194ffa09b3bSGao Xiang strm->z.next_out = kout + j;
195ffa09b3bSGao Xiang }
196ffa09b3bSGao Xiang }
197ffa09b3bSGao Xiang
198ffa09b3bSGao Xiang /*
199ffa09b3bSGao Xiang * Handle overlapping: Use bounced buffer if the compressed
200ffa09b3bSGao Xiang * data is under processing; Or use short-lived pages from the
201ffa09b3bSGao Xiang * on-stack pagepool where pages share among the same request
202ffa09b3bSGao Xiang * and not _all_ inplace I/O pages are needed to be doubled.
203ffa09b3bSGao Xiang */
204ffa09b3bSGao Xiang if (!bounced && rq->out[no] == rq->in[ni]) {
205ffa09b3bSGao Xiang memcpy(strm->bounce, strm->z.next_in, strm->z.avail_in);
206ffa09b3bSGao Xiang strm->z.next_in = strm->bounce;
207ffa09b3bSGao Xiang bounced = true;
208ffa09b3bSGao Xiang }
209ffa09b3bSGao Xiang
210ffa09b3bSGao Xiang for (j = ni + 1; j < nrpages_in; ++j) {
211ffa09b3bSGao Xiang struct page *tmppage;
212ffa09b3bSGao Xiang
213ffa09b3bSGao Xiang if (rq->out[no] != rq->in[j])
214ffa09b3bSGao Xiang continue;
215ffa09b3bSGao Xiang
216ffa09b3bSGao Xiang DBG_BUGON(erofs_page_is_managed(EROFS_SB(sb),
217ffa09b3bSGao Xiang rq->in[j]));
218ffa09b3bSGao Xiang tmppage = erofs_allocpage(pagepool,
219ffa09b3bSGao Xiang GFP_KERNEL | __GFP_NOFAIL);
220ffa09b3bSGao Xiang set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
221ffa09b3bSGao Xiang copy_highpage(tmppage, rq->in[j]);
222ffa09b3bSGao Xiang rq->in[j] = tmppage;
223ffa09b3bSGao Xiang }
224ffa09b3bSGao Xiang
225ffa09b3bSGao Xiang zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH);
226ffa09b3bSGao Xiang if (zerr != Z_OK || !(outsz + strm->z.avail_out)) {
227ffa09b3bSGao Xiang if (zerr == Z_OK && rq->partial_decoding)
228ffa09b3bSGao Xiang break;
229ffa09b3bSGao Xiang if (zerr == Z_STREAM_END && !outsz)
230ffa09b3bSGao Xiang break;
231ffa09b3bSGao Xiang erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
232ffa09b3bSGao Xiang zerr, rq->inputsize, rq->outputsize);
233ffa09b3bSGao Xiang err = -EFSCORRUPTED;
234ffa09b3bSGao Xiang break;
235ffa09b3bSGao Xiang }
236ffa09b3bSGao Xiang }
237ffa09b3bSGao Xiang
238ffa09b3bSGao Xiang if (zlib_inflateEnd(&strm->z) != Z_OK && !err)
239ffa09b3bSGao Xiang err = -EIO;
240ffa09b3bSGao Xiang if (kout)
241ffa09b3bSGao Xiang kunmap_local(kout);
242ffa09b3bSGao Xiang failed_zinit:
243ffa09b3bSGao Xiang kunmap_local(kin);
244ffa09b3bSGao Xiang /* 4. push back DEFLATE stream context to the global list */
245ffa09b3bSGao Xiang spin_lock(&z_erofs_deflate_lock);
246ffa09b3bSGao Xiang strm->next = z_erofs_deflate_head;
247ffa09b3bSGao Xiang z_erofs_deflate_head = strm;
248ffa09b3bSGao Xiang spin_unlock(&z_erofs_deflate_lock);
249ffa09b3bSGao Xiang wake_up(&z_erofs_deflate_wq);
250ffa09b3bSGao Xiang return err;
251ffa09b3bSGao Xiang }
252