Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/xz.h>
3#include "compress.h"
4
5struct z_erofs_lzma {
6 struct z_erofs_lzma *next;
7 struct xz_dec_microlzma *state;
8 struct xz_buf buf;
9 u8 bounce[PAGE_SIZE];
10};
11
12/* considering the LZMA performance, no need to use a lockless list for now */
13static DEFINE_SPINLOCK(z_erofs_lzma_lock);
14static unsigned int z_erofs_lzma_max_dictsize;
15static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
16static struct z_erofs_lzma *z_erofs_lzma_head;
17static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
18
19module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
20
21void z_erofs_lzma_exit(void)
22{
23 /* there should be no running fs instance */
24 while (z_erofs_lzma_avail_strms) {
25 struct z_erofs_lzma *strm;
26
27 spin_lock(&z_erofs_lzma_lock);
28 strm = z_erofs_lzma_head;
29 if (!strm) {
30 spin_unlock(&z_erofs_lzma_lock);
31 DBG_BUGON(1);
32 return;
33 }
34 z_erofs_lzma_head = NULL;
35 spin_unlock(&z_erofs_lzma_lock);
36
37 while (strm) {
38 struct z_erofs_lzma *n = strm->next;
39
40 if (strm->state)
41 xz_dec_microlzma_end(strm->state);
42 kfree(strm);
43 --z_erofs_lzma_avail_strms;
44 strm = n;
45 }
46 }
47}
48
49int __init z_erofs_lzma_init(void)
50{
51 unsigned int i;
52
53 /* by default, use # of possible CPUs instead */
54 if (!z_erofs_lzma_nstrms)
55 z_erofs_lzma_nstrms = num_possible_cpus();
56
57 for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
58 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
59
60 if (!strm) {
61 z_erofs_lzma_exit();
62 return -ENOMEM;
63 }
64 spin_lock(&z_erofs_lzma_lock);
65 strm->next = z_erofs_lzma_head;
66 z_erofs_lzma_head = strm;
67 spin_unlock(&z_erofs_lzma_lock);
68 ++z_erofs_lzma_avail_strms;
69 }
70 return 0;
71}
72
73int z_erofs_load_lzma_config(struct super_block *sb,
74 struct erofs_super_block *dsb, void *data, int size)
75{
76 static DEFINE_MUTEX(lzma_resize_mutex);
77 struct z_erofs_lzma_cfgs *lzma = data;
78 unsigned int dict_size, i;
79 struct z_erofs_lzma *strm, *head = NULL;
80 int err;
81
82 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
83 erofs_err(sb, "invalid lzma cfgs, size=%u", size);
84 return -EINVAL;
85 }
86 if (lzma->format) {
87 erofs_err(sb, "unidentified lzma format %x, please check kernel version",
88 le16_to_cpu(lzma->format));
89 return -EINVAL;
90 }
91 dict_size = le32_to_cpu(lzma->dict_size);
92 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
93 erofs_err(sb, "unsupported lzma dictionary size %u",
94 dict_size);
95 return -EINVAL;
96 }
97
98 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
99 mutex_lock(&lzma_resize_mutex);
100
101 if (z_erofs_lzma_max_dictsize >= dict_size) {
102 mutex_unlock(&lzma_resize_mutex);
103 return 0;
104 }
105
106 /* 1. collect/isolate all streams for the following check */
107 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
108 struct z_erofs_lzma *last;
109
110again:
111 spin_lock(&z_erofs_lzma_lock);
112 strm = z_erofs_lzma_head;
113 if (!strm) {
114 spin_unlock(&z_erofs_lzma_lock);
115 wait_event(z_erofs_lzma_wq,
116 READ_ONCE(z_erofs_lzma_head));
117 goto again;
118 }
119 z_erofs_lzma_head = NULL;
120 spin_unlock(&z_erofs_lzma_lock);
121
122 for (last = strm; last->next; last = last->next)
123 ++i;
124 last->next = head;
125 head = strm;
126 }
127
128 err = 0;
129 /* 2. walk each isolated stream and grow max dict_size if needed */
130 for (strm = head; strm; strm = strm->next) {
131 if (strm->state)
132 xz_dec_microlzma_end(strm->state);
133 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
134 if (!strm->state)
135 err = -ENOMEM;
136 }
137
138 /* 3. push back all to the global list and update max dict_size */
139 spin_lock(&z_erofs_lzma_lock);
140 DBG_BUGON(z_erofs_lzma_head);
141 z_erofs_lzma_head = head;
142 spin_unlock(&z_erofs_lzma_lock);
143 wake_up_all(&z_erofs_lzma_wq);
144
145 z_erofs_lzma_max_dictsize = dict_size;
146 mutex_unlock(&lzma_resize_mutex);
147 return err;
148}
149
150int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
151 struct page **pgpl)
152{
153 const unsigned int nrpages_out =
154 PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
155 const unsigned int nrpages_in =
156 PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
157 unsigned int inlen, outlen, pageofs;
158 struct z_erofs_lzma *strm;
159 u8 *kin;
160 bool bounced = false;
161 int no, ni, j, err = 0;
162
163 /* 1. get the exact LZMA compressed size */
164 kin = kmap(*rq->in);
165 err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
166 min_t(unsigned int, rq->inputsize,
167 rq->sb->s_blocksize - rq->pageofs_in));
168 if (err) {
169 kunmap(*rq->in);
170 return err;
171 }
172
173 /* 2. get an available lzma context */
174again:
175 spin_lock(&z_erofs_lzma_lock);
176 strm = z_erofs_lzma_head;
177 if (!strm) {
178 spin_unlock(&z_erofs_lzma_lock);
179 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
180 goto again;
181 }
182 z_erofs_lzma_head = strm->next;
183 spin_unlock(&z_erofs_lzma_lock);
184
185 /* 3. multi-call decompress */
186 inlen = rq->inputsize;
187 outlen = rq->outputsize;
188 xz_dec_microlzma_reset(strm->state, inlen, outlen,
189 !rq->partial_decoding);
190 pageofs = rq->pageofs_out;
191 strm->buf.in = kin + rq->pageofs_in;
192 strm->buf.in_pos = 0;
193 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
194 inlen -= strm->buf.in_size;
195 strm->buf.out = NULL;
196 strm->buf.out_pos = 0;
197 strm->buf.out_size = 0;
198
199 for (ni = 0, no = -1;;) {
200 enum xz_ret xz_err;
201
202 if (strm->buf.out_pos == strm->buf.out_size) {
203 if (strm->buf.out) {
204 kunmap(rq->out[no]);
205 strm->buf.out = NULL;
206 }
207
208 if (++no >= nrpages_out || !outlen) {
209 erofs_err(rq->sb, "decompressed buf out of bound");
210 err = -EFSCORRUPTED;
211 break;
212 }
213 strm->buf.out_pos = 0;
214 strm->buf.out_size = min_t(u32, outlen,
215 PAGE_SIZE - pageofs);
216 outlen -= strm->buf.out_size;
217 if (!rq->out[no] && rq->fillgaps) { /* deduped */
218 rq->out[no] = erofs_allocpage(pgpl, rq->gfp);
219 if (!rq->out[no]) {
220 err = -ENOMEM;
221 break;
222 }
223 set_page_private(rq->out[no],
224 Z_EROFS_SHORTLIVED_PAGE);
225 }
226 if (rq->out[no])
227 strm->buf.out = kmap(rq->out[no]) + pageofs;
228 pageofs = 0;
229 } else if (strm->buf.in_pos == strm->buf.in_size) {
230 kunmap(rq->in[ni]);
231
232 if (++ni >= nrpages_in || !inlen) {
233 erofs_err(rq->sb, "compressed buf out of bound");
234 err = -EFSCORRUPTED;
235 break;
236 }
237 strm->buf.in_pos = 0;
238 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
239 inlen -= strm->buf.in_size;
240 kin = kmap(rq->in[ni]);
241 strm->buf.in = kin;
242 bounced = false;
243 }
244
245 /*
246 * Handle overlapping: Use bounced buffer if the compressed
247 * data is under processing; Otherwise, Use short-lived pages
248 * from the on-stack pagepool where pages share with the same
249 * request.
250 */
251 if (!bounced && rq->out[no] == rq->in[ni]) {
252 memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
253 strm->buf.in = strm->bounce;
254 bounced = true;
255 }
256 for (j = ni + 1; j < nrpages_in; ++j) {
257 struct page *tmppage;
258
259 if (rq->out[no] != rq->in[j])
260 continue;
261
262 DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
263 rq->in[j]));
264 tmppage = erofs_allocpage(pgpl, rq->gfp);
265 if (!tmppage) {
266 err = -ENOMEM;
267 goto failed;
268 }
269 set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
270 copy_highpage(tmppage, rq->in[j]);
271 rq->in[j] = tmppage;
272 }
273 xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
274 DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
275 DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
276
277 if (xz_err != XZ_OK) {
278 if (xz_err == XZ_STREAM_END && !outlen)
279 break;
280 erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
281 xz_err, rq->inputsize, rq->outputsize);
282 err = -EFSCORRUPTED;
283 break;
284 }
285 }
286failed:
287 if (no < nrpages_out && strm->buf.out)
288 kunmap(rq->out[no]);
289 if (ni < nrpages_in)
290 kunmap(rq->in[ni]);
291 /* 4. push back LZMA stream context to the global list */
292 spin_lock(&z_erofs_lzma_lock);
293 strm->next = z_erofs_lzma_head;
294 z_erofs_lzma_head = strm;
295 spin_unlock(&z_erofs_lzma_lock);
296 wake_up(&z_erofs_lzma_wq);
297 return err;
298}
1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/xz.h>
3#include "compress.h"
4
5struct z_erofs_lzma {
6 struct z_erofs_lzma *next;
7 struct xz_dec_microlzma *state;
8 u8 bounce[PAGE_SIZE];
9};
10
11/* considering the LZMA performance, no need to use a lockless list for now */
12static DEFINE_SPINLOCK(z_erofs_lzma_lock);
13static unsigned int z_erofs_lzma_max_dictsize;
14static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
15static struct z_erofs_lzma *z_erofs_lzma_head;
16static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
17
18module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
19
20static void z_erofs_lzma_exit(void)
21{
22 /* there should be no running fs instance */
23 while (z_erofs_lzma_avail_strms) {
24 struct z_erofs_lzma *strm;
25
26 spin_lock(&z_erofs_lzma_lock);
27 strm = z_erofs_lzma_head;
28 if (!strm) {
29 spin_unlock(&z_erofs_lzma_lock);
30 DBG_BUGON(1);
31 return;
32 }
33 z_erofs_lzma_head = NULL;
34 spin_unlock(&z_erofs_lzma_lock);
35
36 while (strm) {
37 struct z_erofs_lzma *n = strm->next;
38
39 if (strm->state)
40 xz_dec_microlzma_end(strm->state);
41 kfree(strm);
42 --z_erofs_lzma_avail_strms;
43 strm = n;
44 }
45 }
46}
47
48static int __init z_erofs_lzma_init(void)
49{
50 unsigned int i;
51
52 /* by default, use # of possible CPUs instead */
53 if (!z_erofs_lzma_nstrms)
54 z_erofs_lzma_nstrms = num_possible_cpus();
55
56 for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
57 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
58
59 if (!strm) {
60 z_erofs_lzma_exit();
61 return -ENOMEM;
62 }
63 spin_lock(&z_erofs_lzma_lock);
64 strm->next = z_erofs_lzma_head;
65 z_erofs_lzma_head = strm;
66 spin_unlock(&z_erofs_lzma_lock);
67 ++z_erofs_lzma_avail_strms;
68 }
69 return 0;
70}
71
72static int z_erofs_load_lzma_config(struct super_block *sb,
73 struct erofs_super_block *dsb, void *data, int size)
74{
75 static DEFINE_MUTEX(lzma_resize_mutex);
76 struct z_erofs_lzma_cfgs *lzma = data;
77 unsigned int dict_size, i;
78 struct z_erofs_lzma *strm, *head = NULL;
79 int err;
80
81 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
82 erofs_err(sb, "invalid lzma cfgs, size=%u", size);
83 return -EINVAL;
84 }
85 if (lzma->format) {
86 erofs_err(sb, "unidentified lzma format %x, please check kernel version",
87 le16_to_cpu(lzma->format));
88 return -EINVAL;
89 }
90 dict_size = le32_to_cpu(lzma->dict_size);
91 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
92 erofs_err(sb, "unsupported lzma dictionary size %u",
93 dict_size);
94 return -EINVAL;
95 }
96
97 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
98 mutex_lock(&lzma_resize_mutex);
99
100 if (z_erofs_lzma_max_dictsize >= dict_size) {
101 mutex_unlock(&lzma_resize_mutex);
102 return 0;
103 }
104
105 /* 1. collect/isolate all streams for the following check */
106 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
107 struct z_erofs_lzma *last;
108
109again:
110 spin_lock(&z_erofs_lzma_lock);
111 strm = z_erofs_lzma_head;
112 if (!strm) {
113 spin_unlock(&z_erofs_lzma_lock);
114 wait_event(z_erofs_lzma_wq,
115 READ_ONCE(z_erofs_lzma_head));
116 goto again;
117 }
118 z_erofs_lzma_head = NULL;
119 spin_unlock(&z_erofs_lzma_lock);
120
121 for (last = strm; last->next; last = last->next)
122 ++i;
123 last->next = head;
124 head = strm;
125 }
126
127 err = 0;
128 /* 2. walk each isolated stream and grow max dict_size if needed */
129 for (strm = head; strm; strm = strm->next) {
130 if (strm->state)
131 xz_dec_microlzma_end(strm->state);
132 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
133 if (!strm->state)
134 err = -ENOMEM;
135 }
136
137 /* 3. push back all to the global list and update max dict_size */
138 spin_lock(&z_erofs_lzma_lock);
139 DBG_BUGON(z_erofs_lzma_head);
140 z_erofs_lzma_head = head;
141 spin_unlock(&z_erofs_lzma_lock);
142 wake_up_all(&z_erofs_lzma_wq);
143
144 z_erofs_lzma_max_dictsize = dict_size;
145 mutex_unlock(&lzma_resize_mutex);
146 return err;
147}
148
149static int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
150 struct page **pgpl)
151{
152 struct super_block *sb = rq->sb;
153 struct z_erofs_stream_dctx dctx = {
154 .rq = rq,
155 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
156 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
157 >> PAGE_SHIFT,
158 .no = -1, .ni = 0,
159 };
160 struct xz_buf buf = {};
161 struct z_erofs_lzma *strm;
162 enum xz_ret xz_err;
163 int err;
164
165 /* 1. get the exact LZMA compressed size */
166 dctx.kin = kmap_local_page(*rq->in);
167 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
168 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
169 if (err) {
170 kunmap_local(dctx.kin);
171 return err;
172 }
173
174 /* 2. get an available lzma context */
175again:
176 spin_lock(&z_erofs_lzma_lock);
177 strm = z_erofs_lzma_head;
178 if (!strm) {
179 spin_unlock(&z_erofs_lzma_lock);
180 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
181 goto again;
182 }
183 z_erofs_lzma_head = strm->next;
184 spin_unlock(&z_erofs_lzma_lock);
185
186 /* 3. multi-call decompress */
187 xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize,
188 !rq->partial_decoding);
189 buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
190 rq->inputsize -= buf.in_size;
191 buf.in = dctx.kin + rq->pageofs_in;
192 dctx.bounce = strm->bounce;
193 do {
194 dctx.avail_out = buf.out_size - buf.out_pos;
195 dctx.inbuf_sz = buf.in_size;
196 dctx.inbuf_pos = buf.in_pos;
197 err = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out,
198 (void **)&buf.in, pgpl);
199 if (err)
200 break;
201
202 if (buf.out_size == buf.out_pos) {
203 buf.out_size = dctx.avail_out;
204 buf.out_pos = 0;
205 }
206 buf.in_size = dctx.inbuf_sz;
207 buf.in_pos = dctx.inbuf_pos;
208
209 xz_err = xz_dec_microlzma_run(strm->state, &buf);
210 DBG_BUGON(buf.out_pos > buf.out_size);
211 DBG_BUGON(buf.in_pos > buf.in_size);
212
213 if (xz_err != XZ_OK) {
214 if (xz_err == XZ_STREAM_END && !rq->outputsize)
215 break;
216 erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
217 xz_err, rq->inputsize, rq->outputsize);
218 err = -EFSCORRUPTED;
219 break;
220 }
221 } while (1);
222
223 if (dctx.kout)
224 kunmap_local(dctx.kout);
225 kunmap_local(dctx.kin);
226 /* 4. push back LZMA stream context to the global list */
227 spin_lock(&z_erofs_lzma_lock);
228 strm->next = z_erofs_lzma_head;
229 z_erofs_lzma_head = strm;
230 spin_unlock(&z_erofs_lzma_lock);
231 wake_up(&z_erofs_lzma_wq);
232 return err;
233}
234
235const struct z_erofs_decompressor z_erofs_lzma_decomp = {
236 .config = z_erofs_load_lzma_config,
237 .decompress = z_erofs_lzma_decompress,
238 .init = z_erofs_lzma_init,
239 .exit = z_erofs_lzma_exit,
240 .name = "lzma"
241};