1// SPDX-License-Identifier: GPL-2.0-or-later
  2#include <linux/xz.h>
  3#include "compress.h"
  4
  5struct z_erofs_lzma {
  6	struct z_erofs_lzma *next;
  7	struct xz_dec_microlzma *state;
  8	u8 bounce[PAGE_SIZE];
  9};
 10
 11/* considering the LZMA performance, no need to use a lockless list for now */
 12static DEFINE_SPINLOCK(z_erofs_lzma_lock);
 13static unsigned int z_erofs_lzma_max_dictsize;
 14static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
 15static struct z_erofs_lzma *z_erofs_lzma_head;
 16static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
 17
 18module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
 19
 20static void z_erofs_lzma_exit(void)
 21{
 22	/* there should be no running fs instance */
 23	while (z_erofs_lzma_avail_strms) {
 24		struct z_erofs_lzma *strm;
 25
 26		spin_lock(&z_erofs_lzma_lock);
 27		strm = z_erofs_lzma_head;
 28		if (!strm) {
 29			spin_unlock(&z_erofs_lzma_lock);
 30			DBG_BUGON(1);
 31			return;
 32		}
 33		z_erofs_lzma_head = NULL;
 34		spin_unlock(&z_erofs_lzma_lock);
 35
 36		while (strm) {
 37			struct z_erofs_lzma *n = strm->next;
 38
 39			if (strm->state)
 40				xz_dec_microlzma_end(strm->state);
 41			kfree(strm);
 42			--z_erofs_lzma_avail_strms;
 43			strm = n;
 44		}
 45	}
 46}
 47
 48static int __init z_erofs_lzma_init(void)
 49{
 50	unsigned int i;
 51
 52	/* by default, use # of possible CPUs instead */
 53	if (!z_erofs_lzma_nstrms)
 54		z_erofs_lzma_nstrms = num_possible_cpus();
 55
 56	for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
 57		struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
 58
 59		if (!strm) {
 60			z_erofs_lzma_exit();
 61			return -ENOMEM;
 62		}
 63		spin_lock(&z_erofs_lzma_lock);
 64		strm->next = z_erofs_lzma_head;
 65		z_erofs_lzma_head = strm;
 66		spin_unlock(&z_erofs_lzma_lock);
 67		++z_erofs_lzma_avail_strms;
 68	}
 69	return 0;
 70}
 71
 72static int z_erofs_load_lzma_config(struct super_block *sb,
 73			struct erofs_super_block *dsb, void *data, int size)
 74{
 75	static DEFINE_MUTEX(lzma_resize_mutex);
 76	struct z_erofs_lzma_cfgs *lzma = data;
 77	unsigned int dict_size, i;
 78	struct z_erofs_lzma *strm, *head = NULL;
 79	int err;
 80
 81	if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
 82		erofs_err(sb, "invalid lzma cfgs, size=%u", size);
 83		return -EINVAL;
 84	}
 85	if (lzma->format) {
 86		erofs_err(sb, "unidentified lzma format %x, please check kernel version",
 87			  le16_to_cpu(lzma->format));
 88		return -EINVAL;
 89	}
 90	dict_size = le32_to_cpu(lzma->dict_size);
 91	if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
 92		erofs_err(sb, "unsupported lzma dictionary size %u",
 93			  dict_size);
 94		return -EINVAL;
 95	}
 96
 97	/* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
 98	mutex_lock(&lzma_resize_mutex);
 99
100	if (z_erofs_lzma_max_dictsize >= dict_size) {
101		mutex_unlock(&lzma_resize_mutex);
102		return 0;
103	}
104
105	/* 1. collect/isolate all streams for the following check */
106	for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
107		struct z_erofs_lzma *last;
108
109again:
110		spin_lock(&z_erofs_lzma_lock);
111		strm = z_erofs_lzma_head;
112		if (!strm) {
113			spin_unlock(&z_erofs_lzma_lock);
114			wait_event(z_erofs_lzma_wq,
115				   READ_ONCE(z_erofs_lzma_head));
116			goto again;
117		}
118		z_erofs_lzma_head = NULL;
119		spin_unlock(&z_erofs_lzma_lock);
120
121		for (last = strm; last->next; last = last->next)
122			++i;
123		last->next = head;
124		head = strm;
125	}
126
127	err = 0;
128	/* 2. walk each isolated stream and grow max dict_size if needed */
129	for (strm = head; strm; strm = strm->next) {
130		if (strm->state)
131			xz_dec_microlzma_end(strm->state);
132		strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
133		if (!strm->state)
134			err = -ENOMEM;
135	}
136
137	/* 3. push back all to the global list and update max dict_size */
138	spin_lock(&z_erofs_lzma_lock);
139	DBG_BUGON(z_erofs_lzma_head);
140	z_erofs_lzma_head = head;
141	spin_unlock(&z_erofs_lzma_lock);
142	wake_up_all(&z_erofs_lzma_wq);
143
144	z_erofs_lzma_max_dictsize = dict_size;
145	mutex_unlock(&lzma_resize_mutex);
146	return err;
147}
148
149static int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
150				   struct page **pgpl)
151{
152	struct super_block *sb = rq->sb;
153	struct z_erofs_stream_dctx dctx = {
154		.rq = rq,
155		.inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
156		.outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
157				>> PAGE_SHIFT,
158		.no = -1, .ni = 0,
159	};
160	struct xz_buf buf = {};
161	struct z_erofs_lzma *strm;
162	enum xz_ret xz_err;
163	int err;
164
165	/* 1. get the exact LZMA compressed size */
166	dctx.kin = kmap_local_page(*rq->in);
167	err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
168			min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
169	if (err) {
170		kunmap_local(dctx.kin);
171		return err;
172	}
173
174	/* 2. get an available lzma context */
175again:
176	spin_lock(&z_erofs_lzma_lock);
177	strm = z_erofs_lzma_head;
178	if (!strm) {
179		spin_unlock(&z_erofs_lzma_lock);
180		wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
181		goto again;
182	}
183	z_erofs_lzma_head = strm->next;
184	spin_unlock(&z_erofs_lzma_lock);
185
186	/* 3. multi-call decompress */
187	xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize,
188			       !rq->partial_decoding);
189	buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in);
190	rq->inputsize -= buf.in_size;
191	buf.in = dctx.kin + rq->pageofs_in;
192	dctx.bounce = strm->bounce;
193	do {
194		dctx.avail_out = buf.out_size - buf.out_pos;
195		dctx.inbuf_sz = buf.in_size;
196		dctx.inbuf_pos = buf.in_pos;
197		err = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out,
198						 (void **)&buf.in, pgpl);
199		if (err)
200			break;
201
202		if (buf.out_size == buf.out_pos) {
203			buf.out_size = dctx.avail_out;
204			buf.out_pos = 0;
205		}
206		buf.in_size = dctx.inbuf_sz;
207		buf.in_pos = dctx.inbuf_pos;
208
209		xz_err = xz_dec_microlzma_run(strm->state, &buf);
210		DBG_BUGON(buf.out_pos > buf.out_size);
211		DBG_BUGON(buf.in_pos > buf.in_size);
212
213		if (xz_err != XZ_OK) {
214			if (xz_err == XZ_STREAM_END && !rq->outputsize)
215				break;
216			erofs_err(sb, "failed to decompress %d in[%u] out[%u]",
217				  xz_err, rq->inputsize, rq->outputsize);
218			err = -EFSCORRUPTED;
219			break;
220		}
221	} while (1);
222
223	if (dctx.kout)
224		kunmap_local(dctx.kout);
225	kunmap_local(dctx.kin);
226	/* 4. push back LZMA stream context to the global list */
227	spin_lock(&z_erofs_lzma_lock);
228	strm->next = z_erofs_lzma_head;
229	z_erofs_lzma_head = strm;
230	spin_unlock(&z_erofs_lzma_lock);
231	wake_up(&z_erofs_lzma_wq);
232	return err;
233}
234
235const struct z_erofs_decompressor z_erofs_lzma_decomp = {
236	.config = z_erofs_load_lzma_config,
237	.decompress = z_erofs_lzma_decompress,
238	.init = z_erofs_lzma_init,
239	.exit = z_erofs_lzma_exit,
240	.name = "lzma"
241};