1// SPDX-License-Identifier: GPL-2.0-or-later
  2
  3#include <linux/kernel.h>
  4#include <linux/slab.h>
  5#include <linux/vmalloc.h>
  6#include <linux/zstd.h>
  7
  8#include "backend_zstd.h"
  9
 10struct zstd_ctx {
 11	zstd_cctx *cctx;
 12	zstd_dctx *dctx;
 13	void *cctx_mem;
 14	void *dctx_mem;
 15};
 16
 17struct zstd_params {
 18	zstd_custom_mem custom_mem;
 19	zstd_cdict *cdict;
 20	zstd_ddict *ddict;
 21	zstd_parameters cprm;
 22};
 23
 24/*
 25 * For C/D dictionaries we need to provide zstd with zstd_custom_mem,
 26 * which zstd uses internally to allocate/free memory when needed.
 27 *
 28 * This means that allocator.customAlloc() can be called from zcomp_compress()
 29 * under local-lock (per-CPU compression stream), in which case we must use
 30 * GFP_ATOMIC.
 31 *
 32 * Another complication here is that we can be configured as a swap device.
 33 */
 34static void *zstd_custom_alloc(void *opaque, size_t size)
 35{
 36	if (!preemptible())
 37		return kvzalloc(size, GFP_ATOMIC);
 38
 39	return kvzalloc(size, __GFP_KSWAPD_RECLAIM | __GFP_NOWARN);
 40}
 41
 42static void zstd_custom_free(void *opaque, void *address)
 43{
 44	kvfree(address);
 45}
 46
 47static void zstd_release_params(struct zcomp_params *params)
 48{
 49	struct zstd_params *zp = params->drv_data;
 50
 51	params->drv_data = NULL;
 52	if (!zp)
 53		return;
 54
 55	zstd_free_cdict(zp->cdict);
 56	zstd_free_ddict(zp->ddict);
 57	kfree(zp);
 58}
 59
 60static int zstd_setup_params(struct zcomp_params *params)
 61{
 62	zstd_compression_parameters prm;
 63	struct zstd_params *zp;
 64
 65	zp = kzalloc(sizeof(*zp), GFP_KERNEL);
 66	if (!zp)
 67		return -ENOMEM;
 68
 69	params->drv_data = zp;
 70	if (params->level == ZCOMP_PARAM_NO_LEVEL)
 71		params->level = zstd_default_clevel();
 72
 73	zp->cprm = zstd_get_params(params->level, PAGE_SIZE);
 74
 75	zp->custom_mem.customAlloc = zstd_custom_alloc;
 76	zp->custom_mem.customFree = zstd_custom_free;
 77
 78	prm = zstd_get_cparams(params->level, PAGE_SIZE,
 79			       params->dict_sz);
 80
 81	zp->cdict = zstd_create_cdict_byreference(params->dict,
 82						  params->dict_sz,
 83						  prm,
 84						  zp->custom_mem);
 85	if (!zp->cdict)
 86		goto error;
 87
 88	zp->ddict = zstd_create_ddict_byreference(params->dict,
 89						  params->dict_sz,
 90						  zp->custom_mem);
 91	if (!zp->ddict)
 92		goto error;
 93
 94	return 0;
 95
 96error:
 97	zstd_release_params(params);
 98	return -EINVAL;
 99}
100
101static void zstd_destroy(struct zcomp_ctx *ctx)
102{
103	struct zstd_ctx *zctx = ctx->context;
104
105	if (!zctx)
106		return;
107
108	/*
109	 * If ->cctx_mem and ->dctx_mem were allocated then we didn't use
110	 * C/D dictionary and ->cctx / ->dctx were "embedded" into these
111	 * buffers.
112	 *
113	 * If otherwise then we need to explicitly release ->cctx / ->dctx.
114	 */
115	if (zctx->cctx_mem)
116		vfree(zctx->cctx_mem);
117	else
118		zstd_free_cctx(zctx->cctx);
119
120	if (zctx->dctx_mem)
121		vfree(zctx->dctx_mem);
122	else
123		zstd_free_dctx(zctx->dctx);
124
125	kfree(zctx);
126}
127
128static int zstd_create(struct zcomp_params *params, struct zcomp_ctx *ctx)
129{
130	struct zstd_ctx *zctx;
131	zstd_parameters prm;
132	size_t sz;
133
134	zctx = kzalloc(sizeof(*zctx), GFP_KERNEL);
135	if (!zctx)
136		return -ENOMEM;
137
138	ctx->context = zctx;
139	if (params->dict_sz == 0) {
140		prm = zstd_get_params(params->level, PAGE_SIZE);
141		sz = zstd_cctx_workspace_bound(&prm.cParams);
142		zctx->cctx_mem = vzalloc(sz);
143		if (!zctx->cctx_mem)
144			goto error;
145
146		zctx->cctx = zstd_init_cctx(zctx->cctx_mem, sz);
147		if (!zctx->cctx)
148			goto error;
149
150		sz = zstd_dctx_workspace_bound();
151		zctx->dctx_mem = vzalloc(sz);
152		if (!zctx->dctx_mem)
153			goto error;
154
155		zctx->dctx = zstd_init_dctx(zctx->dctx_mem, sz);
156		if (!zctx->dctx)
157			goto error;
158	} else {
159		struct zstd_params *zp = params->drv_data;
160
161		zctx->cctx = zstd_create_cctx_advanced(zp->custom_mem);
162		if (!zctx->cctx)
163			goto error;
164
165		zctx->dctx = zstd_create_dctx_advanced(zp->custom_mem);
166		if (!zctx->dctx)
167			goto error;
168	}
169
170	return 0;
171
172error:
173	zstd_release_params(params);
174	zstd_destroy(ctx);
175	return -EINVAL;
176}
177
178static int zstd_compress(struct zcomp_params *params, struct zcomp_ctx *ctx,
179			 struct zcomp_req *req)
180{
181	struct zstd_params *zp = params->drv_data;
182	struct zstd_ctx *zctx = ctx->context;
183	size_t ret;
184
185	if (params->dict_sz == 0)
186		ret = zstd_compress_cctx(zctx->cctx, req->dst, req->dst_len,
187					 req->src, req->src_len, &zp->cprm);
188	else
189		ret = zstd_compress_using_cdict(zctx->cctx, req->dst,
190						req->dst_len, req->src,
191						req->src_len,
192						zp->cdict);
193	if (zstd_is_error(ret))
194		return -EINVAL;
195	req->dst_len = ret;
196	return 0;
197}
198
199static int zstd_decompress(struct zcomp_params *params, struct zcomp_ctx *ctx,
200			   struct zcomp_req *req)
201{
202	struct zstd_params *zp = params->drv_data;
203	struct zstd_ctx *zctx = ctx->context;
204	size_t ret;
205
206	if (params->dict_sz == 0)
207		ret = zstd_decompress_dctx(zctx->dctx, req->dst, req->dst_len,
208					   req->src, req->src_len);
209	else
210		ret = zstd_decompress_using_ddict(zctx->dctx, req->dst,
211						  req->dst_len, req->src,
212						  req->src_len, zp->ddict);
213	if (zstd_is_error(ret))
214		return -EINVAL;
215	return 0;
216}
217
218const struct zcomp_ops backend_zstd = {
219	.compress	= zstd_compress,
220	.decompress	= zstd_decompress,
221	.create_ctx	= zstd_create,
222	.destroy_ctx	= zstd_destroy,
223	.setup_params	= zstd_setup_params,
224	.release_params	= zstd_release_params,
225	.name		= "zstd",
226};