1// SPDX-License-Identifier: GPL-2.0-only
  2/* Copyright(c) 2022 Intel Corporation */
  3#include <linux/module.h>
  4#include <linux/slab.h>
  5#include "adf_accel_devices.h"
  6#include "adf_common_drv.h"
  7#include "adf_transport.h"
  8#include "adf_transport_access_macros.h"
  9#include "adf_cfg.h"
 10#include "adf_cfg_strings.h"
 11#include "qat_compression.h"
 12#include "icp_qat_fw.h"
 13
 14#define SEC ADF_KERNEL_SEC
 15
 16static struct service_hndl qat_compression;
 17
 18void qat_compression_put_instance(struct qat_compression_instance *inst)
 19{
 20	atomic_dec(&inst->refctr);
 21	adf_dev_put(inst->accel_dev);
 22}
 23
 24static int qat_compression_free_instances(struct adf_accel_dev *accel_dev)
 25{
 26	struct qat_compression_instance *inst;
 27	struct list_head *list_ptr, *tmp;
 28	int i;
 29
 30	list_for_each_safe(list_ptr, tmp, &accel_dev->compression_list) {
 31		inst = list_entry(list_ptr,
 32				  struct qat_compression_instance, list);
 33
 34		for (i = 0; i < atomic_read(&inst->refctr); i++)
 35			qat_compression_put_instance(inst);
 36
 37		if (inst->dc_tx)
 38			adf_remove_ring(inst->dc_tx);
 39
 40		if (inst->dc_rx)
 41			adf_remove_ring(inst->dc_rx);
 42
 43		list_del(list_ptr);
 44		kfree(inst);
 45	}
 46	return 0;
 47}
 48
 49struct qat_compression_instance *qat_compression_get_instance_node(int node)
 50{
 51	struct qat_compression_instance *inst = NULL;
 52	struct adf_accel_dev *accel_dev = NULL;
 53	unsigned long best = ~0;
 54	struct list_head *itr;
 55
 56	list_for_each(itr, adf_devmgr_get_head()) {
 57		struct adf_accel_dev *tmp_dev;
 58		unsigned long ctr;
 59		int tmp_dev_node;
 60
 61		tmp_dev = list_entry(itr, struct adf_accel_dev, list);
 62		tmp_dev_node = dev_to_node(&GET_DEV(tmp_dev));
 63
 64		if ((node == tmp_dev_node || tmp_dev_node < 0) &&
 65		    adf_dev_started(tmp_dev) && !list_empty(&tmp_dev->compression_list)) {
 66			ctr = atomic_read(&tmp_dev->ref_count);
 67			if (best > ctr) {
 68				accel_dev = tmp_dev;
 69				best = ctr;
 70			}
 71		}
 72	}
 73
 74	if (!accel_dev) {
 75		pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node);
 76		/* Get any started device */
 77		list_for_each(itr, adf_devmgr_get_head()) {
 78			struct adf_accel_dev *tmp_dev;
 79
 80			tmp_dev = list_entry(itr, struct adf_accel_dev, list);
 81			if (adf_dev_started(tmp_dev) &&
 82			    !list_empty(&tmp_dev->compression_list)) {
 83				accel_dev = tmp_dev;
 84				break;
 85			}
 86		}
 87	}
 88
 89	if (!accel_dev)
 90		return NULL;
 91
 92	best = ~0;
 93	list_for_each(itr, &accel_dev->compression_list) {
 94		struct qat_compression_instance *tmp_inst;
 95		unsigned long ctr;
 96
 97		tmp_inst = list_entry(itr, struct qat_compression_instance, list);
 98		ctr = atomic_read(&tmp_inst->refctr);
 99		if (best > ctr) {
100			inst = tmp_inst;
101			best = ctr;
102		}
103	}
104	if (inst) {
105		if (adf_dev_get(accel_dev)) {
106			dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n");
107			return NULL;
108		}
109		atomic_inc(&inst->refctr);
110	}
111	return inst;
112}
113
114static int qat_compression_create_instances(struct adf_accel_dev *accel_dev)
115{
116	struct qat_compression_instance *inst;
117	char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
118	char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
119	unsigned long num_inst, num_msg_dc;
120	unsigned long bank;
121	int msg_size;
122	int ret;
123	int i;
124
125	INIT_LIST_HEAD(&accel_dev->compression_list);
126	strscpy(key, ADF_NUM_DC, sizeof(key));
127	ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
128	if (ret)
129		return ret;
130
131	ret = kstrtoul(val, 10, &num_inst);
132	if (ret)
133		return ret;
134
135	for (i = 0; i < num_inst; i++) {
136		inst = kzalloc_node(sizeof(*inst), GFP_KERNEL,
137				    dev_to_node(&GET_DEV(accel_dev)));
138		if (!inst) {
139			ret = -ENOMEM;
140			goto err;
141		}
142
143		list_add_tail(&inst->list, &accel_dev->compression_list);
144		inst->id = i;
145		atomic_set(&inst->refctr, 0);
146		inst->accel_dev = accel_dev;
147		inst->build_deflate_ctx = GET_DC_OPS(accel_dev)->build_deflate_ctx;
148
149		snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i);
150		ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
151		if (ret)
152			return ret;
153
154		ret = kstrtoul(val, 10, &bank);
155		if (ret)
156			return ret;
157
158		snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i);
159		ret = adf_cfg_get_param_value(accel_dev, SEC, key, val);
160		if (ret)
161			return ret;
162
163		ret = kstrtoul(val, 10, &num_msg_dc);
164		if (ret)
165			return ret;
166
167		msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ;
168		snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i);
169		ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc,
170				      msg_size, key, NULL, 0, &inst->dc_tx);
171		if (ret)
172			return ret;
173
174		msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ;
175		snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i);
176		ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc,
177				      msg_size, key, qat_comp_alg_callback, 0,
178				      &inst->dc_rx);
179		if (ret)
180			return ret;
181
182		inst->dc_data = accel_dev->dc_data;
183		INIT_LIST_HEAD(&inst->backlog.list);
184		spin_lock_init(&inst->backlog.lock);
185	}
186	return 0;
187err:
188	qat_compression_free_instances(accel_dev);
189	return ret;
190}
191
192static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev)
193{
194	struct device *dev = &GET_DEV(accel_dev);
195	dma_addr_t obuff_p = DMA_MAPPING_ERROR;
196	size_t ovf_buff_sz = QAT_COMP_MAX_SKID;
197	struct adf_dc_data *dc_data = NULL;
198	u8 *obuff = NULL;
199
200	dc_data = devm_kzalloc(dev, sizeof(*dc_data), GFP_KERNEL);
201	if (!dc_data)
202		goto err;
203
204	obuff = kzalloc_node(ovf_buff_sz, GFP_KERNEL, dev_to_node(dev));
205	if (!obuff)
206		goto err;
207
208	obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_FROM_DEVICE);
209	if (unlikely(dma_mapping_error(dev, obuff_p)))
210		goto err;
211
212	dc_data->ovf_buff = obuff;
213	dc_data->ovf_buff_p = obuff_p;
214	dc_data->ovf_buff_sz = ovf_buff_sz;
215
216	accel_dev->dc_data = dc_data;
217
218	return 0;
219
220err:
221	accel_dev->dc_data = NULL;
222	kfree(obuff);
223	devm_kfree(dev, dc_data);
224	return -ENOMEM;
225}
226
227static void qat_free_dc_data(struct adf_accel_dev *accel_dev)
228{
229	struct adf_dc_data *dc_data = accel_dev->dc_data;
230	struct device *dev = &GET_DEV(accel_dev);
231
232	if (!dc_data)
233		return;
234
235	dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz,
236			 DMA_FROM_DEVICE);
237	kfree_sensitive(dc_data->ovf_buff);
238	devm_kfree(dev, dc_data);
239	accel_dev->dc_data = NULL;
240}
241
242static int qat_compression_init(struct adf_accel_dev *accel_dev)
243{
244	int ret;
245
246	ret = qat_compression_alloc_dc_data(accel_dev);
247	if (ret)
248		return ret;
249
250	ret = qat_compression_create_instances(accel_dev);
251	if (ret)
252		qat_free_dc_data(accel_dev);
253
254	return ret;
255}
256
257static int qat_compression_shutdown(struct adf_accel_dev *accel_dev)
258{
259	qat_free_dc_data(accel_dev);
260	return qat_compression_free_instances(accel_dev);
261}
262
263static int qat_compression_event_handler(struct adf_accel_dev *accel_dev,
264					 enum adf_event event)
265{
266	int ret;
267
268	switch (event) {
269	case ADF_EVENT_INIT:
270		ret = qat_compression_init(accel_dev);
271		break;
272	case ADF_EVENT_SHUTDOWN:
273		ret = qat_compression_shutdown(accel_dev);
274		break;
275	case ADF_EVENT_RESTARTING:
276	case ADF_EVENT_RESTARTED:
277	case ADF_EVENT_START:
278	case ADF_EVENT_STOP:
279	default:
280		ret = 0;
281	}
282	return ret;
283}
284
285int qat_compression_register(void)
286{
287	memset(&qat_compression, 0, sizeof(qat_compression));
288	qat_compression.event_hld = qat_compression_event_handler;
289	qat_compression.name = "qat_compression";
290	return adf_service_register(&qat_compression);
291}
292
293int qat_compression_unregister(void)
294{
295	return adf_service_unregister(&qat_compression);
296}