1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Copyright 2016-2019 HabanaLabs, Ltd.
  5 * All Rights Reserved.
  6 */
  7
  8#include <uapi/misc/habanalabs.h>
  9#include "habanalabs.h"
 10
 11#include <linux/mm.h>
 12#include <linux/slab.h>
 13
 14static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
 15{
 16	hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,
 17			(void *) (uintptr_t) cb->kernel_address,
 18			cb->bus_address);
 19	kfree(cb);
 20}
 21
 22static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
 23{
 24	if (cb->is_pool) {
 25		spin_lock(&hdev->cb_pool_lock);
 26		list_add(&cb->pool_list, &hdev->cb_pool);
 27		spin_unlock(&hdev->cb_pool_lock);
 28	} else {
 29		cb_fini(hdev, cb);
 30	}
 31}
 32
 33static void cb_release(struct kref *ref)
 34{
 35	struct hl_device *hdev;
 36	struct hl_cb *cb;
 37
 38	cb = container_of(ref, struct hl_cb, refcount);
 39	hdev = cb->hdev;
 40
 41	hl_debugfs_remove_cb(cb);
 42
 43	cb_do_release(hdev, cb);
 44}
 45
 46static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
 47					int ctx_id)
 48{
 49	struct hl_cb *cb;
 50	void *p;
 51
 52	/*
 53	 * We use of GFP_ATOMIC here because this function can be called from
 54	 * the latency-sensitive code path for command submission. Due to H/W
 55	 * limitations in some of the ASICs, the kernel must copy the user CB
 56	 * that is designated for an external queue and actually enqueue
 57	 * the kernel's copy. Hence, we must never sleep in this code section
 58	 * and must use GFP_ATOMIC for all memory allocations.
 59	 */
 60	if (ctx_id == HL_KERNEL_ASID_ID)
 61		cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
 62	else
 63		cb = kzalloc(sizeof(*cb), GFP_KERNEL);
 64
 65	if (!cb)
 66		return NULL;
 67
 68	if (ctx_id == HL_KERNEL_ASID_ID)
 69		p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
 70						&cb->bus_address, GFP_ATOMIC);
 71	else
 72		p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
 73						&cb->bus_address,
 74						GFP_USER | __GFP_ZERO);
 75	if (!p) {
 76		dev_err(hdev->dev,
 77			"failed to allocate %d of dma memory for CB\n",
 78			cb_size);
 79		kfree(cb);
 80		return NULL;
 81	}
 82
 83	cb->kernel_address = (u64) (uintptr_t) p;
 84	cb->size = cb_size;
 85
 86	return cb;
 87}
 88
 89int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
 90			u32 cb_size, u64 *handle, int ctx_id)
 91{
 92	struct hl_cb *cb;
 93	bool alloc_new_cb = true;
 94	int rc;
 95
 96	/*
 97	 * Can't use generic function to check this because of special case
 98	 * where we create a CB as part of the reset process
 99	 */
100	if ((hdev->disabled) || ((atomic_read(&hdev->in_reset)) &&
101					(ctx_id != HL_KERNEL_ASID_ID))) {
102		dev_warn_ratelimited(hdev->dev,
103			"Device is disabled or in reset. Can't create new CBs\n");
104		rc = -EBUSY;
105		goto out_err;
106	}
107
108	if (cb_size > HL_MAX_CB_SIZE) {
109		dev_err(hdev->dev,
110			"CB size %d must be less then %d\n",
111			cb_size, HL_MAX_CB_SIZE);
112		rc = -EINVAL;
113		goto out_err;
114	}
115
116	/* Minimum allocation must be PAGE SIZE */
117	if (cb_size < PAGE_SIZE)
118		cb_size = PAGE_SIZE;
119
120	if (ctx_id == HL_KERNEL_ASID_ID &&
121			cb_size <= hdev->asic_prop.cb_pool_cb_size) {
122
123		spin_lock(&hdev->cb_pool_lock);
124		if (!list_empty(&hdev->cb_pool)) {
125			cb = list_first_entry(&hdev->cb_pool, typeof(*cb),
126					pool_list);
127			list_del(&cb->pool_list);
128			spin_unlock(&hdev->cb_pool_lock);
129			alloc_new_cb = false;
130		} else {
131			spin_unlock(&hdev->cb_pool_lock);
132			dev_dbg(hdev->dev, "CB pool is empty\n");
133		}
134	}
135
136	if (alloc_new_cb) {
137		cb = hl_cb_alloc(hdev, cb_size, ctx_id);
138		if (!cb) {
139			rc = -ENOMEM;
140			goto out_err;
141		}
142	}
143
144	cb->hdev = hdev;
145	cb->ctx_id = ctx_id;
146
147	spin_lock(&mgr->cb_lock);
148	rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
149	spin_unlock(&mgr->cb_lock);
150
151	if (rc < 0) {
152		dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
153		goto release_cb;
154	}
155
156	cb->id = rc;
157
158	kref_init(&cb->refcount);
159	spin_lock_init(&cb->lock);
160
161	/*
162	 * idr is 32-bit so we can safely OR it with a mask that is above
163	 * 32 bit
164	 */
165	*handle = cb->id | HL_MMAP_CB_MASK;
166	*handle <<= PAGE_SHIFT;
167
168	hl_debugfs_add_cb(cb);
169
170	return 0;
171
172release_cb:
173	cb_do_release(hdev, cb);
174out_err:
175	*handle = 0;
176
177	return rc;
178}
179
180int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)
181{
182	struct hl_cb *cb;
183	u32 handle;
184	int rc = 0;
185
186	/*
187	 * handle was given to user to do mmap, I need to shift it back to
188	 * how the idr module gave it to me
189	 */
190	cb_handle >>= PAGE_SHIFT;
191	handle = (u32) cb_handle;
192
193	spin_lock(&mgr->cb_lock);
194
195	cb = idr_find(&mgr->cb_handles, handle);
196	if (cb) {
197		idr_remove(&mgr->cb_handles, handle);
198		spin_unlock(&mgr->cb_lock);
199		kref_put(&cb->refcount, cb_release);
200	} else {
201		spin_unlock(&mgr->cb_lock);
202		dev_err(hdev->dev,
203			"CB destroy failed, no match to handle 0x%x\n", handle);
204		rc = -EINVAL;
205	}
206
207	return rc;
208}
209
210int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
211{
212	union hl_cb_args *args = data;
213	struct hl_device *hdev = hpriv->hdev;
214	u64 handle;
215	int rc;
216
217	if (hl_device_disabled_or_in_reset(hdev)) {
218		dev_warn_ratelimited(hdev->dev,
219			"Device is %s. Can't execute CB IOCTL\n",
220			atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
221		return -EBUSY;
222	}
223
224	switch (args->in.op) {
225	case HL_CB_OP_CREATE:
226		rc = hl_cb_create(hdev, &hpriv->cb_mgr, args->in.cb_size,
227					&handle, hpriv->ctx->asid);
228		memset(args, 0, sizeof(*args));
229		args->out.cb_handle = handle;
230		break;
231	case HL_CB_OP_DESTROY:
232		rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
233					args->in.cb_handle);
234		break;
235	default:
236		rc = -ENOTTY;
237		break;
238	}
239
240	return rc;
241}
242
243static void cb_vm_close(struct vm_area_struct *vma)
244{
245	struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;
246	long new_mmap_size;
247
248	new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);
249
250	if (new_mmap_size > 0) {
251		cb->mmap_size = new_mmap_size;
252		return;
253	}
254
255	spin_lock(&cb->lock);
256	cb->mmap = false;
257	spin_unlock(&cb->lock);
258
259	hl_cb_put(cb);
260	vma->vm_private_data = NULL;
261}
262
263static const struct vm_operations_struct cb_vm_ops = {
264	.close = cb_vm_close
265};
266
267int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
268{
269	struct hl_device *hdev = hpriv->hdev;
270	struct hl_cb *cb;
271	phys_addr_t address;
272	u32 handle;
273	int rc;
274
275	handle = vma->vm_pgoff;
276
277	/* reference was taken here */
278	cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
279	if (!cb) {
280		dev_err(hdev->dev,
281			"CB mmap failed, no match to handle %d\n", handle);
282		return -EINVAL;
283	}
284
285	/* Validation check */
286	if ((vma->vm_end - vma->vm_start) != ALIGN(cb->size, PAGE_SIZE)) {
287		dev_err(hdev->dev,
288			"CB mmap failed, mmap size 0x%lx != 0x%x cb size\n",
289			vma->vm_end - vma->vm_start, cb->size);
290		rc = -EINVAL;
291		goto put_cb;
292	}
293
294	spin_lock(&cb->lock);
295
296	if (cb->mmap) {
297		dev_err(hdev->dev,
298			"CB mmap failed, CB already mmaped to user\n");
299		rc = -EINVAL;
300		goto release_lock;
301	}
302
303	cb->mmap = true;
304
305	spin_unlock(&cb->lock);
306
307	vma->vm_ops = &cb_vm_ops;
308
309	/*
310	 * Note: We're transferring the cb reference to
311	 * vma->vm_private_data here.
312	 */
313
314	vma->vm_private_data = cb;
315
316	/* Calculate address for CB */
317	address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);
318
319	rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
320					address, cb->size);
321
322	if (rc) {
323		spin_lock(&cb->lock);
324		cb->mmap = false;
325		goto release_lock;
326	}
327
328	cb->mmap_size = cb->size;
329
330	return 0;
331
332release_lock:
333	spin_unlock(&cb->lock);
334put_cb:
335	hl_cb_put(cb);
336	return rc;
337}
338
339struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
340			u32 handle)
341{
342	struct hl_cb *cb;
343
344	spin_lock(&mgr->cb_lock);
345	cb = idr_find(&mgr->cb_handles, handle);
346
347	if (!cb) {
348		spin_unlock(&mgr->cb_lock);
349		dev_warn(hdev->dev,
350			"CB get failed, no match to handle %d\n", handle);
351		return NULL;
352	}
353
354	kref_get(&cb->refcount);
355
356	spin_unlock(&mgr->cb_lock);
357
358	return cb;
359
360}
361
362void hl_cb_put(struct hl_cb *cb)
363{
364	kref_put(&cb->refcount, cb_release);
365}
366
367void hl_cb_mgr_init(struct hl_cb_mgr *mgr)
368{
369	spin_lock_init(&mgr->cb_lock);
370	idr_init(&mgr->cb_handles);
371}
372
373void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
374{
375	struct hl_cb *cb;
376	struct idr *idp;
377	u32 id;
378
379	idp = &mgr->cb_handles;
380
381	idr_for_each_entry(idp, cb, id) {
382		if (kref_put(&cb->refcount, cb_release) != 1)
383			dev_err(hdev->dev,
384				"CB %d for CTX ID %d is still alive\n",
385				id, cb->ctx_id);
386	}
387
388	idr_destroy(&mgr->cb_handles);
389}
390
391struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size)
392{
393	u64 cb_handle;
394	struct hl_cb *cb;
395	int rc;
396
397	rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
398			HL_KERNEL_ASID_ID);
399	if (rc) {
400		dev_err(hdev->dev,
401			"Failed to allocate CB for the kernel driver %d\n", rc);
402		return NULL;
403	}
404
405	cb_handle >>= PAGE_SHIFT;
406	cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);
407	/* hl_cb_get should never fail here so use kernel WARN */
408	WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle);
409	if (!cb)
410		goto destroy_cb;
411
412	return cb;
413
414destroy_cb:
415	hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb_handle << PAGE_SHIFT);
416
417	return NULL;
418}
419
420int hl_cb_pool_init(struct hl_device *hdev)
421{
422	struct hl_cb *cb;
423	int i;
424
425	INIT_LIST_HEAD(&hdev->cb_pool);
426	spin_lock_init(&hdev->cb_pool_lock);
427
428	for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
429		cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
430				HL_KERNEL_ASID_ID);
431		if (cb) {
432			cb->is_pool = true;
433			list_add(&cb->pool_list, &hdev->cb_pool);
434		} else {
435			hl_cb_pool_fini(hdev);
436			return -ENOMEM;
437		}
438	}
439
440	return 0;
441}
442
443int hl_cb_pool_fini(struct hl_device *hdev)
444{
445	struct hl_cb *cb, *tmp;
446
447	list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
448		list_del(&cb->pool_list);
449		cb_fini(hdev, cb);
450	}
451
452	return 0;
453}