1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Copyright 2016-2019 HabanaLabs, Ltd.
  5 * All Rights Reserved.
  6 */
  7
  8#include <uapi/misc/habanalabs.h>
  9#include "habanalabs.h"
 10
 11#include <linux/mm.h>
 12#include <linux/slab.h>
 13#include <linux/uaccess.h>
 14#include <linux/genalloc.h>
 15
 16static void cb_fini(struct hl_device *hdev, struct hl_cb *cb)
 17{
 18	if (cb->is_internal)
 19		gen_pool_free(hdev->internal_cb_pool,
 20				cb->kernel_address, cb->size);
 21	else
 22		hdev->asic_funcs->asic_dma_free_coherent(hdev, cb->size,
 23				(void *) (uintptr_t) cb->kernel_address,
 24				cb->bus_address);
 25
 26	kfree(cb);
 27}
 28
 29static void cb_do_release(struct hl_device *hdev, struct hl_cb *cb)
 30{
 31	if (cb->is_pool) {
 32		spin_lock(&hdev->cb_pool_lock);
 33		list_add(&cb->pool_list, &hdev->cb_pool);
 34		spin_unlock(&hdev->cb_pool_lock);
 35	} else {
 36		cb_fini(hdev, cb);
 37	}
 38}
 39
 40static void cb_release(struct kref *ref)
 41{
 42	struct hl_device *hdev;
 43	struct hl_cb *cb;
 44
 45	cb = container_of(ref, struct hl_cb, refcount);
 46	hdev = cb->hdev;
 47
 48	hl_debugfs_remove_cb(cb);
 49
 50	cb_do_release(hdev, cb);
 51}
 52
 53static struct hl_cb *hl_cb_alloc(struct hl_device *hdev, u32 cb_size,
 54					int ctx_id, bool internal_cb)
 55{
 56	struct hl_cb *cb;
 57	u32 cb_offset;
 58	void *p;
 59
 60	/*
 61	 * We use of GFP_ATOMIC here because this function can be called from
 62	 * the latency-sensitive code path for command submission. Due to H/W
 63	 * limitations in some of the ASICs, the kernel must copy the user CB
 64	 * that is designated for an external queue and actually enqueue
 65	 * the kernel's copy. Hence, we must never sleep in this code section
 66	 * and must use GFP_ATOMIC for all memory allocations.
 67	 */
 68	if (ctx_id == HL_KERNEL_ASID_ID)
 69		cb = kzalloc(sizeof(*cb), GFP_ATOMIC);
 70	else
 71		cb = kzalloc(sizeof(*cb), GFP_KERNEL);
 72
 73	if (!cb)
 74		return NULL;
 75
 76	if (internal_cb) {
 77		p = (void *) gen_pool_alloc(hdev->internal_cb_pool, cb_size);
 78		if (!p) {
 79			kfree(cb);
 80			return NULL;
 81		}
 82
 83		cb_offset = p - hdev->internal_cb_pool_virt_addr;
 84		cb->is_internal = true;
 85		cb->bus_address =  hdev->internal_cb_va_base + cb_offset;
 86	} else if (ctx_id == HL_KERNEL_ASID_ID) {
 87		p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
 88						&cb->bus_address, GFP_ATOMIC);
 89	} else {
 90		p = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, cb_size,
 91						&cb->bus_address,
 92						GFP_USER | __GFP_ZERO);
 93	}
 94
 95	if (!p) {
 96		dev_err(hdev->dev,
 97			"failed to allocate %d of dma memory for CB\n",
 98			cb_size);
 99		kfree(cb);
100		return NULL;
101	}
102
103	cb->kernel_address = (u64) (uintptr_t) p;
104	cb->size = cb_size;
105
106	return cb;
107}
108
109int hl_cb_create(struct hl_device *hdev, struct hl_cb_mgr *mgr,
110			u32 cb_size, u64 *handle, int ctx_id, bool internal_cb)
111{
112	struct hl_cb *cb;
113	bool alloc_new_cb = true;
114	int rc;
115
116	/*
117	 * Can't use generic function to check this because of special case
118	 * where we create a CB as part of the reset process
119	 */
120	if ((hdev->disabled) || ((atomic_read(&hdev->in_reset)) &&
121					(ctx_id != HL_KERNEL_ASID_ID))) {
122		dev_warn_ratelimited(hdev->dev,
123			"Device is disabled or in reset. Can't create new CBs\n");
124		rc = -EBUSY;
125		goto out_err;
126	}
127
128	if (cb_size > SZ_2M) {
129		dev_err(hdev->dev, "CB size %d must be less than %d\n",
130			cb_size, SZ_2M);
131		rc = -EINVAL;
132		goto out_err;
133	}
134
135	if (!internal_cb) {
136		/* Minimum allocation must be PAGE SIZE */
137		if (cb_size < PAGE_SIZE)
138			cb_size = PAGE_SIZE;
139
140		if (ctx_id == HL_KERNEL_ASID_ID &&
141				cb_size <= hdev->asic_prop.cb_pool_cb_size) {
142
143			spin_lock(&hdev->cb_pool_lock);
144			if (!list_empty(&hdev->cb_pool)) {
145				cb = list_first_entry(&hdev->cb_pool,
146						typeof(*cb), pool_list);
147				list_del(&cb->pool_list);
148				spin_unlock(&hdev->cb_pool_lock);
149				alloc_new_cb = false;
150			} else {
151				spin_unlock(&hdev->cb_pool_lock);
152				dev_dbg(hdev->dev, "CB pool is empty\n");
153			}
154		}
155	}
156
157	if (alloc_new_cb) {
158		cb = hl_cb_alloc(hdev, cb_size, ctx_id, internal_cb);
159		if (!cb) {
160			rc = -ENOMEM;
161			goto out_err;
162		}
163	}
164
165	cb->hdev = hdev;
166	cb->ctx_id = ctx_id;
167
168	spin_lock(&mgr->cb_lock);
169	rc = idr_alloc(&mgr->cb_handles, cb, 1, 0, GFP_ATOMIC);
170	spin_unlock(&mgr->cb_lock);
171
172	if (rc < 0) {
173		dev_err(hdev->dev, "Failed to allocate IDR for a new CB\n");
174		goto release_cb;
175	}
176
177	cb->id = rc;
178
179	kref_init(&cb->refcount);
180	spin_lock_init(&cb->lock);
181
182	/*
183	 * idr is 32-bit so we can safely OR it with a mask that is above
184	 * 32 bit
185	 */
186	*handle = cb->id | HL_MMAP_CB_MASK;
187	*handle <<= PAGE_SHIFT;
188
189	hl_debugfs_add_cb(cb);
190
191	return 0;
192
193release_cb:
194	cb_do_release(hdev, cb);
195out_err:
196	*handle = 0;
197
198	return rc;
199}
200
201int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle)
202{
203	struct hl_cb *cb;
204	u32 handle;
205	int rc = 0;
206
207	/*
208	 * handle was given to user to do mmap, I need to shift it back to
209	 * how the idr module gave it to me
210	 */
211	cb_handle >>= PAGE_SHIFT;
212	handle = (u32) cb_handle;
213
214	spin_lock(&mgr->cb_lock);
215
216	cb = idr_find(&mgr->cb_handles, handle);
217	if (cb) {
218		idr_remove(&mgr->cb_handles, handle);
219		spin_unlock(&mgr->cb_lock);
220		kref_put(&cb->refcount, cb_release);
221	} else {
222		spin_unlock(&mgr->cb_lock);
223		dev_err(hdev->dev,
224			"CB destroy failed, no match to handle 0x%x\n", handle);
225		rc = -EINVAL;
226	}
227
228	return rc;
229}
230
231int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data)
232{
233	union hl_cb_args *args = data;
234	struct hl_device *hdev = hpriv->hdev;
235	u64 handle = 0;
236	int rc;
237
238	if (hl_device_disabled_or_in_reset(hdev)) {
239		dev_warn_ratelimited(hdev->dev,
240			"Device is %s. Can't execute CB IOCTL\n",
241			atomic_read(&hdev->in_reset) ? "in_reset" : "disabled");
242		return -EBUSY;
243	}
244
245	switch (args->in.op) {
246	case HL_CB_OP_CREATE:
247		if (args->in.cb_size > HL_MAX_CB_SIZE) {
248			dev_err(hdev->dev,
249				"User requested CB size %d must be less than %d\n",
250				args->in.cb_size, HL_MAX_CB_SIZE);
251			rc = -EINVAL;
252		} else {
253			rc = hl_cb_create(hdev, &hpriv->cb_mgr,
254					args->in.cb_size, &handle,
255					hpriv->ctx->asid, false);
256		}
257
258		memset(args, 0, sizeof(*args));
259		args->out.cb_handle = handle;
260		break;
261
262	case HL_CB_OP_DESTROY:
263		rc = hl_cb_destroy(hdev, &hpriv->cb_mgr,
264					args->in.cb_handle);
265		break;
266
267	default:
268		rc = -ENOTTY;
269		break;
270	}
271
272	return rc;
273}
274
275static void cb_vm_close(struct vm_area_struct *vma)
276{
277	struct hl_cb *cb = (struct hl_cb *) vma->vm_private_data;
278	long new_mmap_size;
279
280	new_mmap_size = cb->mmap_size - (vma->vm_end - vma->vm_start);
281
282	if (new_mmap_size > 0) {
283		cb->mmap_size = new_mmap_size;
284		return;
285	}
286
287	spin_lock(&cb->lock);
288	cb->mmap = false;
289	spin_unlock(&cb->lock);
290
291	hl_cb_put(cb);
292	vma->vm_private_data = NULL;
293}
294
295static const struct vm_operations_struct cb_vm_ops = {
296	.close = cb_vm_close
297};
298
299int hl_cb_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
300{
301	struct hl_device *hdev = hpriv->hdev;
302	struct hl_cb *cb;
303	phys_addr_t address;
304	u32 handle, user_cb_size;
305	int rc;
306
307	handle = vma->vm_pgoff;
308
309	/* reference was taken here */
310	cb = hl_cb_get(hdev, &hpriv->cb_mgr, handle);
311	if (!cb) {
312		dev_err(hdev->dev,
313			"CB mmap failed, no match to handle 0x%x\n", handle);
314		return -EINVAL;
315	}
316
317	/* Validation check */
318	user_cb_size = vma->vm_end - vma->vm_start;
319	if (user_cb_size != ALIGN(cb->size, PAGE_SIZE)) {
320		dev_err(hdev->dev,
321			"CB mmap failed, mmap size 0x%lx != 0x%x cb size\n",
322			vma->vm_end - vma->vm_start, cb->size);
323		rc = -EINVAL;
324		goto put_cb;
325	}
326
327	if (!access_ok((void __user *) (uintptr_t) vma->vm_start,
328							user_cb_size)) {
329		dev_err(hdev->dev,
330			"user pointer is invalid - 0x%lx\n",
331			vma->vm_start);
332
333		rc = -EINVAL;
334		goto put_cb;
335	}
336
337	spin_lock(&cb->lock);
338
339	if (cb->mmap) {
340		dev_err(hdev->dev,
341			"CB mmap failed, CB already mmaped to user\n");
342		rc = -EINVAL;
343		goto release_lock;
344	}
345
346	cb->mmap = true;
347
348	spin_unlock(&cb->lock);
349
350	vma->vm_ops = &cb_vm_ops;
351
352	/*
353	 * Note: We're transferring the cb reference to
354	 * vma->vm_private_data here.
355	 */
356
357	vma->vm_private_data = cb;
358
359	/* Calculate address for CB */
360	address = virt_to_phys((void *) (uintptr_t) cb->kernel_address);
361
362	rc = hdev->asic_funcs->cb_mmap(hdev, vma, cb->kernel_address,
363					address, cb->size);
364
365	if (rc) {
366		spin_lock(&cb->lock);
367		cb->mmap = false;
368		goto release_lock;
369	}
370
371	cb->mmap_size = cb->size;
372
373	return 0;
374
375release_lock:
376	spin_unlock(&cb->lock);
377put_cb:
378	hl_cb_put(cb);
379	return rc;
380}
381
382struct hl_cb *hl_cb_get(struct hl_device *hdev, struct hl_cb_mgr *mgr,
383			u32 handle)
384{
385	struct hl_cb *cb;
386
387	spin_lock(&mgr->cb_lock);
388	cb = idr_find(&mgr->cb_handles, handle);
389
390	if (!cb) {
391		spin_unlock(&mgr->cb_lock);
392		dev_warn(hdev->dev,
393			"CB get failed, no match to handle 0x%x\n", handle);
394		return NULL;
395	}
396
397	kref_get(&cb->refcount);
398
399	spin_unlock(&mgr->cb_lock);
400
401	return cb;
402
403}
404
405void hl_cb_put(struct hl_cb *cb)
406{
407	kref_put(&cb->refcount, cb_release);
408}
409
410void hl_cb_mgr_init(struct hl_cb_mgr *mgr)
411{
412	spin_lock_init(&mgr->cb_lock);
413	idr_init(&mgr->cb_handles);
414}
415
416void hl_cb_mgr_fini(struct hl_device *hdev, struct hl_cb_mgr *mgr)
417{
418	struct hl_cb *cb;
419	struct idr *idp;
420	u32 id;
421
422	idp = &mgr->cb_handles;
423
424	idr_for_each_entry(idp, cb, id) {
425		if (kref_put(&cb->refcount, cb_release) != 1)
426			dev_err(hdev->dev,
427				"CB %d for CTX ID %d is still alive\n",
428				id, cb->ctx_id);
429	}
430
431	idr_destroy(&mgr->cb_handles);
432}
433
434struct hl_cb *hl_cb_kernel_create(struct hl_device *hdev, u32 cb_size,
435					bool internal_cb)
436{
437	u64 cb_handle;
438	struct hl_cb *cb;
439	int rc;
440
441	rc = hl_cb_create(hdev, &hdev->kernel_cb_mgr, cb_size, &cb_handle,
442			HL_KERNEL_ASID_ID, internal_cb);
443	if (rc) {
444		dev_err(hdev->dev,
445			"Failed to allocate CB for the kernel driver %d\n", rc);
446		return NULL;
447	}
448
449	cb_handle >>= PAGE_SHIFT;
450	cb = hl_cb_get(hdev, &hdev->kernel_cb_mgr, (u32) cb_handle);
451	/* hl_cb_get should never fail here so use kernel WARN */
452	WARN(!cb, "Kernel CB handle invalid 0x%x\n", (u32) cb_handle);
453	if (!cb)
454		goto destroy_cb;
455
456	return cb;
457
458destroy_cb:
459	hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb_handle << PAGE_SHIFT);
460
461	return NULL;
462}
463
464int hl_cb_pool_init(struct hl_device *hdev)
465{
466	struct hl_cb *cb;
467	int i;
468
469	INIT_LIST_HEAD(&hdev->cb_pool);
470	spin_lock_init(&hdev->cb_pool_lock);
471
472	for (i = 0 ; i < hdev->asic_prop.cb_pool_cb_cnt ; i++) {
473		cb = hl_cb_alloc(hdev, hdev->asic_prop.cb_pool_cb_size,
474				HL_KERNEL_ASID_ID, false);
475		if (cb) {
476			cb->is_pool = true;
477			list_add(&cb->pool_list, &hdev->cb_pool);
478		} else {
479			hl_cb_pool_fini(hdev);
480			return -ENOMEM;
481		}
482	}
483
484	return 0;
485}
486
487int hl_cb_pool_fini(struct hl_device *hdev)
488{
489	struct hl_cb *cb, *tmp;
490
491	list_for_each_entry_safe(cb, tmp, &hdev->cb_pool, pool_list) {
492		list_del(&cb->pool_list);
493		cb_fini(hdev, cb);
494	}
495
496	return 0;
497}