1// SPDX-License-Identifier: GPL-2.0
  2
  3/*
  4 * Copyright 2016-2019 HabanaLabs, Ltd.
  5 * All Rights Reserved.
  6 */
  7
  8#include "habanalabs.h"
  9
 10#include <linux/slab.h>
 11
 12static void hl_ctx_fini(struct hl_ctx *ctx)
 13{
 14	struct hl_device *hdev = ctx->hdev;
 15	int i;
 16
 17	/*
 18	 * If we arrived here, there are no jobs waiting for this context
 19	 * on its queues so we can safely remove it.
 20	 * This is because for each CS, we increment the ref count and for
 21	 * every CS that was finished we decrement it and we won't arrive
 22	 * to this function unless the ref count is 0
 23	 */
 24
 25	for (i = 0 ; i < hdev->asic_prop.max_pending_cs ; i++)
 26		dma_fence_put(ctx->cs_pending[i]);
 27
 28	kfree(ctx->cs_pending);
 29
 30	if (ctx->asid != HL_KERNEL_ASID_ID) {
 31		/* The engines are stopped as there is no executing CS, but the
 32		 * Coresight might be still working by accessing addresses
 33		 * related to the stopped engines. Hence stop it explicitly.
 34		 * Stop only if this is the compute context, as there can be
 35		 * only one compute context
 36		 */
 37		if ((hdev->in_debug) && (hdev->compute_ctx == ctx))
 38			hl_device_set_debug_mode(hdev, false);
 39
 40		hl_vm_ctx_fini(ctx);
 41		hl_asid_free(hdev, ctx->asid);
 42	} else {
 43		hl_mmu_ctx_fini(ctx);
 44	}
 45}
 46
 47void hl_ctx_do_release(struct kref *ref)
 48{
 49	struct hl_ctx *ctx;
 50
 51	ctx = container_of(ref, struct hl_ctx, refcount);
 52
 53	hl_ctx_fini(ctx);
 54
 55	if (ctx->hpriv)
 56		hl_hpriv_put(ctx->hpriv);
 57
 58	kfree(ctx);
 59}
 60
 61int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv)
 62{
 63	struct hl_ctx_mgr *mgr = &hpriv->ctx_mgr;
 64	struct hl_ctx *ctx;
 65	int rc;
 66
 67	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 68	if (!ctx) {
 69		rc = -ENOMEM;
 70		goto out_err;
 71	}
 72
 73	mutex_lock(&mgr->ctx_lock);
 74	rc = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL);
 75	mutex_unlock(&mgr->ctx_lock);
 76
 77	if (rc < 0) {
 78		dev_err(hdev->dev, "Failed to allocate IDR for a new CTX\n");
 79		goto free_ctx;
 80	}
 81
 82	ctx->handle = rc;
 83
 84	rc = hl_ctx_init(hdev, ctx, false);
 85	if (rc)
 86		goto remove_from_idr;
 87
 88	hl_hpriv_get(hpriv);
 89	ctx->hpriv = hpriv;
 90
 91	/* TODO: remove for multiple contexts per process */
 92	hpriv->ctx = ctx;
 93
 94	/* TODO: remove the following line for multiple process support */
 95	hdev->compute_ctx = ctx;
 96
 97	return 0;
 98
 99remove_from_idr:
100	mutex_lock(&mgr->ctx_lock);
101	idr_remove(&mgr->ctx_handles, ctx->handle);
102	mutex_unlock(&mgr->ctx_lock);
103free_ctx:
104	kfree(ctx);
105out_err:
106	return rc;
107}
108
109void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx)
110{
111	if (kref_put(&ctx->refcount, hl_ctx_do_release) == 1)
112		return;
113
114	dev_warn(hdev->dev,
115		"user process released device but its command submissions are still executing\n");
116}
117
118int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool is_kernel_ctx)
119{
120	int rc = 0;
121
122	ctx->hdev = hdev;
123
124	kref_init(&ctx->refcount);
125
126	ctx->cs_sequence = 1;
127	spin_lock_init(&ctx->cs_lock);
128	atomic_set(&ctx->thread_ctx_switch_token, 1);
129	ctx->thread_ctx_switch_wait_token = 0;
130	ctx->cs_pending = kcalloc(hdev->asic_prop.max_pending_cs,
131				sizeof(struct dma_fence *),
132				GFP_KERNEL);
133	if (!ctx->cs_pending)
134		return -ENOMEM;
135
136	if (is_kernel_ctx) {
137		ctx->asid = HL_KERNEL_ASID_ID; /* Kernel driver gets ASID 0 */
138		rc = hl_mmu_ctx_init(ctx);
139		if (rc) {
140			dev_err(hdev->dev, "Failed to init mmu ctx module\n");
141			goto err_free_cs_pending;
142		}
143	} else {
144		ctx->asid = hl_asid_alloc(hdev);
145		if (!ctx->asid) {
146			dev_err(hdev->dev, "No free ASID, failed to create context\n");
147			rc = -ENOMEM;
148			goto err_free_cs_pending;
149		}
150
151		rc = hl_vm_ctx_init(ctx);
152		if (rc) {
153			dev_err(hdev->dev, "Failed to init mem ctx module\n");
154			rc = -ENOMEM;
155			goto err_asid_free;
156		}
157
158		rc = hdev->asic_funcs->ctx_init(ctx);
159		if (rc) {
160			dev_err(hdev->dev, "ctx_init failed\n");
161			goto err_vm_ctx_fini;
162		}
163	}
164
165	return 0;
166
167err_vm_ctx_fini:
168	hl_vm_ctx_fini(ctx);
169err_asid_free:
170	hl_asid_free(hdev, ctx->asid);
171err_free_cs_pending:
172	kfree(ctx->cs_pending);
173
174	return rc;
175}
176
177void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx)
178{
179	kref_get(&ctx->refcount);
180}
181
182int hl_ctx_put(struct hl_ctx *ctx)
183{
184	return kref_put(&ctx->refcount, hl_ctx_do_release);
185}
186
187struct dma_fence *hl_ctx_get_fence(struct hl_ctx *ctx, u64 seq)
188{
189	struct asic_fixed_properties *asic_prop = &ctx->hdev->asic_prop;
190	struct dma_fence *fence;
191
192	spin_lock(&ctx->cs_lock);
193
194	if (seq >= ctx->cs_sequence) {
195		spin_unlock(&ctx->cs_lock);
196		return ERR_PTR(-EINVAL);
197	}
198
199	if (seq + asic_prop->max_pending_cs < ctx->cs_sequence) {
200		spin_unlock(&ctx->cs_lock);
201		return NULL;
202	}
203
204	fence = dma_fence_get(
205			ctx->cs_pending[seq & (asic_prop->max_pending_cs - 1)]);
206	spin_unlock(&ctx->cs_lock);
207
208	return fence;
209}
210
211/*
212 * hl_ctx_mgr_init - initialize the context manager
213 *
214 * @mgr: pointer to context manager structure
215 *
216 * This manager is an object inside the hpriv object of the user process.
217 * The function is called when a user process opens the FD.
218 */
219void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr)
220{
221	mutex_init(&mgr->ctx_lock);
222	idr_init(&mgr->ctx_handles);
223}
224
225/*
226 * hl_ctx_mgr_fini - finalize the context manager
227 *
228 * @hdev: pointer to device structure
229 * @mgr: pointer to context manager structure
230 *
231 * This function goes over all the contexts in the manager and frees them.
232 * It is called when a process closes the FD.
233 */
234void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr)
235{
236	struct hl_ctx *ctx;
237	struct idr *idp;
238	u32 id;
239
240	idp = &mgr->ctx_handles;
241
242	idr_for_each_entry(idp, ctx, id)
243		hl_ctx_free(hdev, ctx);
244
245	idr_destroy(&mgr->ctx_handles);
246	mutex_destroy(&mgr->ctx_lock);
247}