Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2016-2019 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8#include "habanalabs.h"
9
10#include <linux/slab.h>
11
12/*
13 * hl_queue_add_ptr - add to pi or ci and checks if it wraps around
14 *
15 * @ptr: the current pi/ci value
16 * @val: the amount to add
17 *
18 * Add val to ptr. It can go until twice the queue length.
19 */
20inline u32 hl_hw_queue_add_ptr(u32 ptr, u16 val)
21{
22 ptr += val;
23 ptr &= ((HL_QUEUE_LENGTH << 1) - 1);
24 return ptr;
25}
26static inline int queue_ci_get(atomic_t *ci, u32 queue_len)
27{
28 return atomic_read(ci) & ((queue_len << 1) - 1);
29}
30
31static inline int queue_free_slots(struct hl_hw_queue *q, u32 queue_len)
32{
33 int delta = (q->pi - queue_ci_get(&q->ci, queue_len));
34
35 if (delta >= 0)
36 return (queue_len - delta);
37 else
38 return (abs(delta) - queue_len);
39}
40
41void hl_hw_queue_update_ci(struct hl_cs *cs)
42{
43 struct hl_device *hdev = cs->ctx->hdev;
44 struct hl_hw_queue *q;
45 int i;
46
47 if (hdev->disabled)
48 return;
49
50 q = &hdev->kernel_queues[0];
51
52 /* There are no internal queues if H/W queues are being used */
53 if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW)
54 return;
55
56 /* We must increment CI for every queue that will never get a
57 * completion, there are 2 scenarios this can happen:
58 * 1. All queues of a non completion CS will never get a completion.
59 * 2. Internal queues never gets completion.
60 */
61 for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {
62 if (!cs_needs_completion(cs) || q->queue_type == QUEUE_TYPE_INT)
63 atomic_add(cs->jobs_in_queue_cnt[i], &q->ci);
64 }
65}
66
67/*
68 * hl_hw_queue_submit_bd() - Submit a buffer descriptor to an external or a
69 * H/W queue.
70 * @hdev: pointer to habanalabs device structure
71 * @q: pointer to habanalabs queue structure
72 * @ctl: BD's control word
73 * @len: BD's length
74 * @ptr: BD's pointer
75 *
76 * This function assumes there is enough space on the queue to submit a new
77 * BD to it. It initializes the next BD and calls the device specific
78 * function to set the pi (and doorbell)
79 *
80 * This function must be called when the scheduler mutex is taken
81 *
82 */
83void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,
84 u32 ctl, u32 len, u64 ptr)
85{
86 struct hl_bd *bd;
87
88 bd = q->kernel_address;
89 bd += hl_pi_2_offset(q->pi);
90 bd->ctl = cpu_to_le32(ctl);
91 bd->len = cpu_to_le32(len);
92 bd->ptr = cpu_to_le64(ptr);
93
94 q->pi = hl_queue_inc_ptr(q->pi);
95 hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
96}
97
98/*
99 * ext_queue_sanity_checks - perform some sanity checks on external queue
100 *
101 * @hdev : pointer to hl_device structure
102 * @q : pointer to hl_hw_queue structure
103 * @num_of_entries : how many entries to check for space
104 * @reserve_cq_entry : whether to reserve an entry in the cq
105 *
106 * H/W queues spinlock should be taken before calling this function
107 *
108 * Perform the following:
109 * - Make sure we have enough space in the h/w queue
110 * - Make sure we have enough space in the completion queue
111 * - Reserve space in the completion queue (needs to be reversed if there
112 * is a failure down the road before the actual submission of work). Only
113 * do this action if reserve_cq_entry is true
114 *
115 */
116static int ext_queue_sanity_checks(struct hl_device *hdev,
117 struct hl_hw_queue *q, int num_of_entries,
118 bool reserve_cq_entry)
119{
120 atomic_t *free_slots =
121 &hdev->completion_queue[q->cq_id].free_slots_cnt;
122 int free_slots_cnt;
123
124 /* Check we have enough space in the queue */
125 free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);
126
127 if (free_slots_cnt < num_of_entries) {
128 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
129 q->hw_queue_id, num_of_entries);
130 return -EAGAIN;
131 }
132
133 if (reserve_cq_entry) {
134 /*
135 * Check we have enough space in the completion queue
136 * Add -1 to counter (decrement) unless counter was already 0
137 * In that case, CQ is full so we can't submit a new CB because
138 * we won't get ack on its completion
139 * atomic_add_unless will return 0 if counter was already 0
140 */
141 if (atomic_add_negative(num_of_entries * -1, free_slots)) {
142 dev_dbg(hdev->dev, "No space for %d on CQ %d\n",
143 num_of_entries, q->hw_queue_id);
144 atomic_add(num_of_entries, free_slots);
145 return -EAGAIN;
146 }
147 }
148
149 return 0;
150}
151
152/*
153 * int_queue_sanity_checks - perform some sanity checks on internal queue
154 *
155 * @hdev : pointer to hl_device structure
156 * @q : pointer to hl_hw_queue structure
157 * @num_of_entries : how many entries to check for space
158 *
159 * H/W queues spinlock should be taken before calling this function
160 *
161 * Perform the following:
162 * - Make sure we have enough space in the h/w queue
163 *
164 */
165static int int_queue_sanity_checks(struct hl_device *hdev,
166 struct hl_hw_queue *q,
167 int num_of_entries)
168{
169 int free_slots_cnt;
170
171 if (num_of_entries > q->int_queue_len) {
172 dev_err(hdev->dev,
173 "Cannot populate queue %u with %u jobs\n",
174 q->hw_queue_id, num_of_entries);
175 return -ENOMEM;
176 }
177
178 /* Check we have enough space in the queue */
179 free_slots_cnt = queue_free_slots(q, q->int_queue_len);
180
181 if (free_slots_cnt < num_of_entries) {
182 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
183 q->hw_queue_id, num_of_entries);
184 return -EAGAIN;
185 }
186
187 return 0;
188}
189
190/*
191 * hw_queue_sanity_checks() - Make sure we have enough space in the h/w queue
192 * @hdev: Pointer to hl_device structure.
193 * @q: Pointer to hl_hw_queue structure.
194 * @num_of_entries: How many entries to check for space.
195 *
196 * Notice: We do not reserve queue entries so this function mustn't be called
197 * more than once per CS for the same queue
198 *
199 */
200static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,
201 int num_of_entries)
202{
203 int free_slots_cnt;
204
205 /* Check we have enough space in the queue */
206 free_slots_cnt = queue_free_slots(q, HL_QUEUE_LENGTH);
207
208 if (free_slots_cnt < num_of_entries) {
209 dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",
210 q->hw_queue_id, num_of_entries);
211 return -EAGAIN;
212 }
213
214 return 0;
215}
216
217/*
218 * hl_hw_queue_send_cb_no_cmpl - send a single CB (not a JOB) without completion
219 *
220 * @hdev: pointer to hl_device structure
221 * @hw_queue_id: Queue's type
222 * @cb_size: size of CB
223 * @cb_ptr: pointer to CB location
224 *
225 * This function sends a single CB, that must NOT generate a completion entry.
226 * Sending CPU messages can be done instead via 'hl_hw_queue_submit_bd()'
227 */
228int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,
229 u32 cb_size, u64 cb_ptr)
230{
231 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
232 int rc = 0;
233
234 hdev->asic_funcs->hw_queues_lock(hdev);
235
236 if (hdev->disabled) {
237 rc = -EPERM;
238 goto out;
239 }
240
241 /*
242 * hl_hw_queue_send_cb_no_cmpl() is called for queues of a H/W queue
243 * type only on init phase, when the queues are empty and being tested,
244 * so there is no need for sanity checks.
245 */
246 if (q->queue_type != QUEUE_TYPE_HW) {
247 rc = ext_queue_sanity_checks(hdev, q, 1, false);
248 if (rc)
249 goto out;
250 }
251
252 hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);
253
254out:
255 hdev->asic_funcs->hw_queues_unlock(hdev);
256
257 return rc;
258}
259
260/*
261 * ext_queue_schedule_job - submit a JOB to an external queue
262 *
263 * @job: pointer to the job that needs to be submitted to the queue
264 *
265 * This function must be called when the scheduler mutex is taken
266 *
267 */
268static void ext_queue_schedule_job(struct hl_cs_job *job)
269{
270 struct hl_device *hdev = job->cs->ctx->hdev;
271 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
272 struct hl_cq_entry cq_pkt;
273 struct hl_cq *cq;
274 u64 cq_addr;
275 struct hl_cb *cb;
276 u32 ctl;
277 u32 len;
278 u64 ptr;
279
280 /*
281 * Update the JOB ID inside the BD CTL so the device would know what
282 * to write in the completion queue
283 */
284 ctl = ((q->pi << BD_CTL_SHADOW_INDEX_SHIFT) & BD_CTL_SHADOW_INDEX_MASK);
285
286 cb = job->patched_cb;
287 len = job->job_cb_size;
288 ptr = cb->bus_address;
289
290 /* Skip completion flow in case this is a non completion CS */
291 if (!cs_needs_completion(job->cs))
292 goto submit_bd;
293
294 cq_pkt.data = cpu_to_le32(
295 ((q->pi << CQ_ENTRY_SHADOW_INDEX_SHIFT)
296 & CQ_ENTRY_SHADOW_INDEX_MASK) |
297 FIELD_PREP(CQ_ENTRY_SHADOW_INDEX_VALID_MASK, 1) |
298 FIELD_PREP(CQ_ENTRY_READY_MASK, 1));
299
300 /*
301 * No need to protect pi_offset because scheduling to the
302 * H/W queues is done under the scheduler mutex
303 *
304 * No need to check if CQ is full because it was already
305 * checked in ext_queue_sanity_checks
306 */
307 cq = &hdev->completion_queue[q->cq_id];
308 cq_addr = cq->bus_address + cq->pi * sizeof(struct hl_cq_entry);
309
310 hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len,
311 job->user_cb_size,
312 cq_addr,
313 le32_to_cpu(cq_pkt.data),
314 q->msi_vec,
315 job->contains_dma_pkt);
316
317 q->shadow_queue[hl_pi_2_offset(q->pi)] = job;
318
319 cq->pi = hl_cq_inc_ptr(cq->pi);
320
321submit_bd:
322 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
323}
324
325/*
326 * int_queue_schedule_job - submit a JOB to an internal queue
327 *
328 * @job: pointer to the job that needs to be submitted to the queue
329 *
330 * This function must be called when the scheduler mutex is taken
331 *
332 */
333static void int_queue_schedule_job(struct hl_cs_job *job)
334{
335 struct hl_device *hdev = job->cs->ctx->hdev;
336 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
337 struct hl_bd bd;
338 __le64 *pi;
339
340 bd.ctl = 0;
341 bd.len = cpu_to_le32(job->job_cb_size);
342
343 if (job->is_kernel_allocated_cb)
344 /* bus_address is actually a mmu mapped address
345 * allocated from an internal pool
346 */
347 bd.ptr = cpu_to_le64(job->user_cb->bus_address);
348 else
349 bd.ptr = cpu_to_le64((u64) (uintptr_t) job->user_cb);
350
351 pi = q->kernel_address + (q->pi & (q->int_queue_len - 1)) * sizeof(bd);
352
353 q->pi++;
354 q->pi &= ((q->int_queue_len << 1) - 1);
355
356 hdev->asic_funcs->pqe_write(hdev, pi, &bd);
357
358 hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);
359}
360
361/*
362 * hw_queue_schedule_job - submit a JOB to a H/W queue
363 *
364 * @job: pointer to the job that needs to be submitted to the queue
365 *
366 * This function must be called when the scheduler mutex is taken
367 *
368 */
369static void hw_queue_schedule_job(struct hl_cs_job *job)
370{
371 struct hl_device *hdev = job->cs->ctx->hdev;
372 struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];
373 u64 ptr;
374 u32 offset, ctl, len;
375
376 /*
377 * Upon PQE completion, COMP_DATA is used as the write data to the
378 * completion queue (QMAN HBW message), and COMP_OFFSET is used as the
379 * write address offset in the SM block (QMAN LBW message).
380 * The write address offset is calculated as "COMP_OFFSET << 2".
381 */
382 offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1);
383 ctl = ((offset << BD_CTL_COMP_OFFSET_SHIFT) & BD_CTL_COMP_OFFSET_MASK) |
384 ((q->pi << BD_CTL_COMP_DATA_SHIFT) & BD_CTL_COMP_DATA_MASK);
385
386 len = job->job_cb_size;
387
388 /*
389 * A patched CB is created only if a user CB was allocated by driver and
390 * MMU is disabled. If MMU is enabled, the user CB should be used
391 * instead. If the user CB wasn't allocated by driver, assume that it
392 * holds an address.
393 */
394 if (job->patched_cb)
395 ptr = job->patched_cb->bus_address;
396 else if (job->is_kernel_allocated_cb)
397 ptr = job->user_cb->bus_address;
398 else
399 ptr = (u64) (uintptr_t) job->user_cb;
400
401 hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
402}
403
404static int init_signal_cs(struct hl_device *hdev,
405 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
406{
407 struct hl_sync_stream_properties *prop;
408 struct hl_hw_sob *hw_sob;
409 u32 q_idx;
410 int rc = 0;
411
412 q_idx = job->hw_queue_id;
413 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
414 hw_sob = &prop->hw_sob[prop->curr_sob_offset];
415
416 cs_cmpl->hw_sob = hw_sob;
417 cs_cmpl->sob_val = prop->next_sob_val;
418
419 dev_dbg(hdev->dev,
420 "generate signal CB, sob_id: %d, sob val: %u, q_idx: %d, seq: %llu\n",
421 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, q_idx,
422 cs_cmpl->cs_seq);
423
424 /* we set an EB since we must make sure all oeprations are done
425 * when sending the signal
426 */
427 hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,
428 cs_cmpl->hw_sob->sob_id, 0, true);
429
430 rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1,
431 false);
432
433 job->cs->sob_addr_offset = hw_sob->sob_addr;
434 job->cs->initial_sob_count = prop->next_sob_val - 1;
435
436 return rc;
437}
438
439void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,
440 struct hl_cs *cs, struct hl_cs_job *job,
441 struct hl_cs_compl *cs_cmpl)
442{
443 struct hl_cs_encaps_sig_handle *handle = cs->encaps_sig_hdl;
444 u32 offset = 0;
445
446 cs_cmpl->hw_sob = handle->hw_sob;
447
448 /* Note that encaps_sig_wait_offset was validated earlier in the flow
449 * for offset value which exceeds the max reserved signal count.
450 * always decrement 1 of the offset since when the user
451 * set offset 1 for example he mean to wait only for the first
452 * signal only, which will be pre_sob_val, and if he set offset 2
453 * then the value required is (pre_sob_val + 1) and so on...
454 * if user set wait offset to 0, then treat it as legacy wait cs,
455 * wait for the next signal.
456 */
457 if (job->encaps_sig_wait_offset)
458 offset = job->encaps_sig_wait_offset - 1;
459
460 cs_cmpl->sob_val = handle->pre_sob_val + offset;
461}
462
463static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,
464 struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
465{
466 struct hl_gen_wait_properties wait_prop;
467 struct hl_sync_stream_properties *prop;
468 struct hl_cs_compl *signal_cs_cmpl;
469 u32 q_idx;
470
471 q_idx = job->hw_queue_id;
472 prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
473
474 signal_cs_cmpl = container_of(cs->signal_fence,
475 struct hl_cs_compl,
476 base_fence);
477
478 if (cs->encaps_signals) {
479 /* use the encaps signal handle stored earlier in the flow
480 * and set the SOB information from the encaps
481 * signals handle
482 */
483 hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl);
484
485 dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, offset: %u\n",
486 cs->encaps_sig_hdl->q_idx,
487 cs->encaps_sig_hdl->cs_seq,
488 cs_cmpl->sob_val,
489 job->encaps_sig_wait_offset);
490 } else {
491 /* Copy the SOB id and value of the signal CS */
492 cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob;
493 cs_cmpl->sob_val = signal_cs_cmpl->sob_val;
494 }
495
496 /* check again if the signal cs already completed.
497 * if yes then don't send any wait cs since the hw_sob
498 * could be in reset already. if signal is not completed
499 * then get refcount to hw_sob to prevent resetting the sob
500 * while wait cs is not submitted.
501 * note that this check is protected by two locks,
502 * hw queue lock and completion object lock,
503 * and the same completion object lock also protects
504 * the hw_sob reset handler function.
505 * The hw_queue lock prevent out of sync of hw_sob
506 * refcount value, changed by signal/wait flows.
507 */
508 spin_lock(&signal_cs_cmpl->lock);
509
510 if (completion_done(&cs->signal_fence->completion)) {
511 spin_unlock(&signal_cs_cmpl->lock);
512 return -EINVAL;
513 }
514
515 kref_get(&cs_cmpl->hw_sob->kref);
516
517 spin_unlock(&signal_cs_cmpl->lock);
518
519 dev_dbg(hdev->dev,
520 "generate wait CB, sob_id: %d, sob_val: 0x%x, mon_id: %d, q_idx: %d, seq: %llu\n",
521 cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val,
522 prop->base_mon_id, q_idx, cs->sequence);
523
524 wait_prop.data = (void *) job->patched_cb;
525 wait_prop.sob_base = cs_cmpl->hw_sob->sob_id;
526 wait_prop.sob_mask = 0x1;
527 wait_prop.sob_val = cs_cmpl->sob_val;
528 wait_prop.mon_id = prop->base_mon_id;
529 wait_prop.q_idx = q_idx;
530 wait_prop.size = 0;
531
532 hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);
533
534 mb();
535 hl_fence_put(cs->signal_fence);
536 cs->signal_fence = NULL;
537
538 return 0;
539}
540
541/*
542 * init_signal_wait_cs - initialize a signal/wait CS
543 * @cs: pointer to the signal/wait CS
544 *
545 * H/W queues spinlock should be taken before calling this function
546 */
547static int init_signal_wait_cs(struct hl_cs *cs)
548{
549 struct hl_ctx *ctx = cs->ctx;
550 struct hl_device *hdev = ctx->hdev;
551 struct hl_cs_job *job;
552 struct hl_cs_compl *cs_cmpl =
553 container_of(cs->fence, struct hl_cs_compl, base_fence);
554 int rc = 0;
555
556 /* There is only one job in a signal/wait CS */
557 job = list_first_entry(&cs->job_list, struct hl_cs_job,
558 cs_node);
559
560 if (cs->type & CS_TYPE_SIGNAL)
561 rc = init_signal_cs(hdev, job, cs_cmpl);
562 else if (cs->type & CS_TYPE_WAIT)
563 rc = init_wait_cs(hdev, cs, job, cs_cmpl);
564
565 return rc;
566}
567
568static int encaps_sig_first_staged_cs_handler
569 (struct hl_device *hdev, struct hl_cs *cs)
570{
571 struct hl_cs_compl *cs_cmpl =
572 container_of(cs->fence,
573 struct hl_cs_compl, base_fence);
574 struct hl_cs_encaps_sig_handle *encaps_sig_hdl;
575 struct hl_encaps_signals_mgr *mgr;
576 int rc = 0;
577
578 mgr = &cs->ctx->sig_mgr;
579
580 spin_lock(&mgr->lock);
581 encaps_sig_hdl = idr_find(&mgr->handles, cs->encaps_sig_hdl_id);
582 if (encaps_sig_hdl) {
583 /*
584 * Set handler CS sequence,
585 * the CS which contains the encapsulated signals.
586 */
587 encaps_sig_hdl->cs_seq = cs->sequence;
588 /* store the handle and set encaps signal indication,
589 * to be used later in cs_do_release to put the last
590 * reference to encaps signals handlers.
591 */
592 cs_cmpl->encaps_signals = true;
593 cs_cmpl->encaps_sig_hdl = encaps_sig_hdl;
594
595 /* set hw_sob pointer in completion object
596 * since it's used in cs_do_release flow to put
597 * refcount to sob
598 */
599 cs_cmpl->hw_sob = encaps_sig_hdl->hw_sob;
600 cs_cmpl->sob_val = encaps_sig_hdl->pre_sob_val +
601 encaps_sig_hdl->count;
602
603 dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob(%u), val(%u)\n",
604 cs->sequence, encaps_sig_hdl->id,
605 encaps_sig_hdl->count,
606 encaps_sig_hdl->q_idx,
607 cs_cmpl->hw_sob->sob_id,
608 cs_cmpl->sob_val);
609
610 } else {
611 dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n",
612 cs->encaps_sig_hdl_id);
613 rc = -EINVAL;
614 }
615
616 spin_unlock(&mgr->lock);
617
618 return rc;
619}
620
621/*
622 * hl_hw_queue_schedule_cs - schedule a command submission
623 * @cs: pointer to the CS
624 */
625int hl_hw_queue_schedule_cs(struct hl_cs *cs)
626{
627 enum hl_device_status status;
628 struct hl_cs_counters_atomic *cntr;
629 struct hl_ctx *ctx = cs->ctx;
630 struct hl_device *hdev = ctx->hdev;
631 struct hl_cs_job *job, *tmp;
632 struct hl_hw_queue *q;
633 int rc = 0, i, cq_cnt;
634 bool first_entry;
635 u32 max_queues;
636
637 cntr = &hdev->aggregated_cs_counters;
638
639 hdev->asic_funcs->hw_queues_lock(hdev);
640
641 if (!hl_device_operational(hdev, &status)) {
642 atomic64_inc(&cntr->device_in_reset_drop_cnt);
643 atomic64_inc(&ctx->cs_counters.device_in_reset_drop_cnt);
644 dev_err(hdev->dev,
645 "device is %s, CS rejected!\n", hdev->status[status]);
646 rc = -EPERM;
647 goto out;
648 }
649
650 max_queues = hdev->asic_prop.max_queues;
651
652 q = &hdev->kernel_queues[0];
653 for (i = 0, cq_cnt = 0 ; i < max_queues ; i++, q++) {
654 if (cs->jobs_in_queue_cnt[i]) {
655 switch (q->queue_type) {
656 case QUEUE_TYPE_EXT:
657 rc = ext_queue_sanity_checks(hdev, q,
658 cs->jobs_in_queue_cnt[i],
659 cs_needs_completion(cs) ?
660 true : false);
661 break;
662 case QUEUE_TYPE_INT:
663 rc = int_queue_sanity_checks(hdev, q,
664 cs->jobs_in_queue_cnt[i]);
665 break;
666 case QUEUE_TYPE_HW:
667 rc = hw_queue_sanity_checks(hdev, q,
668 cs->jobs_in_queue_cnt[i]);
669 break;
670 default:
671 dev_err(hdev->dev, "Queue type %d is invalid\n",
672 q->queue_type);
673 rc = -EINVAL;
674 break;
675 }
676
677 if (rc) {
678 atomic64_inc(
679 &ctx->cs_counters.queue_full_drop_cnt);
680 atomic64_inc(&cntr->queue_full_drop_cnt);
681 goto unroll_cq_resv;
682 }
683
684 if (q->queue_type == QUEUE_TYPE_EXT)
685 cq_cnt++;
686 }
687 }
688
689 if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) {
690 rc = init_signal_wait_cs(cs);
691 if (rc)
692 goto unroll_cq_resv;
693 } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) {
694 rc = hdev->asic_funcs->collective_wait_init_cs(cs);
695 if (rc)
696 goto unroll_cq_resv;
697 }
698
699 rc = hdev->asic_funcs->pre_schedule_cs(cs);
700 if (rc) {
701 dev_err(hdev->dev,
702 "Failed in pre-submission operations of CS %d.%llu\n",
703 ctx->asid, cs->sequence);
704 goto unroll_cq_resv;
705 }
706
707 hdev->shadow_cs_queue[cs->sequence &
708 (hdev->asic_prop.max_pending_cs - 1)] = cs;
709
710 if (cs->encaps_signals && cs->staged_first) {
711 rc = encaps_sig_first_staged_cs_handler(hdev, cs);
712 if (rc)
713 goto unroll_cq_resv;
714 }
715
716 spin_lock(&hdev->cs_mirror_lock);
717
718 /* Verify staged CS exists and add to the staged list */
719 if (cs->staged_cs && !cs->staged_first) {
720 struct hl_cs *staged_cs;
721
722 staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);
723 if (!staged_cs) {
724 dev_err(hdev->dev,
725 "Cannot find staged submission sequence %llu",
726 cs->staged_sequence);
727 rc = -EINVAL;
728 goto unlock_cs_mirror;
729 }
730
731 if (is_staged_cs_last_exists(hdev, staged_cs)) {
732 dev_err(hdev->dev,
733 "Staged submission sequence %llu already submitted",
734 cs->staged_sequence);
735 rc = -EINVAL;
736 goto unlock_cs_mirror;
737 }
738
739 list_add_tail(&cs->staged_cs_node, &staged_cs->staged_cs_node);
740
741 /* update stream map of the first CS */
742 if (hdev->supports_wait_for_multi_cs)
743 staged_cs->fence->stream_master_qid_map |=
744 cs->fence->stream_master_qid_map;
745 }
746
747 list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);
748
749 /* Queue TDR if the CS is the first entry and if timeout is wanted */
750 first_entry = list_first_entry(&hdev->cs_mirror_list,
751 struct hl_cs, mirror_node) == cs;
752 if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&
753 first_entry && cs_needs_timeout(cs)) {
754 cs->tdr_active = true;
755 schedule_delayed_work(&cs->work_tdr, cs->timeout_jiffies);
756
757 }
758
759 spin_unlock(&hdev->cs_mirror_lock);
760
761 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
762 switch (job->queue_type) {
763 case QUEUE_TYPE_EXT:
764 ext_queue_schedule_job(job);
765 break;
766 case QUEUE_TYPE_INT:
767 int_queue_schedule_job(job);
768 break;
769 case QUEUE_TYPE_HW:
770 hw_queue_schedule_job(job);
771 break;
772 default:
773 break;
774 }
775
776 cs->submitted = true;
777
778 goto out;
779
780unlock_cs_mirror:
781 spin_unlock(&hdev->cs_mirror_lock);
782unroll_cq_resv:
783 q = &hdev->kernel_queues[0];
784 for (i = 0 ; (i < max_queues) && (cq_cnt > 0) ; i++, q++) {
785 if ((q->queue_type == QUEUE_TYPE_EXT) &&
786 (cs->jobs_in_queue_cnt[i])) {
787 atomic_t *free_slots =
788 &hdev->completion_queue[i].free_slots_cnt;
789 atomic_add(cs->jobs_in_queue_cnt[i], free_slots);
790 cq_cnt--;
791 }
792 }
793
794out:
795 hdev->asic_funcs->hw_queues_unlock(hdev);
796
797 return rc;
798}
799
800/*
801 * hl_hw_queue_inc_ci_kernel - increment ci for kernel's queue
802 *
803 * @hdev: pointer to hl_device structure
804 * @hw_queue_id: which queue to increment its ci
805 */
806void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)
807{
808 struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];
809
810 atomic_inc(&q->ci);
811}
812
813static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,
814 bool is_cpu_queue)
815{
816 void *p;
817 int rc;
818
819 if (is_cpu_queue)
820 p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address);
821 else
822 p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,
823 GFP_KERNEL | __GFP_ZERO);
824 if (!p)
825 return -ENOMEM;
826
827 q->kernel_address = p;
828
829 q->shadow_queue = kmalloc_array(HL_QUEUE_LENGTH, sizeof(struct hl_cs_job *), GFP_KERNEL);
830 if (!q->shadow_queue) {
831 dev_err(hdev->dev,
832 "Failed to allocate shadow queue for H/W queue %d\n",
833 q->hw_queue_id);
834 rc = -ENOMEM;
835 goto free_queue;
836 }
837
838 /* Make sure read/write pointers are initialized to start of queue */
839 atomic_set(&q->ci, 0);
840 q->pi = 0;
841
842 return 0;
843
844free_queue:
845 if (is_cpu_queue)
846 hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);
847 else
848 hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,
849 q->bus_address);
850
851 return rc;
852}
853
854static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
855{
856 void *p;
857
858 p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,
859 &q->bus_address, &q->int_queue_len);
860 if (!p) {
861 dev_err(hdev->dev,
862 "Failed to get base address for internal queue %d\n",
863 q->hw_queue_id);
864 return -EFAULT;
865 }
866
867 q->kernel_address = p;
868 q->pi = 0;
869 atomic_set(&q->ci, 0);
870
871 return 0;
872}
873
874static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
875{
876 return ext_and_cpu_queue_init(hdev, q, true);
877}
878
879static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
880{
881 return ext_and_cpu_queue_init(hdev, q, false);
882}
883
884static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)
885{
886 void *p;
887
888 p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,
889 GFP_KERNEL | __GFP_ZERO);
890 if (!p)
891 return -ENOMEM;
892
893 q->kernel_address = p;
894
895 /* Make sure read/write pointers are initialized to start of queue */
896 atomic_set(&q->ci, 0);
897 q->pi = 0;
898
899 return 0;
900}
901
902static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx)
903{
904 struct hl_sync_stream_properties *sync_stream_prop;
905 struct asic_fixed_properties *prop = &hdev->asic_prop;
906 struct hl_hw_sob *hw_sob;
907 int sob, reserved_mon_idx, queue_idx;
908
909 sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
910
911 /* We use 'collective_mon_idx' as a running index in order to reserve
912 * monitors for collective master/slave queues.
913 * collective master queue gets 2 reserved monitors
914 * collective slave queue gets 1 reserved monitor
915 */
916 if (hdev->kernel_queues[q_idx].collective_mode ==
917 HL_COLLECTIVE_MASTER) {
918 reserved_mon_idx = hdev->collective_mon_idx;
919
920 /* reserve the first monitor for collective master queue */
921 sync_stream_prop->collective_mstr_mon_id[0] =
922 prop->collective_first_mon + reserved_mon_idx;
923
924 /* reserve the second monitor for collective master queue */
925 sync_stream_prop->collective_mstr_mon_id[1] =
926 prop->collective_first_mon + reserved_mon_idx + 1;
927
928 hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS;
929 } else if (hdev->kernel_queues[q_idx].collective_mode ==
930 HL_COLLECTIVE_SLAVE) {
931 reserved_mon_idx = hdev->collective_mon_idx++;
932
933 /* reserve a monitor for collective slave queue */
934 sync_stream_prop->collective_slave_mon_id =
935 prop->collective_first_mon + reserved_mon_idx;
936 }
937
938 if (!hdev->kernel_queues[q_idx].supports_sync_stream)
939 return;
940
941 queue_idx = hdev->sync_stream_queue_idx++;
942
943 sync_stream_prop->base_sob_id = prop->sync_stream_first_sob +
944 (queue_idx * HL_RSVD_SOBS);
945 sync_stream_prop->base_mon_id = prop->sync_stream_first_mon +
946 (queue_idx * HL_RSVD_MONS);
947 sync_stream_prop->next_sob_val = 1;
948 sync_stream_prop->curr_sob_offset = 0;
949
950 for (sob = 0 ; sob < HL_RSVD_SOBS ; sob++) {
951 hw_sob = &sync_stream_prop->hw_sob[sob];
952 hw_sob->hdev = hdev;
953 hw_sob->sob_id = sync_stream_prop->base_sob_id + sob;
954 hw_sob->sob_addr =
955 hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);
956 hw_sob->q_idx = q_idx;
957 kref_init(&hw_sob->kref);
958 }
959}
960
961static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx)
962{
963 struct hl_sync_stream_properties *prop =
964 &hdev->kernel_queues[q_idx].sync_stream_prop;
965
966 /*
967 * In case we got here due to a stuck CS, the refcnt might be bigger
968 * than 1 and therefore we reset it.
969 */
970 kref_init(&prop->hw_sob[prop->curr_sob_offset].kref);
971 prop->curr_sob_offset = 0;
972 prop->next_sob_val = 1;
973}
974
975/*
976 * queue_init - main initialization function for H/W queue object
977 *
978 * @hdev: pointer to hl_device device structure
979 * @q: pointer to hl_hw_queue queue structure
980 * @hw_queue_id: The id of the H/W queue
981 *
982 * Allocate dma-able memory for the queue and initialize fields
983 * Returns 0 on success
984 */
985static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q,
986 u32 hw_queue_id)
987{
988 int rc;
989
990 q->hw_queue_id = hw_queue_id;
991
992 switch (q->queue_type) {
993 case QUEUE_TYPE_EXT:
994 rc = ext_queue_init(hdev, q);
995 break;
996 case QUEUE_TYPE_INT:
997 rc = int_queue_init(hdev, q);
998 break;
999 case QUEUE_TYPE_CPU:
1000 rc = cpu_queue_init(hdev, q);
1001 break;
1002 case QUEUE_TYPE_HW:
1003 rc = hw_queue_init(hdev, q);
1004 break;
1005 case QUEUE_TYPE_NA:
1006 q->valid = 0;
1007 return 0;
1008 default:
1009 dev_crit(hdev->dev, "wrong queue type %d during init\n",
1010 q->queue_type);
1011 rc = -EINVAL;
1012 break;
1013 }
1014
1015 sync_stream_queue_init(hdev, q->hw_queue_id);
1016
1017 if (rc)
1018 return rc;
1019
1020 q->valid = 1;
1021
1022 return 0;
1023}
1024
1025/*
1026 * hw_queue_fini - destroy queue
1027 *
1028 * @hdev: pointer to hl_device device structure
1029 * @q: pointer to hl_hw_queue queue structure
1030 *
1031 * Free the queue memory
1032 */
1033static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)
1034{
1035 if (!q->valid)
1036 return;
1037
1038 /*
1039 * If we arrived here, there are no jobs waiting on this queue
1040 * so we can safely remove it.
1041 * This is because this function can only called when:
1042 * 1. Either a context is deleted, which only can occur if all its
1043 * jobs were finished
1044 * 2. A context wasn't able to be created due to failure or timeout,
1045 * which means there are no jobs on the queue yet
1046 *
1047 * The only exception are the queues of the kernel context, but
1048 * if they are being destroyed, it means that the entire module is
1049 * being removed. If the module is removed, it means there is no open
1050 * user context. It also means that if a job was submitted by
1051 * the kernel driver (e.g. context creation), the job itself was
1052 * released by the kernel driver when a timeout occurred on its
1053 * Completion. Thus, we don't need to release it again.
1054 */
1055
1056 if (q->queue_type == QUEUE_TYPE_INT)
1057 return;
1058
1059 kfree(q->shadow_queue);
1060
1061 if (q->queue_type == QUEUE_TYPE_CPU)
1062 hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);
1063 else
1064 hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,
1065 q->bus_address);
1066}
1067
1068int hl_hw_queues_create(struct hl_device *hdev)
1069{
1070 struct asic_fixed_properties *asic = &hdev->asic_prop;
1071 struct hl_hw_queue *q;
1072 int i, rc, q_ready_cnt;
1073
1074 hdev->kernel_queues = kcalloc(asic->max_queues,
1075 sizeof(*hdev->kernel_queues), GFP_KERNEL);
1076
1077 if (!hdev->kernel_queues) {
1078 dev_err(hdev->dev, "Not enough memory for H/W queues\n");
1079 return -ENOMEM;
1080 }
1081
1082 /* Initialize the H/W queues */
1083 for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;
1084 i < asic->max_queues ; i++, q_ready_cnt++, q++) {
1085
1086 q->queue_type = asic->hw_queues_props[i].type;
1087 q->supports_sync_stream =
1088 asic->hw_queues_props[i].supports_sync_stream;
1089 q->collective_mode = asic->hw_queues_props[i].collective_mode;
1090 rc = queue_init(hdev, q, i);
1091 if (rc) {
1092 dev_err(hdev->dev,
1093 "failed to initialize queue %d\n", i);
1094 goto release_queues;
1095 }
1096 }
1097
1098 return 0;
1099
1100release_queues:
1101 for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)
1102 queue_fini(hdev, q);
1103
1104 kfree(hdev->kernel_queues);
1105
1106 return rc;
1107}
1108
1109void hl_hw_queues_destroy(struct hl_device *hdev)
1110{
1111 struct hl_hw_queue *q;
1112 u32 max_queues = hdev->asic_prop.max_queues;
1113 int i;
1114
1115 for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++)
1116 queue_fini(hdev, q);
1117
1118 kfree(hdev->kernel_queues);
1119}
1120
1121void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)
1122{
1123 struct hl_hw_queue *q;
1124 u32 max_queues = hdev->asic_prop.max_queues;
1125 int i;
1126
1127 for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) {
1128 if ((!q->valid) ||
1129 ((!hard_reset) && (q->queue_type == QUEUE_TYPE_CPU)))
1130 continue;
1131 q->pi = 0;
1132 atomic_set(&q->ci, 0);
1133
1134 if (q->supports_sync_stream)
1135 sync_stream_queue_reset(hdev, q->hw_queue_id);
1136 }
1137}