Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2
3/*
4 * Copyright 2016-2019 HabanaLabs, Ltd.
5 * All Rights Reserved.
6 */
7
8#define pr_fmt(fmt) "habanalabs: " fmt
9
10#include "habanalabs.h"
11
12#include <linux/pci.h>
13#include <linux/sched/signal.h>
14#include <linux/hwmon.h>
15#include <uapi/misc/habanalabs.h>
16
17#define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10)
18
19bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
20{
21 if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))
22 return true;
23 else
24 return false;
25}
26
27enum hl_device_status hl_device_status(struct hl_device *hdev)
28{
29 enum hl_device_status status;
30
31 if (hdev->disabled)
32 status = HL_DEVICE_STATUS_MALFUNCTION;
33 else if (atomic_read(&hdev->in_reset))
34 status = HL_DEVICE_STATUS_IN_RESET;
35 else
36 status = HL_DEVICE_STATUS_OPERATIONAL;
37
38 return status;
39};
40
41static void hpriv_release(struct kref *ref)
42{
43 struct hl_fpriv *hpriv;
44 struct hl_device *hdev;
45 struct hl_ctx *ctx;
46
47 hpriv = container_of(ref, struct hl_fpriv, refcount);
48
49 hdev = hpriv->hdev;
50 ctx = hpriv->ctx;
51
52 put_pid(hpriv->taskpid);
53
54 hl_debugfs_remove_file(hpriv);
55
56 mutex_destroy(&hpriv->restore_phase_mutex);
57
58 mutex_lock(&hdev->fpriv_list_lock);
59 list_del(&hpriv->dev_node);
60 hdev->compute_ctx = NULL;
61 mutex_unlock(&hdev->fpriv_list_lock);
62
63 kfree(hpriv);
64}
65
66void hl_hpriv_get(struct hl_fpriv *hpriv)
67{
68 kref_get(&hpriv->refcount);
69}
70
71void hl_hpriv_put(struct hl_fpriv *hpriv)
72{
73 kref_put(&hpriv->refcount, hpriv_release);
74}
75
76/*
77 * hl_device_release - release function for habanalabs device
78 *
79 * @inode: pointer to inode structure
80 * @filp: pointer to file structure
81 *
82 * Called when process closes an habanalabs device
83 */
84static int hl_device_release(struct inode *inode, struct file *filp)
85{
86 struct hl_fpriv *hpriv = filp->private_data;
87
88 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
89 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
90
91 filp->private_data = NULL;
92
93 hl_hpriv_put(hpriv);
94
95 return 0;
96}
97
98static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
99{
100 struct hl_fpriv *hpriv = filp->private_data;
101 struct hl_device *hdev;
102
103 filp->private_data = NULL;
104
105 hdev = hpriv->hdev;
106
107 mutex_lock(&hdev->fpriv_list_lock);
108 list_del(&hpriv->dev_node);
109 mutex_unlock(&hdev->fpriv_list_lock);
110
111 kfree(hpriv);
112
113 return 0;
114}
115
116/*
117 * hl_mmap - mmap function for habanalabs device
118 *
119 * @*filp: pointer to file structure
120 * @*vma: pointer to vm_area_struct of the process
121 *
122 * Called when process does an mmap on habanalabs device. Call the device's mmap
123 * function at the end of the common code.
124 */
125static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
126{
127 struct hl_fpriv *hpriv = filp->private_data;
128
129 if ((vma->vm_pgoff & HL_MMAP_CB_MASK) == HL_MMAP_CB_MASK) {
130 vma->vm_pgoff ^= HL_MMAP_CB_MASK;
131 return hl_cb_mmap(hpriv, vma);
132 }
133
134 return -EINVAL;
135}
136
137static const struct file_operations hl_ops = {
138 .owner = THIS_MODULE,
139 .open = hl_device_open,
140 .release = hl_device_release,
141 .mmap = hl_mmap,
142 .unlocked_ioctl = hl_ioctl,
143 .compat_ioctl = hl_ioctl
144};
145
146static const struct file_operations hl_ctrl_ops = {
147 .owner = THIS_MODULE,
148 .open = hl_device_open_ctrl,
149 .release = hl_device_release_ctrl,
150 .unlocked_ioctl = hl_ioctl_control,
151 .compat_ioctl = hl_ioctl_control
152};
153
154static void device_release_func(struct device *dev)
155{
156 kfree(dev);
157}
158
159/*
160 * device_init_cdev - Initialize cdev and device for habanalabs device
161 *
162 * @hdev: pointer to habanalabs device structure
163 * @hclass: pointer to the class object of the device
164 * @minor: minor number of the specific device
165 * @fpos: file operations to install for this device
166 * @name: name of the device as it will appear in the filesystem
167 * @cdev: pointer to the char device object that will be initialized
168 * @dev: pointer to the device object that will be initialized
169 *
170 * Initialize a cdev and a Linux device for habanalabs's device.
171 */
172static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
173 int minor, const struct file_operations *fops,
174 char *name, struct cdev *cdev,
175 struct device **dev)
176{
177 cdev_init(cdev, fops);
178 cdev->owner = THIS_MODULE;
179
180 *dev = kzalloc(sizeof(**dev), GFP_KERNEL);
181 if (!*dev)
182 return -ENOMEM;
183
184 device_initialize(*dev);
185 (*dev)->devt = MKDEV(hdev->major, minor);
186 (*dev)->class = hclass;
187 (*dev)->release = device_release_func;
188 dev_set_drvdata(*dev, hdev);
189 dev_set_name(*dev, "%s", name);
190
191 return 0;
192}
193
194static int device_cdev_sysfs_add(struct hl_device *hdev)
195{
196 int rc;
197
198 rc = cdev_device_add(&hdev->cdev, hdev->dev);
199 if (rc) {
200 dev_err(hdev->dev,
201 "failed to add a char device to the system\n");
202 return rc;
203 }
204
205 rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
206 if (rc) {
207 dev_err(hdev->dev,
208 "failed to add a control char device to the system\n");
209 goto delete_cdev_device;
210 }
211
212 /* hl_sysfs_init() must be done after adding the device to the system */
213 rc = hl_sysfs_init(hdev);
214 if (rc) {
215 dev_err(hdev->dev, "failed to initialize sysfs\n");
216 goto delete_ctrl_cdev_device;
217 }
218
219 hdev->cdev_sysfs_created = true;
220
221 return 0;
222
223delete_ctrl_cdev_device:
224 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
225delete_cdev_device:
226 cdev_device_del(&hdev->cdev, hdev->dev);
227 return rc;
228}
229
230static void device_cdev_sysfs_del(struct hl_device *hdev)
231{
232 /* device_release() won't be called so must free devices explicitly */
233 if (!hdev->cdev_sysfs_created) {
234 kfree(hdev->dev_ctrl);
235 kfree(hdev->dev);
236 return;
237 }
238
239 hl_sysfs_fini(hdev);
240 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
241 cdev_device_del(&hdev->cdev, hdev->dev);
242}
243
244/*
245 * device_early_init - do some early initialization for the habanalabs device
246 *
247 * @hdev: pointer to habanalabs device structure
248 *
249 * Install the relevant function pointers and call the early_init function,
250 * if such a function exists
251 */
252static int device_early_init(struct hl_device *hdev)
253{
254 int rc;
255
256 switch (hdev->asic_type) {
257 case ASIC_GOYA:
258 goya_set_asic_funcs(hdev);
259 strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
260 break;
261 default:
262 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
263 hdev->asic_type);
264 return -EINVAL;
265 }
266
267 rc = hdev->asic_funcs->early_init(hdev);
268 if (rc)
269 return rc;
270
271 rc = hl_asid_init(hdev);
272 if (rc)
273 goto early_fini;
274
275 hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0);
276 if (hdev->cq_wq == NULL) {
277 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
278 rc = -ENOMEM;
279 goto asid_fini;
280 }
281
282 hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
283 if (hdev->eq_wq == NULL) {
284 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
285 rc = -ENOMEM;
286 goto free_cq_wq;
287 }
288
289 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
290 GFP_KERNEL);
291 if (!hdev->hl_chip_info) {
292 rc = -ENOMEM;
293 goto free_eq_wq;
294 }
295
296 hdev->idle_busy_ts_arr = kmalloc_array(HL_IDLE_BUSY_TS_ARR_SIZE,
297 sizeof(struct hl_device_idle_busy_ts),
298 (GFP_KERNEL | __GFP_ZERO));
299 if (!hdev->idle_busy_ts_arr) {
300 rc = -ENOMEM;
301 goto free_chip_info;
302 }
303
304 hl_cb_mgr_init(&hdev->kernel_cb_mgr);
305
306 mutex_init(&hdev->send_cpu_message_lock);
307 mutex_init(&hdev->debug_lock);
308 mutex_init(&hdev->mmu_cache_lock);
309 INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
310 spin_lock_init(&hdev->hw_queues_mirror_lock);
311 INIT_LIST_HEAD(&hdev->fpriv_list);
312 mutex_init(&hdev->fpriv_list_lock);
313 atomic_set(&hdev->in_reset, 0);
314
315 return 0;
316
317free_chip_info:
318 kfree(hdev->hl_chip_info);
319free_eq_wq:
320 destroy_workqueue(hdev->eq_wq);
321free_cq_wq:
322 destroy_workqueue(hdev->cq_wq);
323asid_fini:
324 hl_asid_fini(hdev);
325early_fini:
326 if (hdev->asic_funcs->early_fini)
327 hdev->asic_funcs->early_fini(hdev);
328
329 return rc;
330}
331
332/*
333 * device_early_fini - finalize all that was done in device_early_init
334 *
335 * @hdev: pointer to habanalabs device structure
336 *
337 */
338static void device_early_fini(struct hl_device *hdev)
339{
340 mutex_destroy(&hdev->mmu_cache_lock);
341 mutex_destroy(&hdev->debug_lock);
342 mutex_destroy(&hdev->send_cpu_message_lock);
343
344 mutex_destroy(&hdev->fpriv_list_lock);
345
346 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
347
348 kfree(hdev->idle_busy_ts_arr);
349 kfree(hdev->hl_chip_info);
350
351 destroy_workqueue(hdev->eq_wq);
352 destroy_workqueue(hdev->cq_wq);
353
354 hl_asid_fini(hdev);
355
356 if (hdev->asic_funcs->early_fini)
357 hdev->asic_funcs->early_fini(hdev);
358}
359
360static void set_freq_to_low_job(struct work_struct *work)
361{
362 struct hl_device *hdev = container_of(work, struct hl_device,
363 work_freq.work);
364
365 mutex_lock(&hdev->fpriv_list_lock);
366
367 if (!hdev->compute_ctx)
368 hl_device_set_frequency(hdev, PLL_LOW);
369
370 mutex_unlock(&hdev->fpriv_list_lock);
371
372 schedule_delayed_work(&hdev->work_freq,
373 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
374}
375
376static void hl_device_heartbeat(struct work_struct *work)
377{
378 struct hl_device *hdev = container_of(work, struct hl_device,
379 work_heartbeat.work);
380
381 if (hl_device_disabled_or_in_reset(hdev))
382 goto reschedule;
383
384 if (!hdev->asic_funcs->send_heartbeat(hdev))
385 goto reschedule;
386
387 dev_err(hdev->dev, "Device heartbeat failed!\n");
388 hl_device_reset(hdev, true, false);
389
390 return;
391
392reschedule:
393 schedule_delayed_work(&hdev->work_heartbeat,
394 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
395}
396
397/*
398 * device_late_init - do late stuff initialization for the habanalabs device
399 *
400 * @hdev: pointer to habanalabs device structure
401 *
402 * Do stuff that either needs the device H/W queues to be active or needs
403 * to happen after all the rest of the initialization is finished
404 */
405static int device_late_init(struct hl_device *hdev)
406{
407 int rc;
408
409 if (hdev->asic_funcs->late_init) {
410 rc = hdev->asic_funcs->late_init(hdev);
411 if (rc) {
412 dev_err(hdev->dev,
413 "failed late initialization for the H/W\n");
414 return rc;
415 }
416 }
417
418 hdev->high_pll = hdev->asic_prop.high_pll;
419
420 /* force setting to low frequency */
421 hdev->curr_pll_profile = PLL_LOW;
422
423 if (hdev->pm_mng_profile == PM_AUTO)
424 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
425 else
426 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
427
428 INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
429 schedule_delayed_work(&hdev->work_freq,
430 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
431
432 if (hdev->heartbeat) {
433 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
434 schedule_delayed_work(&hdev->work_heartbeat,
435 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
436 }
437
438 hdev->late_init_done = true;
439
440 return 0;
441}
442
443/*
444 * device_late_fini - finalize all that was done in device_late_init
445 *
446 * @hdev: pointer to habanalabs device structure
447 *
448 */
449static void device_late_fini(struct hl_device *hdev)
450{
451 if (!hdev->late_init_done)
452 return;
453
454 cancel_delayed_work_sync(&hdev->work_freq);
455 if (hdev->heartbeat)
456 cancel_delayed_work_sync(&hdev->work_heartbeat);
457
458 if (hdev->asic_funcs->late_fini)
459 hdev->asic_funcs->late_fini(hdev);
460
461 hdev->late_init_done = false;
462}
463
464uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms)
465{
466 struct hl_device_idle_busy_ts *ts;
467 ktime_t zero_ktime, curr = ktime_get();
468 u32 overlap_cnt = 0, last_index = hdev->idle_busy_ts_idx;
469 s64 period_us, last_start_us, last_end_us, last_busy_time_us,
470 total_busy_time_us = 0, total_busy_time_ms;
471
472 zero_ktime = ktime_set(0, 0);
473 period_us = period_ms * USEC_PER_MSEC;
474 ts = &hdev->idle_busy_ts_arr[last_index];
475
476 /* check case that device is currently in idle */
477 if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime) &&
478 !ktime_compare(ts->idle_to_busy_ts, zero_ktime)) {
479
480 last_index--;
481 /* Handle case idle_busy_ts_idx was 0 */
482 if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
483 last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
484
485 ts = &hdev->idle_busy_ts_arr[last_index];
486 }
487
488 while (overlap_cnt < HL_IDLE_BUSY_TS_ARR_SIZE) {
489 /* Check if we are in last sample case. i.e. if the sample
490 * begun before the sampling period. This could be a real
491 * sample or 0 so need to handle both cases
492 */
493 last_start_us = ktime_to_us(
494 ktime_sub(curr, ts->idle_to_busy_ts));
495
496 if (last_start_us > period_us) {
497
498 /* First check two cases:
499 * 1. If the device is currently busy
500 * 2. If the device was idle during the whole sampling
501 * period
502 */
503
504 if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime)) {
505 /* Check if the device is currently busy */
506 if (ktime_compare(ts->idle_to_busy_ts,
507 zero_ktime))
508 return 100;
509
510 /* We either didn't have any activity or we
511 * reached an entry which is 0. Either way,
512 * exit and return what was accumulated so far
513 */
514 break;
515 }
516
517 /* If sample has finished, check it is relevant */
518 last_end_us = ktime_to_us(
519 ktime_sub(curr, ts->busy_to_idle_ts));
520
521 if (last_end_us > period_us)
522 break;
523
524 /* It is relevant so add it but with adjustment */
525 last_busy_time_us = ktime_to_us(
526 ktime_sub(ts->busy_to_idle_ts,
527 ts->idle_to_busy_ts));
528 total_busy_time_us += last_busy_time_us -
529 (last_start_us - period_us);
530 break;
531 }
532
533 /* Check if the sample is finished or still open */
534 if (ktime_compare(ts->busy_to_idle_ts, zero_ktime))
535 last_busy_time_us = ktime_to_us(
536 ktime_sub(ts->busy_to_idle_ts,
537 ts->idle_to_busy_ts));
538 else
539 last_busy_time_us = ktime_to_us(
540 ktime_sub(curr, ts->idle_to_busy_ts));
541
542 total_busy_time_us += last_busy_time_us;
543
544 last_index--;
545 /* Handle case idle_busy_ts_idx was 0 */
546 if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
547 last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
548
549 ts = &hdev->idle_busy_ts_arr[last_index];
550
551 overlap_cnt++;
552 }
553
554 total_busy_time_ms = DIV_ROUND_UP_ULL(total_busy_time_us,
555 USEC_PER_MSEC);
556
557 return DIV_ROUND_UP_ULL(total_busy_time_ms * 100, period_ms);
558}
559
560/*
561 * hl_device_set_frequency - set the frequency of the device
562 *
563 * @hdev: pointer to habanalabs device structure
564 * @freq: the new frequency value
565 *
566 * Change the frequency if needed. This function has no protection against
567 * concurrency, therefore it is assumed that the calling function has protected
568 * itself against the case of calling this function from multiple threads with
569 * different values
570 *
571 * Returns 0 if no change was done, otherwise returns 1
572 */
573int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
574{
575 if ((hdev->pm_mng_profile == PM_MANUAL) ||
576 (hdev->curr_pll_profile == freq))
577 return 0;
578
579 dev_dbg(hdev->dev, "Changing device frequency to %s\n",
580 freq == PLL_HIGH ? "high" : "low");
581
582 hdev->asic_funcs->set_pll_profile(hdev, freq);
583
584 hdev->curr_pll_profile = freq;
585
586 return 1;
587}
588
589int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
590{
591 int rc = 0;
592
593 mutex_lock(&hdev->debug_lock);
594
595 if (!enable) {
596 if (!hdev->in_debug) {
597 dev_err(hdev->dev,
598 "Failed to disable debug mode because device was not in debug mode\n");
599 rc = -EFAULT;
600 goto out;
601 }
602
603 hdev->asic_funcs->halt_coresight(hdev);
604 hdev->in_debug = 0;
605
606 goto out;
607 }
608
609 if (hdev->in_debug) {
610 dev_err(hdev->dev,
611 "Failed to enable debug mode because device is already in debug mode\n");
612 rc = -EFAULT;
613 goto out;
614 }
615
616 hdev->in_debug = 1;
617
618out:
619 mutex_unlock(&hdev->debug_lock);
620
621 return rc;
622}
623
624/*
625 * hl_device_suspend - initiate device suspend
626 *
627 * @hdev: pointer to habanalabs device structure
628 *
629 * Puts the hw in the suspend state (all asics).
630 * Returns 0 for success or an error on failure.
631 * Called at driver suspend.
632 */
633int hl_device_suspend(struct hl_device *hdev)
634{
635 int rc;
636
637 pci_save_state(hdev->pdev);
638
639 /* Block future CS/VM/JOB completion operations */
640 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
641 if (rc) {
642 dev_err(hdev->dev, "Can't suspend while in reset\n");
643 return -EIO;
644 }
645
646 /* This blocks all other stuff that is not blocked by in_reset */
647 hdev->disabled = true;
648
649 /*
650 * Flush anyone that is inside the critical section of enqueue
651 * jobs to the H/W
652 */
653 hdev->asic_funcs->hw_queues_lock(hdev);
654 hdev->asic_funcs->hw_queues_unlock(hdev);
655
656 /* Flush processes that are sending message to CPU */
657 mutex_lock(&hdev->send_cpu_message_lock);
658 mutex_unlock(&hdev->send_cpu_message_lock);
659
660 rc = hdev->asic_funcs->suspend(hdev);
661 if (rc)
662 dev_err(hdev->dev,
663 "Failed to disable PCI access of device CPU\n");
664
665 /* Shut down the device */
666 pci_disable_device(hdev->pdev);
667 pci_set_power_state(hdev->pdev, PCI_D3hot);
668
669 return 0;
670}
671
672/*
673 * hl_device_resume - initiate device resume
674 *
675 * @hdev: pointer to habanalabs device structure
676 *
677 * Bring the hw back to operating state (all asics).
678 * Returns 0 for success or an error on failure.
679 * Called at driver resume.
680 */
681int hl_device_resume(struct hl_device *hdev)
682{
683 int rc;
684
685 pci_set_power_state(hdev->pdev, PCI_D0);
686 pci_restore_state(hdev->pdev);
687 rc = pci_enable_device_mem(hdev->pdev);
688 if (rc) {
689 dev_err(hdev->dev,
690 "Failed to enable PCI device in resume\n");
691 return rc;
692 }
693
694 pci_set_master(hdev->pdev);
695
696 rc = hdev->asic_funcs->resume(hdev);
697 if (rc) {
698 dev_err(hdev->dev, "Failed to resume device after suspend\n");
699 goto disable_device;
700 }
701
702
703 hdev->disabled = false;
704 atomic_set(&hdev->in_reset, 0);
705
706 rc = hl_device_reset(hdev, true, false);
707 if (rc) {
708 dev_err(hdev->dev, "Failed to reset device during resume\n");
709 goto disable_device;
710 }
711
712 return 0;
713
714disable_device:
715 pci_clear_master(hdev->pdev);
716 pci_disable_device(hdev->pdev);
717
718 return rc;
719}
720
721static void device_kill_open_processes(struct hl_device *hdev)
722{
723 u16 pending_total, pending_cnt;
724 struct hl_fpriv *hpriv;
725 struct task_struct *task = NULL;
726
727 if (hdev->pldm)
728 pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
729 else
730 pending_total = HL_PENDING_RESET_PER_SEC;
731
732 /* Giving time for user to close FD, and for processes that are inside
733 * hl_device_open to finish
734 */
735 if (!list_empty(&hdev->fpriv_list))
736 ssleep(1);
737
738 mutex_lock(&hdev->fpriv_list_lock);
739
740 /* This section must be protected because we are dereferencing
741 * pointers that are freed if the process exits
742 */
743 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
744 task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
745 if (task) {
746 dev_info(hdev->dev, "Killing user process pid=%d\n",
747 task_pid_nr(task));
748 send_sig(SIGKILL, task, 1);
749 usleep_range(1000, 10000);
750
751 put_task_struct(task);
752 }
753 }
754
755 mutex_unlock(&hdev->fpriv_list_lock);
756
757 /* We killed the open users, but because the driver cleans up after the
758 * user contexts are closed (e.g. mmu mappings), we need to wait again
759 * to make sure the cleaning phase is finished before continuing with
760 * the reset
761 */
762
763 pending_cnt = pending_total;
764
765 while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {
766 dev_info(hdev->dev,
767 "Waiting for all unmap operations to finish before hard reset\n");
768
769 pending_cnt--;
770
771 ssleep(1);
772 }
773
774 if (!list_empty(&hdev->fpriv_list))
775 dev_crit(hdev->dev,
776 "Going to hard reset with open user contexts\n");
777}
778
779static void device_hard_reset_pending(struct work_struct *work)
780{
781 struct hl_device_reset_work *device_reset_work =
782 container_of(work, struct hl_device_reset_work, reset_work);
783 struct hl_device *hdev = device_reset_work->hdev;
784
785 hl_device_reset(hdev, true, true);
786
787 kfree(device_reset_work);
788}
789
790/*
791 * hl_device_reset - reset the device
792 *
793 * @hdev: pointer to habanalabs device structure
794 * @hard_reset: should we do hard reset to all engines or just reset the
795 * compute/dma engines
796 *
797 * Block future CS and wait for pending CS to be enqueued
798 * Call ASIC H/W fini
799 * Flush all completions
800 * Re-initialize all internal data structures
801 * Call ASIC H/W init, late_init
802 * Test queues
803 * Enable device
804 *
805 * Returns 0 for success or an error on failure.
806 */
807int hl_device_reset(struct hl_device *hdev, bool hard_reset,
808 bool from_hard_reset_thread)
809{
810 int i, rc;
811
812 if (!hdev->init_done) {
813 dev_err(hdev->dev,
814 "Can't reset before initialization is done\n");
815 return 0;
816 }
817
818 /*
819 * Prevent concurrency in this function - only one reset should be
820 * done at any given time. Only need to perform this if we didn't
821 * get from the dedicated hard reset thread
822 */
823 if (!from_hard_reset_thread) {
824 /* Block future CS/VM/JOB completion operations */
825 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
826 if (rc)
827 return 0;
828
829 /* This also blocks future CS/VM/JOB completion operations */
830 hdev->disabled = true;
831
832 /* Flush anyone that is inside the critical section of enqueue
833 * jobs to the H/W
834 */
835 hdev->asic_funcs->hw_queues_lock(hdev);
836 hdev->asic_funcs->hw_queues_unlock(hdev);
837
838 /* Flush anyone that is inside device open */
839 mutex_lock(&hdev->fpriv_list_lock);
840 mutex_unlock(&hdev->fpriv_list_lock);
841
842 dev_err(hdev->dev, "Going to RESET device!\n");
843 }
844
845again:
846 if ((hard_reset) && (!from_hard_reset_thread)) {
847 struct hl_device_reset_work *device_reset_work;
848
849 hdev->hard_reset_pending = true;
850
851 device_reset_work = kzalloc(sizeof(*device_reset_work),
852 GFP_ATOMIC);
853 if (!device_reset_work) {
854 rc = -ENOMEM;
855 goto out_err;
856 }
857
858 /*
859 * Because the reset function can't run from interrupt or
860 * from heartbeat work, we need to call the reset function
861 * from a dedicated work
862 */
863 INIT_WORK(&device_reset_work->reset_work,
864 device_hard_reset_pending);
865 device_reset_work->hdev = hdev;
866 schedule_work(&device_reset_work->reset_work);
867
868 return 0;
869 }
870
871 if (hard_reset) {
872 device_late_fini(hdev);
873
874 /*
875 * Now that the heartbeat thread is closed, flush processes
876 * which are sending messages to CPU
877 */
878 mutex_lock(&hdev->send_cpu_message_lock);
879 mutex_unlock(&hdev->send_cpu_message_lock);
880 }
881
882 /*
883 * Halt the engines and disable interrupts so we won't get any more
884 * completions from H/W and we won't have any accesses from the
885 * H/W to the host machine
886 */
887 hdev->asic_funcs->halt_engines(hdev, hard_reset);
888
889 /* Go over all the queues, release all CS and their jobs */
890 hl_cs_rollback_all(hdev);
891
892 /* Kill processes here after CS rollback. This is because the process
893 * can't really exit until all its CSs are done, which is what we
894 * do in cs rollback
895 */
896 if (from_hard_reset_thread)
897 device_kill_open_processes(hdev);
898
899 /* Release kernel context */
900 if ((hard_reset) && (hl_ctx_put(hdev->kernel_ctx) == 1))
901 hdev->kernel_ctx = NULL;
902
903 /* Reset the H/W. It will be in idle state after this returns */
904 hdev->asic_funcs->hw_fini(hdev, hard_reset);
905
906 if (hard_reset) {
907 hl_vm_fini(hdev);
908 hl_mmu_fini(hdev);
909 hl_eq_reset(hdev, &hdev->event_queue);
910 }
911
912 /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
913 hl_hw_queue_reset(hdev, hard_reset);
914 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
915 hl_cq_reset(hdev, &hdev->completion_queue[i]);
916
917 hdev->idle_busy_ts_idx = 0;
918 hdev->idle_busy_ts_arr[0].busy_to_idle_ts = ktime_set(0, 0);
919 hdev->idle_busy_ts_arr[0].idle_to_busy_ts = ktime_set(0, 0);
920
921 if (hdev->cs_active_cnt)
922 dev_crit(hdev->dev, "CS active cnt %d is not 0 during reset\n",
923 hdev->cs_active_cnt);
924
925 mutex_lock(&hdev->fpriv_list_lock);
926
927 /* Make sure the context switch phase will run again */
928 if (hdev->compute_ctx) {
929 atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);
930 hdev->compute_ctx->thread_ctx_switch_wait_token = 0;
931 }
932
933 mutex_unlock(&hdev->fpriv_list_lock);
934
935 /* Finished tear-down, starting to re-initialize */
936
937 if (hard_reset) {
938 hdev->device_cpu_disabled = false;
939 hdev->hard_reset_pending = false;
940
941 if (hdev->kernel_ctx) {
942 dev_crit(hdev->dev,
943 "kernel ctx was alive during hard reset, something is terribly wrong\n");
944 rc = -EBUSY;
945 goto out_err;
946 }
947
948 rc = hl_mmu_init(hdev);
949 if (rc) {
950 dev_err(hdev->dev,
951 "Failed to initialize MMU S/W after hard reset\n");
952 goto out_err;
953 }
954
955 /* Allocate the kernel context */
956 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
957 GFP_KERNEL);
958 if (!hdev->kernel_ctx) {
959 rc = -ENOMEM;
960 goto out_err;
961 }
962
963 hdev->compute_ctx = NULL;
964
965 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
966 if (rc) {
967 dev_err(hdev->dev,
968 "failed to init kernel ctx in hard reset\n");
969 kfree(hdev->kernel_ctx);
970 hdev->kernel_ctx = NULL;
971 goto out_err;
972 }
973 }
974
975 rc = hdev->asic_funcs->hw_init(hdev);
976 if (rc) {
977 dev_err(hdev->dev,
978 "failed to initialize the H/W after reset\n");
979 goto out_err;
980 }
981
982 hdev->disabled = false;
983
984 /* Check that the communication with the device is working */
985 rc = hdev->asic_funcs->test_queues(hdev);
986 if (rc) {
987 dev_err(hdev->dev,
988 "Failed to detect if device is alive after reset\n");
989 goto out_err;
990 }
991
992 if (hard_reset) {
993 rc = device_late_init(hdev);
994 if (rc) {
995 dev_err(hdev->dev,
996 "Failed late init after hard reset\n");
997 goto out_err;
998 }
999
1000 rc = hl_vm_init(hdev);
1001 if (rc) {
1002 dev_err(hdev->dev,
1003 "Failed to init memory module after hard reset\n");
1004 goto out_err;
1005 }
1006
1007 hl_set_max_power(hdev, hdev->max_power);
1008 } else {
1009 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
1010 if (rc) {
1011 dev_err(hdev->dev,
1012 "Failed late init after soft reset\n");
1013 goto out_err;
1014 }
1015 }
1016
1017 atomic_set(&hdev->in_reset, 0);
1018
1019 if (hard_reset)
1020 hdev->hard_reset_cnt++;
1021 else
1022 hdev->soft_reset_cnt++;
1023
1024 dev_warn(hdev->dev, "Successfully finished resetting the device\n");
1025
1026 return 0;
1027
1028out_err:
1029 hdev->disabled = true;
1030
1031 if (hard_reset) {
1032 dev_err(hdev->dev,
1033 "Failed to reset! Device is NOT usable\n");
1034 hdev->hard_reset_cnt++;
1035 } else {
1036 dev_err(hdev->dev,
1037 "Failed to do soft-reset, trying hard reset\n");
1038 hdev->soft_reset_cnt++;
1039 hard_reset = true;
1040 goto again;
1041 }
1042
1043 atomic_set(&hdev->in_reset, 0);
1044
1045 return rc;
1046}
1047
1048/*
1049 * hl_device_init - main initialization function for habanalabs device
1050 *
1051 * @hdev: pointer to habanalabs device structure
1052 *
1053 * Allocate an id for the device, do early initialization and then call the
1054 * ASIC specific initialization functions. Finally, create the cdev and the
1055 * Linux device to expose it to the user
1056 */
1057int hl_device_init(struct hl_device *hdev, struct class *hclass)
1058{
1059 int i, rc, cq_ready_cnt;
1060 char *name;
1061 bool add_cdev_sysfs_on_err = false;
1062
1063 name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);
1064 if (!name) {
1065 rc = -ENOMEM;
1066 goto out_disabled;
1067 }
1068
1069 /* Initialize cdev and device structures */
1070 rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,
1071 &hdev->cdev, &hdev->dev);
1072
1073 kfree(name);
1074
1075 if (rc)
1076 goto out_disabled;
1077
1078 name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);
1079 if (!name) {
1080 rc = -ENOMEM;
1081 goto free_dev;
1082 }
1083
1084 /* Initialize cdev and device structures for control device */
1085 rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
1086 name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
1087
1088 kfree(name);
1089
1090 if (rc)
1091 goto free_dev;
1092
1093 /* Initialize ASIC function pointers and perform early init */
1094 rc = device_early_init(hdev);
1095 if (rc)
1096 goto free_dev_ctrl;
1097
1098 /*
1099 * Start calling ASIC initialization. First S/W then H/W and finally
1100 * late init
1101 */
1102 rc = hdev->asic_funcs->sw_init(hdev);
1103 if (rc)
1104 goto early_fini;
1105
1106 /*
1107 * Initialize the H/W queues. Must be done before hw_init, because
1108 * there the addresses of the kernel queue are being written to the
1109 * registers of the device
1110 */
1111 rc = hl_hw_queues_create(hdev);
1112 if (rc) {
1113 dev_err(hdev->dev, "failed to initialize kernel queues\n");
1114 goto sw_fini;
1115 }
1116
1117 /*
1118 * Initialize the completion queues. Must be done before hw_init,
1119 * because there the addresses of the completion queues are being
1120 * passed as arguments to request_irq
1121 */
1122 hdev->completion_queue =
1123 kcalloc(hdev->asic_prop.completion_queues_count,
1124 sizeof(*hdev->completion_queue), GFP_KERNEL);
1125
1126 if (!hdev->completion_queue) {
1127 dev_err(hdev->dev, "failed to allocate completion queues\n");
1128 rc = -ENOMEM;
1129 goto hw_queues_destroy;
1130 }
1131
1132 for (i = 0, cq_ready_cnt = 0;
1133 i < hdev->asic_prop.completion_queues_count;
1134 i++, cq_ready_cnt++) {
1135 rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
1136 if (rc) {
1137 dev_err(hdev->dev,
1138 "failed to initialize completion queue\n");
1139 goto cq_fini;
1140 }
1141 }
1142
1143 /*
1144 * Initialize the event queue. Must be done before hw_init,
1145 * because there the address of the event queue is being
1146 * passed as argument to request_irq
1147 */
1148 rc = hl_eq_init(hdev, &hdev->event_queue);
1149 if (rc) {
1150 dev_err(hdev->dev, "failed to initialize event queue\n");
1151 goto cq_fini;
1152 }
1153
1154 /* MMU S/W must be initialized before kernel context is created */
1155 rc = hl_mmu_init(hdev);
1156 if (rc) {
1157 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
1158 goto eq_fini;
1159 }
1160
1161 /* Allocate the kernel context */
1162 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
1163 if (!hdev->kernel_ctx) {
1164 rc = -ENOMEM;
1165 goto mmu_fini;
1166 }
1167
1168 hdev->compute_ctx = NULL;
1169
1170 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1171 if (rc) {
1172 dev_err(hdev->dev, "failed to initialize kernel context\n");
1173 kfree(hdev->kernel_ctx);
1174 goto mmu_fini;
1175 }
1176
1177 rc = hl_cb_pool_init(hdev);
1178 if (rc) {
1179 dev_err(hdev->dev, "failed to initialize CB pool\n");
1180 goto release_ctx;
1181 }
1182
1183 hl_debugfs_add_device(hdev);
1184
1185 if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
1186 dev_info(hdev->dev,
1187 "H/W state is dirty, must reset before initializing\n");
1188 hdev->asic_funcs->hw_fini(hdev, true);
1189 }
1190
1191 /*
1192 * From this point, in case of an error, add char devices and create
1193 * sysfs nodes as part of the error flow, to allow debugging.
1194 */
1195 add_cdev_sysfs_on_err = true;
1196
1197 rc = hdev->asic_funcs->hw_init(hdev);
1198 if (rc) {
1199 dev_err(hdev->dev, "failed to initialize the H/W\n");
1200 rc = 0;
1201 goto out_disabled;
1202 }
1203
1204 hdev->disabled = false;
1205
1206 /* Check that the communication with the device is working */
1207 rc = hdev->asic_funcs->test_queues(hdev);
1208 if (rc) {
1209 dev_err(hdev->dev, "Failed to detect if device is alive\n");
1210 rc = 0;
1211 goto out_disabled;
1212 }
1213
1214 rc = device_late_init(hdev);
1215 if (rc) {
1216 dev_err(hdev->dev, "Failed late initialization\n");
1217 rc = 0;
1218 goto out_disabled;
1219 }
1220
1221 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
1222 hdev->asic_name,
1223 hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
1224
1225 rc = hl_vm_init(hdev);
1226 if (rc) {
1227 dev_err(hdev->dev, "Failed to initialize memory module\n");
1228 rc = 0;
1229 goto out_disabled;
1230 }
1231
1232 /*
1233 * Expose devices and sysfs nodes to user.
1234 * From here there is no need to add char devices and create sysfs nodes
1235 * in case of an error.
1236 */
1237 add_cdev_sysfs_on_err = false;
1238 rc = device_cdev_sysfs_add(hdev);
1239 if (rc) {
1240 dev_err(hdev->dev,
1241 "Failed to add char devices and sysfs nodes\n");
1242 rc = 0;
1243 goto out_disabled;
1244 }
1245
1246 /*
1247 * hl_hwmon_init() must be called after device_late_init(), because only
1248 * there we get the information from the device about which
1249 * hwmon-related sensors the device supports.
1250 * Furthermore, it must be done after adding the device to the system.
1251 */
1252 rc = hl_hwmon_init(hdev);
1253 if (rc) {
1254 dev_err(hdev->dev, "Failed to initialize hwmon\n");
1255 rc = 0;
1256 goto out_disabled;
1257 }
1258
1259 dev_notice(hdev->dev,
1260 "Successfully added device to habanalabs driver\n");
1261
1262 hdev->init_done = true;
1263
1264 return 0;
1265
1266release_ctx:
1267 if (hl_ctx_put(hdev->kernel_ctx) != 1)
1268 dev_err(hdev->dev,
1269 "kernel ctx is still alive on initialization failure\n");
1270mmu_fini:
1271 hl_mmu_fini(hdev);
1272eq_fini:
1273 hl_eq_fini(hdev, &hdev->event_queue);
1274cq_fini:
1275 for (i = 0 ; i < cq_ready_cnt ; i++)
1276 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1277 kfree(hdev->completion_queue);
1278hw_queues_destroy:
1279 hl_hw_queues_destroy(hdev);
1280sw_fini:
1281 hdev->asic_funcs->sw_fini(hdev);
1282early_fini:
1283 device_early_fini(hdev);
1284free_dev_ctrl:
1285 kfree(hdev->dev_ctrl);
1286free_dev:
1287 kfree(hdev->dev);
1288out_disabled:
1289 hdev->disabled = true;
1290 if (add_cdev_sysfs_on_err)
1291 device_cdev_sysfs_add(hdev);
1292 if (hdev->pdev)
1293 dev_err(&hdev->pdev->dev,
1294 "Failed to initialize hl%d. Device is NOT usable !\n",
1295 hdev->id / 2);
1296 else
1297 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
1298 hdev->id / 2);
1299
1300 return rc;
1301}
1302
1303/*
1304 * hl_device_fini - main tear-down function for habanalabs device
1305 *
1306 * @hdev: pointer to habanalabs device structure
1307 *
1308 * Destroy the device, call ASIC fini functions and release the id
1309 */
1310void hl_device_fini(struct hl_device *hdev)
1311{
1312 int i, rc;
1313 ktime_t timeout;
1314
1315 dev_info(hdev->dev, "Removing device\n");
1316
1317 /*
1318 * This function is competing with the reset function, so try to
1319 * take the reset atomic and if we are already in middle of reset,
1320 * wait until reset function is finished. Reset function is designed
1321 * to always finish (could take up to a few seconds in worst case).
1322 */
1323
1324 timeout = ktime_add_us(ktime_get(),
1325 HL_PENDING_RESET_PER_SEC * 1000 * 1000 * 4);
1326 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1327 while (rc) {
1328 usleep_range(50, 200);
1329 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1330 if (ktime_compare(ktime_get(), timeout) > 0) {
1331 WARN(1, "Failed to remove device because reset function did not finish\n");
1332 return;
1333 }
1334 }
1335
1336 /* Mark device as disabled */
1337 hdev->disabled = true;
1338
1339 /* Flush anyone that is inside the critical section of enqueue
1340 * jobs to the H/W
1341 */
1342 hdev->asic_funcs->hw_queues_lock(hdev);
1343 hdev->asic_funcs->hw_queues_unlock(hdev);
1344
1345 /* Flush anyone that is inside device open */
1346 mutex_lock(&hdev->fpriv_list_lock);
1347 mutex_unlock(&hdev->fpriv_list_lock);
1348
1349 hdev->hard_reset_pending = true;
1350
1351 hl_hwmon_fini(hdev);
1352
1353 device_late_fini(hdev);
1354
1355 hl_debugfs_remove_device(hdev);
1356
1357 /*
1358 * Halt the engines and disable interrupts so we won't get any more
1359 * completions from H/W and we won't have any accesses from the
1360 * H/W to the host machine
1361 */
1362 hdev->asic_funcs->halt_engines(hdev, true);
1363
1364 /* Go over all the queues, release all CS and their jobs */
1365 hl_cs_rollback_all(hdev);
1366
1367 /* Kill processes here after CS rollback. This is because the process
1368 * can't really exit until all its CSs are done, which is what we
1369 * do in cs rollback
1370 */
1371 device_kill_open_processes(hdev);
1372
1373 hl_cb_pool_fini(hdev);
1374
1375 /* Release kernel context */
1376 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1377 dev_err(hdev->dev, "kernel ctx is still alive\n");
1378
1379 /* Reset the H/W. It will be in idle state after this returns */
1380 hdev->asic_funcs->hw_fini(hdev, true);
1381
1382 hl_vm_fini(hdev);
1383
1384 hl_mmu_fini(hdev);
1385
1386 hl_eq_fini(hdev, &hdev->event_queue);
1387
1388 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1389 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1390 kfree(hdev->completion_queue);
1391
1392 hl_hw_queues_destroy(hdev);
1393
1394 /* Call ASIC S/W finalize function */
1395 hdev->asic_funcs->sw_fini(hdev);
1396
1397 device_early_fini(hdev);
1398
1399 /* Hide devices and sysfs nodes from user */
1400 device_cdev_sysfs_del(hdev);
1401
1402 pr_info("removed device successfully\n");
1403}
1404
1405/*
1406 * MMIO register access helper functions.
1407 */
1408
1409/*
1410 * hl_rreg - Read an MMIO register
1411 *
1412 * @hdev: pointer to habanalabs device structure
1413 * @reg: MMIO register offset (in bytes)
1414 *
1415 * Returns the value of the MMIO register we are asked to read
1416 *
1417 */
1418inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1419{
1420 return readl(hdev->rmmio + reg);
1421}
1422
1423/*
1424 * hl_wreg - Write to an MMIO register
1425 *
1426 * @hdev: pointer to habanalabs device structure
1427 * @reg: MMIO register offset (in bytes)
1428 * @val: 32-bit value
1429 *
1430 * Writes the 32-bit value into the MMIO register
1431 *
1432 */
1433inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1434{
1435 writel(val, hdev->rmmio + reg);
1436}