Linux Audio

Check our new training course

Loading...
Note: File does not exist in v5.4.
  1// SPDX-License-Identifier: GPL-2.0 or MIT
  2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
  3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
  4/* Copyright 2023 Collabora ltd. */
  5
  6#include <linux/clk.h>
  7#include <linux/mm.h>
  8#include <linux/platform_device.h>
  9#include <linux/pm_domain.h>
 10#include <linux/pm_runtime.h>
 11#include <linux/regulator/consumer.h>
 12#include <linux/reset.h>
 13
 14#include <drm/drm_drv.h>
 15#include <drm/drm_managed.h>
 16
 17#include "panthor_devfreq.h"
 18#include "panthor_device.h"
 19#include "panthor_fw.h"
 20#include "panthor_gpu.h"
 21#include "panthor_mmu.h"
 22#include "panthor_regs.h"
 23#include "panthor_sched.h"
 24
 25static int panthor_clk_init(struct panthor_device *ptdev)
 26{
 27	ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL);
 28	if (IS_ERR(ptdev->clks.core))
 29		return dev_err_probe(ptdev->base.dev,
 30				     PTR_ERR(ptdev->clks.core),
 31				     "get 'core' clock failed");
 32
 33	ptdev->clks.stacks = devm_clk_get_optional(ptdev->base.dev, "stacks");
 34	if (IS_ERR(ptdev->clks.stacks))
 35		return dev_err_probe(ptdev->base.dev,
 36				     PTR_ERR(ptdev->clks.stacks),
 37				     "get 'stacks' clock failed");
 38
 39	ptdev->clks.coregroup = devm_clk_get_optional(ptdev->base.dev, "coregroup");
 40	if (IS_ERR(ptdev->clks.coregroup))
 41		return dev_err_probe(ptdev->base.dev,
 42				     PTR_ERR(ptdev->clks.coregroup),
 43				     "get 'coregroup' clock failed");
 44
 45	drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core));
 46	return 0;
 47}
 48
 49void panthor_device_unplug(struct panthor_device *ptdev)
 50{
 51	/* This function can be called from two different path: the reset work
 52	 * and the platform device remove callback. drm_dev_unplug() doesn't
 53	 * deal with concurrent callers, so we have to protect drm_dev_unplug()
 54	 * calls with our own lock, and bail out if the device is already
 55	 * unplugged.
 56	 */
 57	mutex_lock(&ptdev->unplug.lock);
 58	if (drm_dev_is_unplugged(&ptdev->base)) {
 59		/* Someone beat us, release the lock and wait for the unplug
 60		 * operation to be reported as done.
 61		 **/
 62		mutex_unlock(&ptdev->unplug.lock);
 63		wait_for_completion(&ptdev->unplug.done);
 64		return;
 65	}
 66
 67	/* Call drm_dev_unplug() so any access to HW blocks happening after
 68	 * that point get rejected.
 69	 */
 70	drm_dev_unplug(&ptdev->base);
 71
 72	/* We do the rest of the unplug with the unplug lock released,
 73	 * future callers will wait on ptdev->unplug.done anyway.
 74	 */
 75	mutex_unlock(&ptdev->unplug.lock);
 76
 77	drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0);
 78
 79	/* Now, try to cleanly shutdown the GPU before the device resources
 80	 * get reclaimed.
 81	 */
 82	panthor_sched_unplug(ptdev);
 83	panthor_fw_unplug(ptdev);
 84	panthor_mmu_unplug(ptdev);
 85	panthor_gpu_unplug(ptdev);
 86
 87	pm_runtime_dont_use_autosuspend(ptdev->base.dev);
 88	pm_runtime_put_sync_suspend(ptdev->base.dev);
 89
 90	/* If PM is disabled, we need to call the suspend handler manually. */
 91	if (!IS_ENABLED(CONFIG_PM))
 92		panthor_device_suspend(ptdev->base.dev);
 93
 94	/* Report the unplug operation as done to unblock concurrent
 95	 * panthor_device_unplug() callers.
 96	 */
 97	complete_all(&ptdev->unplug.done);
 98}
 99
100static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data)
101{
102	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
103
104	cancel_work_sync(&ptdev->reset.work);
105	destroy_workqueue(ptdev->reset.wq);
106}
107
108static void panthor_device_reset_work(struct work_struct *work)
109{
110	struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work);
111	int ret = 0, cookie;
112
113	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE) {
114		/*
115		 * No need for a reset as the device has been (or will be)
116		 * powered down
117		 */
118		atomic_set(&ptdev->reset.pending, 0);
119		return;
120	}
121
122	if (!drm_dev_enter(&ptdev->base, &cookie))
123		return;
124
125	panthor_sched_pre_reset(ptdev);
126	panthor_fw_pre_reset(ptdev, true);
127	panthor_mmu_pre_reset(ptdev);
128	panthor_gpu_soft_reset(ptdev);
129	panthor_gpu_l2_power_on(ptdev);
130	panthor_mmu_post_reset(ptdev);
131	ret = panthor_fw_post_reset(ptdev);
132	atomic_set(&ptdev->reset.pending, 0);
133	panthor_sched_post_reset(ptdev, ret != 0);
134	drm_dev_exit(cookie);
135
136	if (ret) {
137		panthor_device_unplug(ptdev);
138		drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable.");
139	}
140}
141
142static bool panthor_device_is_initialized(struct panthor_device *ptdev)
143{
144	return !!ptdev->scheduler;
145}
146
147static void panthor_device_free_page(struct drm_device *ddev, void *data)
148{
149	__free_page(data);
150}
151
152int panthor_device_init(struct panthor_device *ptdev)
153{
154	u32 *dummy_page_virt;
155	struct resource *res;
156	struct page *p;
157	int ret;
158
159	ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;
160
161	init_completion(&ptdev->unplug.done);
162	ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock);
163	if (ret)
164		return ret;
165
166	ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock);
167	if (ret)
168		return ret;
169
170	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
171	p = alloc_page(GFP_KERNEL | __GFP_ZERO);
172	if (!p)
173		return -ENOMEM;
174
175	ptdev->pm.dummy_latest_flush = p;
176	dummy_page_virt = page_address(p);
177	ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page,
178				       ptdev->pm.dummy_latest_flush);
179	if (ret)
180		return ret;
181
182	/*
183	 * Set the dummy page holding the latest flush to 1. This will cause the
184	 * flush to avoided as we know it isn't necessary if the submission
185	 * happens while the dummy page is mapped. Zero cannot be used because
186	 * that means 'always flush'.
187	 */
188	*dummy_page_virt = 1;
189
190	INIT_WORK(&ptdev->reset.work, panthor_device_reset_work);
191	ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0);
192	if (!ptdev->reset.wq)
193		return -ENOMEM;
194
195	ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL);
196	if (ret)
197		return ret;
198
199	ret = panthor_clk_init(ptdev);
200	if (ret)
201		return ret;
202
203	ret = panthor_devfreq_init(ptdev);
204	if (ret)
205		return ret;
206
207	ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev),
208							      0, &res);
209	if (IS_ERR(ptdev->iomem))
210		return PTR_ERR(ptdev->iomem);
211
212	ptdev->phys_addr = res->start;
213
214	ret = devm_pm_runtime_enable(ptdev->base.dev);
215	if (ret)
216		return ret;
217
218	ret = pm_runtime_resume_and_get(ptdev->base.dev);
219	if (ret)
220		return ret;
221
222	/* If PM is disabled, we need to call panthor_device_resume() manually. */
223	if (!IS_ENABLED(CONFIG_PM)) {
224		ret = panthor_device_resume(ptdev->base.dev);
225		if (ret)
226			return ret;
227	}
228
229	ret = panthor_gpu_init(ptdev);
230	if (ret)
231		goto err_rpm_put;
232
233	ret = panthor_mmu_init(ptdev);
234	if (ret)
235		goto err_unplug_gpu;
236
237	ret = panthor_fw_init(ptdev);
238	if (ret)
239		goto err_unplug_mmu;
240
241	ret = panthor_sched_init(ptdev);
242	if (ret)
243		goto err_unplug_fw;
244
245	/* ~3 frames */
246	pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50);
247	pm_runtime_use_autosuspend(ptdev->base.dev);
248
249	ret = drm_dev_register(&ptdev->base, 0);
250	if (ret)
251		goto err_disable_autosuspend;
252
253	pm_runtime_put_autosuspend(ptdev->base.dev);
254	return 0;
255
256err_disable_autosuspend:
257	pm_runtime_dont_use_autosuspend(ptdev->base.dev);
258	panthor_sched_unplug(ptdev);
259
260err_unplug_fw:
261	panthor_fw_unplug(ptdev);
262
263err_unplug_mmu:
264	panthor_mmu_unplug(ptdev);
265
266err_unplug_gpu:
267	panthor_gpu_unplug(ptdev);
268
269err_rpm_put:
270	pm_runtime_put_sync_suspend(ptdev->base.dev);
271	return ret;
272}
273
274#define PANTHOR_EXCEPTION(id) \
275	[DRM_PANTHOR_EXCEPTION_ ## id] = { \
276		.name = #id, \
277	}
278
279struct panthor_exception_info {
280	const char *name;
281};
282
283static const struct panthor_exception_info panthor_exception_infos[] = {
284	PANTHOR_EXCEPTION(OK),
285	PANTHOR_EXCEPTION(TERMINATED),
286	PANTHOR_EXCEPTION(KABOOM),
287	PANTHOR_EXCEPTION(EUREKA),
288	PANTHOR_EXCEPTION(ACTIVE),
289	PANTHOR_EXCEPTION(CS_RES_TERM),
290	PANTHOR_EXCEPTION(CS_CONFIG_FAULT),
291	PANTHOR_EXCEPTION(CS_UNRECOVERABLE),
292	PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT),
293	PANTHOR_EXCEPTION(CS_BUS_FAULT),
294	PANTHOR_EXCEPTION(CS_INSTR_INVALID),
295	PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW),
296	PANTHOR_EXCEPTION(CS_INHERIT_FAULT),
297	PANTHOR_EXCEPTION(INSTR_INVALID_PC),
298	PANTHOR_EXCEPTION(INSTR_INVALID_ENC),
299	PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT),
300	PANTHOR_EXCEPTION(DATA_INVALID_FAULT),
301	PANTHOR_EXCEPTION(TILE_RANGE_FAULT),
302	PANTHOR_EXCEPTION(ADDR_RANGE_FAULT),
303	PANTHOR_EXCEPTION(IMPRECISE_FAULT),
304	PANTHOR_EXCEPTION(OOM),
305	PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR),
306	PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT),
307	PANTHOR_EXCEPTION(GPU_BUS_FAULT),
308	PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT),
309	PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT),
310	PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT),
311	PANTHOR_EXCEPTION(TRANSLATION_FAULT_0),
312	PANTHOR_EXCEPTION(TRANSLATION_FAULT_1),
313	PANTHOR_EXCEPTION(TRANSLATION_FAULT_2),
314	PANTHOR_EXCEPTION(TRANSLATION_FAULT_3),
315	PANTHOR_EXCEPTION(TRANSLATION_FAULT_4),
316	PANTHOR_EXCEPTION(PERM_FAULT_0),
317	PANTHOR_EXCEPTION(PERM_FAULT_1),
318	PANTHOR_EXCEPTION(PERM_FAULT_2),
319	PANTHOR_EXCEPTION(PERM_FAULT_3),
320	PANTHOR_EXCEPTION(ACCESS_FLAG_1),
321	PANTHOR_EXCEPTION(ACCESS_FLAG_2),
322	PANTHOR_EXCEPTION(ACCESS_FLAG_3),
323	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN),
324	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
325	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
326	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
327	PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
328	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0),
329	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1),
330	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2),
331	PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3),
332};
333
334const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code)
335{
336	if (exception_code >= ARRAY_SIZE(panthor_exception_infos) ||
337	    !panthor_exception_infos[exception_code].name)
338		return "Unknown exception type";
339
340	return panthor_exception_infos[exception_code].name;
341}
342
343static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf)
344{
345	struct vm_area_struct *vma = vmf->vma;
346	struct panthor_device *ptdev = vma->vm_private_data;
347	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
348	unsigned long pfn;
349	pgprot_t pgprot;
350	vm_fault_t ret;
351	bool active;
352	int cookie;
353
354	if (!drm_dev_enter(&ptdev->base, &cookie))
355		return VM_FAULT_SIGBUS;
356
357	mutex_lock(&ptdev->pm.mmio_lock);
358	active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE;
359
360	switch (offset) {
361	case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
362		if (active)
363			pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID);
364		else
365			pfn = page_to_pfn(ptdev->pm.dummy_latest_flush);
366		break;
367
368	default:
369		ret = VM_FAULT_SIGBUS;
370		goto out_unlock;
371	}
372
373	pgprot = vma->vm_page_prot;
374	if (active)
375		pgprot = pgprot_noncached(pgprot);
376
377	ret = vmf_insert_pfn_prot(vma, vmf->address, pfn, pgprot);
378
379out_unlock:
380	mutex_unlock(&ptdev->pm.mmio_lock);
381	drm_dev_exit(cookie);
382	return ret;
383}
384
385static const struct vm_operations_struct panthor_mmio_vm_ops = {
386	.fault = panthor_mmio_vm_fault,
387};
388
389int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma)
390{
391	u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT;
392
393	if ((vma->vm_flags & VM_SHARED) == 0)
394		return -EINVAL;
395
396	switch (offset) {
397	case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET:
398		if (vma->vm_end - vma->vm_start != PAGE_SIZE ||
399		    (vma->vm_flags & (VM_WRITE | VM_EXEC)))
400			return -EINVAL;
401		vm_flags_clear(vma, VM_MAYWRITE);
402
403		break;
404
405	default:
406		return -EINVAL;
407	}
408
409	/* Defer actual mapping to the fault handler. */
410	vma->vm_private_data = ptdev;
411	vma->vm_ops = &panthor_mmio_vm_ops;
412	vm_flags_set(vma,
413		     VM_IO | VM_DONTCOPY | VM_DONTEXPAND |
414		     VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
415	return 0;
416}
417
418int panthor_device_resume(struct device *dev)
419{
420	struct panthor_device *ptdev = dev_get_drvdata(dev);
421	int ret, cookie;
422
423	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED)
424		return -EINVAL;
425
426	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_RESUMING);
427
428	ret = clk_prepare_enable(ptdev->clks.core);
429	if (ret)
430		goto err_set_suspended;
431
432	ret = clk_prepare_enable(ptdev->clks.stacks);
433	if (ret)
434		goto err_disable_core_clk;
435
436	ret = clk_prepare_enable(ptdev->clks.coregroup);
437	if (ret)
438		goto err_disable_stacks_clk;
439
440	ret = panthor_devfreq_resume(ptdev);
441	if (ret)
442		goto err_disable_coregroup_clk;
443
444	if (panthor_device_is_initialized(ptdev) &&
445	    drm_dev_enter(&ptdev->base, &cookie)) {
446		panthor_gpu_resume(ptdev);
447		panthor_mmu_resume(ptdev);
448		ret = panthor_fw_resume(ptdev);
449		if (!drm_WARN_ON(&ptdev->base, ret)) {
450			panthor_sched_resume(ptdev);
451		} else {
452			panthor_mmu_suspend(ptdev);
453			panthor_gpu_suspend(ptdev);
454		}
455
456		drm_dev_exit(cookie);
457
458		if (ret)
459			goto err_suspend_devfreq;
460	}
461
462	if (atomic_read(&ptdev->reset.pending))
463		queue_work(ptdev->reset.wq, &ptdev->reset.work);
464
465	/* Clear all IOMEM mappings pointing to this device after we've
466	 * resumed. This way the fake mappings pointing to the dummy pages
467	 * are removed and the real iomem mapping will be restored on next
468	 * access.
469	 */
470	mutex_lock(&ptdev->pm.mmio_lock);
471	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
472			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
473	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);
474	mutex_unlock(&ptdev->pm.mmio_lock);
475	return 0;
476
477err_suspend_devfreq:
478	panthor_devfreq_suspend(ptdev);
479
480err_disable_coregroup_clk:
481	clk_disable_unprepare(ptdev->clks.coregroup);
482
483err_disable_stacks_clk:
484	clk_disable_unprepare(ptdev->clks.stacks);
485
486err_disable_core_clk:
487	clk_disable_unprepare(ptdev->clks.core);
488
489err_set_suspended:
490	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
491	return ret;
492}
493
494int panthor_device_suspend(struct device *dev)
495{
496	struct panthor_device *ptdev = dev_get_drvdata(dev);
497	int ret, cookie;
498
499	if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE)
500		return -EINVAL;
501
502	/* Clear all IOMEM mappings pointing to this device before we
503	 * shutdown the power-domain and clocks. Failing to do that results
504	 * in external aborts when the process accesses the iomem region.
505	 * We change the state and call unmap_mapping_range() with the
506	 * mmio_lock held to make sure the vm_fault handler won't set up
507	 * invalid mappings.
508	 */
509	mutex_lock(&ptdev->pm.mmio_lock);
510	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDING);
511	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
512			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
513	mutex_unlock(&ptdev->pm.mmio_lock);
514
515	if (panthor_device_is_initialized(ptdev) &&
516	    drm_dev_enter(&ptdev->base, &cookie)) {
517		cancel_work_sync(&ptdev->reset.work);
518
519		/* We prepare everything as if we were resetting the GPU.
520		 * The end of the reset will happen in the resume path though.
521		 */
522		panthor_sched_suspend(ptdev);
523		panthor_fw_suspend(ptdev);
524		panthor_mmu_suspend(ptdev);
525		panthor_gpu_suspend(ptdev);
526		drm_dev_exit(cookie);
527	}
528
529	ret = panthor_devfreq_suspend(ptdev);
530	if (ret) {
531		if (panthor_device_is_initialized(ptdev) &&
532		    drm_dev_enter(&ptdev->base, &cookie)) {
533			panthor_gpu_resume(ptdev);
534			panthor_mmu_resume(ptdev);
535			drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev));
536			panthor_sched_resume(ptdev);
537			drm_dev_exit(cookie);
538		}
539
540		goto err_set_active;
541	}
542
543	clk_disable_unprepare(ptdev->clks.coregroup);
544	clk_disable_unprepare(ptdev->clks.stacks);
545	clk_disable_unprepare(ptdev->clks.core);
546	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED);
547	return 0;
548
549err_set_active:
550	/* If something failed and we have to revert back to an
551	 * active state, we also need to clear the MMIO userspace
552	 * mappings, so any dumb pages that were mapped while we
553	 * were trying to suspend gets invalidated.
554	 */
555	mutex_lock(&ptdev->pm.mmio_lock);
556	atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE);
557	unmap_mapping_range(ptdev->base.anon_inode->i_mapping,
558			    DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1);
559	mutex_unlock(&ptdev->pm.mmio_lock);
560	return ret;
561}