1// SPDX-License-Identifier: GPL-2.0
  2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
  3/* Copyright 2019 Linaro, Ltd., Rob Herring <robh@kernel.org> */
  4/* Copyright 2019 Collabora ltd. */
  5
  6#include <linux/module.h>
  7#include <linux/of_platform.h>
  8#include <linux/pagemap.h>
  9#include <linux/pm_runtime.h>
 10#include <drm/panfrost_drm.h>
 11#include <drm/drm_drv.h>
 12#include <drm/drm_ioctl.h>
 13#include <drm/drm_syncobj.h>
 14#include <drm/drm_utils.h>
 15
 16#include "panfrost_device.h"
 17#include "panfrost_devfreq.h"
 18#include "panfrost_gem.h"
 19#include "panfrost_mmu.h"
 20#include "panfrost_job.h"
 21#include "panfrost_gpu.h"
 22#include "panfrost_perfcnt.h"
 23
 24static bool unstable_ioctls;
 25module_param_unsafe(unstable_ioctls, bool, 0600);
 26
 27static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file)
 28{
 29	struct drm_panfrost_get_param *param = data;
 30	struct panfrost_device *pfdev = ddev->dev_private;
 31
 32	if (param->pad != 0)
 33		return -EINVAL;
 34
 35#define PANFROST_FEATURE(name, member)			\
 36	case DRM_PANFROST_PARAM_ ## name:		\
 37		param->value = pfdev->features.member;	\
 38		break
 39#define PANFROST_FEATURE_ARRAY(name, member, max)			\
 40	case DRM_PANFROST_PARAM_ ## name ## 0 ...			\
 41		DRM_PANFROST_PARAM_ ## name ## max:			\
 42		param->value = pfdev->features.member[param->param -	\
 43			DRM_PANFROST_PARAM_ ## name ## 0];		\
 44		break
 45
 46	switch (param->param) {
 47		PANFROST_FEATURE(GPU_PROD_ID, id);
 48		PANFROST_FEATURE(GPU_REVISION, revision);
 49		PANFROST_FEATURE(SHADER_PRESENT, shader_present);
 50		PANFROST_FEATURE(TILER_PRESENT, tiler_present);
 51		PANFROST_FEATURE(L2_PRESENT, l2_present);
 52		PANFROST_FEATURE(STACK_PRESENT, stack_present);
 53		PANFROST_FEATURE(AS_PRESENT, as_present);
 54		PANFROST_FEATURE(JS_PRESENT, js_present);
 55		PANFROST_FEATURE(L2_FEATURES, l2_features);
 56		PANFROST_FEATURE(CORE_FEATURES, core_features);
 57		PANFROST_FEATURE(TILER_FEATURES, tiler_features);
 58		PANFROST_FEATURE(MEM_FEATURES, mem_features);
 59		PANFROST_FEATURE(MMU_FEATURES, mmu_features);
 60		PANFROST_FEATURE(THREAD_FEATURES, thread_features);
 61		PANFROST_FEATURE(MAX_THREADS, max_threads);
 62		PANFROST_FEATURE(THREAD_MAX_WORKGROUP_SZ,
 63				thread_max_workgroup_sz);
 64		PANFROST_FEATURE(THREAD_MAX_BARRIER_SZ,
 65				thread_max_barrier_sz);
 66		PANFROST_FEATURE(COHERENCY_FEATURES, coherency_features);
 67		PANFROST_FEATURE_ARRAY(TEXTURE_FEATURES, texture_features, 3);
 68		PANFROST_FEATURE_ARRAY(JS_FEATURES, js_features, 15);
 69		PANFROST_FEATURE(NR_CORE_GROUPS, nr_core_groups);
 70		PANFROST_FEATURE(THREAD_TLS_ALLOC, thread_tls_alloc);
 71	default:
 72		return -EINVAL;
 73	}
 74
 75	return 0;
 76}
 77
 78static int panfrost_ioctl_create_bo(struct drm_device *dev, void *data,
 79		struct drm_file *file)
 80{
 81	struct panfrost_gem_object *bo;
 82	struct drm_panfrost_create_bo *args = data;
 83
 84	if (!args->size || args->pad ||
 85	    (args->flags & ~(PANFROST_BO_NOEXEC | PANFROST_BO_HEAP)))
 86		return -EINVAL;
 87
 88	/* Heaps should never be executable */
 89	if ((args->flags & PANFROST_BO_HEAP) &&
 90	    !(args->flags & PANFROST_BO_NOEXEC))
 91		return -EINVAL;
 92
 93	bo = panfrost_gem_create_with_handle(file, dev, args->size, args->flags,
 94					     &args->handle);
 95	if (IS_ERR(bo))
 96		return PTR_ERR(bo);
 97
 98	args->offset = bo->node.start << PAGE_SHIFT;
 99
100	return 0;
101}
102
103/**
104 * panfrost_lookup_bos() - Sets up job->bo[] with the GEM objects
105 * referenced by the job.
106 * @dev: DRM device
107 * @file_priv: DRM file for this fd
108 * @args: IOCTL args
109 * @job: job being set up
110 *
111 * Resolve handles from userspace to BOs and attach them to job.
112 *
113 * Note that this function doesn't need to unreference the BOs on
114 * failure, because that will happen at panfrost_job_cleanup() time.
115 */
116static int
117panfrost_lookup_bos(struct drm_device *dev,
118		  struct drm_file *file_priv,
119		  struct drm_panfrost_submit *args,
120		  struct panfrost_job *job)
121{
122	job->bo_count = args->bo_handle_count;
123
124	if (!job->bo_count)
125		return 0;
126
127	job->implicit_fences = kvmalloc_array(job->bo_count,
128				  sizeof(struct dma_fence *),
129				  GFP_KERNEL | __GFP_ZERO);
130	if (!job->implicit_fences)
131		return -ENOMEM;
132
133	return drm_gem_objects_lookup(file_priv,
134				      (void __user *)(uintptr_t)args->bo_handles,
135				      job->bo_count, &job->bos);
136}
137
138/**
139 * panfrost_copy_in_sync() - Sets up job->in_fences[] with the sync objects
140 * referenced by the job.
141 * @dev: DRM device
142 * @file_priv: DRM file for this fd
143 * @args: IOCTL args
144 * @job: job being set up
145 *
146 * Resolve syncobjs from userspace to fences and attach them to job.
147 *
148 * Note that this function doesn't need to unreference the fences on
149 * failure, because that will happen at panfrost_job_cleanup() time.
150 */
151static int
152panfrost_copy_in_sync(struct drm_device *dev,
153		  struct drm_file *file_priv,
154		  struct drm_panfrost_submit *args,
155		  struct panfrost_job *job)
156{
157	u32 *handles;
158	int ret = 0;
159	int i;
160
161	job->in_fence_count = args->in_sync_count;
162
163	if (!job->in_fence_count)
164		return 0;
165
166	job->in_fences = kvmalloc_array(job->in_fence_count,
167					sizeof(struct dma_fence *),
168					GFP_KERNEL | __GFP_ZERO);
169	if (!job->in_fences) {
170		DRM_DEBUG("Failed to allocate job in fences\n");
171		return -ENOMEM;
172	}
173
174	handles = kvmalloc_array(job->in_fence_count, sizeof(u32), GFP_KERNEL);
175	if (!handles) {
176		ret = -ENOMEM;
177		DRM_DEBUG("Failed to allocate incoming syncobj handles\n");
178		goto fail;
179	}
180
181	if (copy_from_user(handles,
182			   (void __user *)(uintptr_t)args->in_syncs,
183			   job->in_fence_count * sizeof(u32))) {
184		ret = -EFAULT;
185		DRM_DEBUG("Failed to copy in syncobj handles\n");
186		goto fail;
187	}
188
189	for (i = 0; i < job->in_fence_count; i++) {
190		ret = drm_syncobj_find_fence(file_priv, handles[i], 0, 0,
191					     &job->in_fences[i]);
192		if (ret == -EINVAL)
193			goto fail;
194	}
195
196fail:
197	kvfree(handles);
198	return ret;
199}
200
201static int panfrost_ioctl_submit(struct drm_device *dev, void *data,
202		struct drm_file *file)
203{
204	struct panfrost_device *pfdev = dev->dev_private;
205	struct drm_panfrost_submit *args = data;
206	struct drm_syncobj *sync_out = NULL;
207	struct panfrost_job *job;
208	int ret = 0;
209
210	if (!args->jc)
211		return -EINVAL;
212
213	if (args->requirements && args->requirements != PANFROST_JD_REQ_FS)
214		return -EINVAL;
215
216	if (args->out_sync > 0) {
217		sync_out = drm_syncobj_find(file, args->out_sync);
218		if (!sync_out)
219			return -ENODEV;
220	}
221
222	job = kzalloc(sizeof(*job), GFP_KERNEL);
223	if (!job) {
224		ret = -ENOMEM;
225		goto fail_out_sync;
226	}
227
228	kref_init(&job->refcount);
229
230	job->pfdev = pfdev;
231	job->jc = args->jc;
232	job->requirements = args->requirements;
233	job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev);
234	job->file_priv = file->driver_priv;
235
236	ret = panfrost_copy_in_sync(dev, file, args, job);
237	if (ret)
238		goto fail_job;
239
240	ret = panfrost_lookup_bos(dev, file, args, job);
241	if (ret)
242		goto fail_job;
243
244	ret = panfrost_job_push(job);
245	if (ret)
246		goto fail_job;
247
248	/* Update the return sync object for the job */
249	if (sync_out)
250		drm_syncobj_replace_fence(sync_out, job->render_done_fence);
251
252fail_job:
253	panfrost_job_put(job);
254fail_out_sync:
255	if (sync_out)
256		drm_syncobj_put(sync_out);
257
258	return ret;
259}
260
261static int
262panfrost_ioctl_wait_bo(struct drm_device *dev, void *data,
263		       struct drm_file *file_priv)
264{
265	long ret;
266	struct drm_panfrost_wait_bo *args = data;
267	struct drm_gem_object *gem_obj;
268	unsigned long timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
269
270	if (args->pad)
271		return -EINVAL;
272
273	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
274	if (!gem_obj)
275		return -ENOENT;
276
277	ret = dma_resv_wait_timeout_rcu(gem_obj->resv, true,
278						  true, timeout);
279	if (!ret)
280		ret = timeout ? -ETIMEDOUT : -EBUSY;
281
282	drm_gem_object_put_unlocked(gem_obj);
283
284	return ret;
285}
286
287static int panfrost_ioctl_mmap_bo(struct drm_device *dev, void *data,
288		      struct drm_file *file_priv)
289{
290	struct drm_panfrost_mmap_bo *args = data;
291	struct drm_gem_object *gem_obj;
292	int ret;
293
294	if (args->flags != 0) {
295		DRM_INFO("unknown mmap_bo flags: %d\n", args->flags);
296		return -EINVAL;
297	}
298
299	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
300	if (!gem_obj) {
301		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
302		return -ENOENT;
303	}
304
305	/* Don't allow mmapping of heap objects as pages are not pinned. */
306	if (to_panfrost_bo(gem_obj)->is_heap)
307		return -EINVAL;
308
309	ret = drm_gem_create_mmap_offset(gem_obj);
310	if (ret == 0)
311		args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node);
312	drm_gem_object_put_unlocked(gem_obj);
313
314	return ret;
315}
316
317static int panfrost_ioctl_get_bo_offset(struct drm_device *dev, void *data,
318			    struct drm_file *file_priv)
319{
320	struct drm_panfrost_get_bo_offset *args = data;
321	struct drm_gem_object *gem_obj;
322	struct panfrost_gem_object *bo;
323
324	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
325	if (!gem_obj) {
326		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
327		return -ENOENT;
328	}
329	bo = to_panfrost_bo(gem_obj);
330
331	args->offset = bo->node.start << PAGE_SHIFT;
332
333	drm_gem_object_put_unlocked(gem_obj);
334	return 0;
335}
336
337static int panfrost_ioctl_madvise(struct drm_device *dev, void *data,
338				  struct drm_file *file_priv)
339{
340	struct drm_panfrost_madvise *args = data;
341	struct panfrost_device *pfdev = dev->dev_private;
342	struct drm_gem_object *gem_obj;
343
344	gem_obj = drm_gem_object_lookup(file_priv, args->handle);
345	if (!gem_obj) {
346		DRM_DEBUG("Failed to look up GEM BO %d\n", args->handle);
347		return -ENOENT;
348	}
349
350	args->retained = drm_gem_shmem_madvise(gem_obj, args->madv);
351
352	if (args->retained) {
353		struct panfrost_gem_object *bo = to_panfrost_bo(gem_obj);
354
355		mutex_lock(&pfdev->shrinker_lock);
356
357		if (args->madv == PANFROST_MADV_DONTNEED)
358			list_add_tail(&bo->base.madv_list, &pfdev->shrinker_list);
359		else if (args->madv == PANFROST_MADV_WILLNEED)
360			list_del_init(&bo->base.madv_list);
361
362		mutex_unlock(&pfdev->shrinker_lock);
363	}
364
365	drm_gem_object_put_unlocked(gem_obj);
366	return 0;
367}
368
369int panfrost_unstable_ioctl_check(void)
370{
371	if (!unstable_ioctls)
372		return -ENOSYS;
373
374	return 0;
375}
376
377#define PFN_4G		(SZ_4G >> PAGE_SHIFT)
378#define PFN_4G_MASK	(PFN_4G - 1)
379#define PFN_16M		(SZ_16M >> PAGE_SHIFT)
380
381static void panfrost_drm_mm_color_adjust(const struct drm_mm_node *node,
382					 unsigned long color,
383					 u64 *start, u64 *end)
384{
385	/* Executable buffers can't start or end on a 4GB boundary */
386	if (!(color & PANFROST_BO_NOEXEC)) {
387		u64 next_seg;
388
389		if ((*start & PFN_4G_MASK) == 0)
390			(*start)++;
391
392		if ((*end & PFN_4G_MASK) == 0)
393			(*end)--;
394
395		next_seg = ALIGN(*start, PFN_4G);
396		if (next_seg - *start <= PFN_16M)
397			*start = next_seg + 1;
398
399		*end = min(*end, ALIGN(*start, PFN_4G) - 1);
400	}
401}
402
403static int
404panfrost_open(struct drm_device *dev, struct drm_file *file)
405{
406	int ret;
407	struct panfrost_device *pfdev = dev->dev_private;
408	struct panfrost_file_priv *panfrost_priv;
409
410	panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL);
411	if (!panfrost_priv)
412		return -ENOMEM;
413
414	panfrost_priv->pfdev = pfdev;
415	file->driver_priv = panfrost_priv;
416
417	spin_lock_init(&panfrost_priv->mm_lock);
418
419	/* 4G enough for now. can be 48-bit */
420	drm_mm_init(&panfrost_priv->mm, SZ_32M >> PAGE_SHIFT, (SZ_4G - SZ_32M) >> PAGE_SHIFT);
421	panfrost_priv->mm.color_adjust = panfrost_drm_mm_color_adjust;
422
423	ret = panfrost_mmu_pgtable_alloc(panfrost_priv);
424	if (ret)
425		goto err_pgtable;
426
427	ret = panfrost_job_open(panfrost_priv);
428	if (ret)
429		goto err_job;
430
431	return 0;
432
433err_job:
434	panfrost_mmu_pgtable_free(panfrost_priv);
435err_pgtable:
436	drm_mm_takedown(&panfrost_priv->mm);
437	kfree(panfrost_priv);
438	return ret;
439}
440
441static void
442panfrost_postclose(struct drm_device *dev, struct drm_file *file)
443{
444	struct panfrost_file_priv *panfrost_priv = file->driver_priv;
445
446	panfrost_perfcnt_close(panfrost_priv);
447	panfrost_job_close(panfrost_priv);
448
449	panfrost_mmu_pgtable_free(panfrost_priv);
450	drm_mm_takedown(&panfrost_priv->mm);
451	kfree(panfrost_priv);
452}
453
454/* DRM_AUTH is required on SUBMIT for now, while all clients share a single
455 * address space.  Note that render nodes would be able to submit jobs that
456 * could access BOs from clients authenticated with the master node.
457 */
458static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = {
459#define PANFROST_IOCTL(n, func, flags) \
460	DRM_IOCTL_DEF_DRV(PANFROST_##n, panfrost_ioctl_##func, flags)
461
462	PANFROST_IOCTL(SUBMIT,		submit,		DRM_RENDER_ALLOW | DRM_AUTH),
463	PANFROST_IOCTL(WAIT_BO,		wait_bo,	DRM_RENDER_ALLOW),
464	PANFROST_IOCTL(CREATE_BO,	create_bo,	DRM_RENDER_ALLOW),
465	PANFROST_IOCTL(MMAP_BO,		mmap_bo,	DRM_RENDER_ALLOW),
466	PANFROST_IOCTL(GET_PARAM,	get_param,	DRM_RENDER_ALLOW),
467	PANFROST_IOCTL(GET_BO_OFFSET,	get_bo_offset,	DRM_RENDER_ALLOW),
468	PANFROST_IOCTL(PERFCNT_ENABLE,	perfcnt_enable,	DRM_RENDER_ALLOW),
469	PANFROST_IOCTL(PERFCNT_DUMP,	perfcnt_dump,	DRM_RENDER_ALLOW),
470	PANFROST_IOCTL(MADVISE,		madvise,	DRM_RENDER_ALLOW),
471};
472
473DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops);
474
475/*
476 * Panfrost driver version:
477 * - 1.0 - initial interface
478 * - 1.1 - adds HEAP and NOEXEC flags for CREATE_BO
479 */
480static struct drm_driver panfrost_drm_driver = {
481	.driver_features	= DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ,
482	.open			= panfrost_open,
483	.postclose		= panfrost_postclose,
484	.ioctls			= panfrost_drm_driver_ioctls,
485	.num_ioctls		= ARRAY_SIZE(panfrost_drm_driver_ioctls),
486	.fops			= &panfrost_drm_driver_fops,
487	.name			= "panfrost",
488	.desc			= "panfrost DRM",
489	.date			= "20180908",
490	.major			= 1,
491	.minor			= 1,
492
493	.gem_create_object	= panfrost_gem_create_object,
494	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd,
495	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle,
496	.gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table,
497	.gem_prime_mmap		= drm_gem_prime_mmap,
498};
499
500static int panfrost_probe(struct platform_device *pdev)
501{
502	struct panfrost_device *pfdev;
503	struct drm_device *ddev;
504	int err;
505
506	pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL);
507	if (!pfdev)
508		return -ENOMEM;
509
510	pfdev->pdev = pdev;
511	pfdev->dev = &pdev->dev;
512
513	platform_set_drvdata(pdev, pfdev);
514
515	/* Allocate and initialze the DRM device. */
516	ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev);
517	if (IS_ERR(ddev))
518		return PTR_ERR(ddev);
519
520	ddev->dev_private = pfdev;
521	pfdev->ddev = ddev;
522
523	mutex_init(&pfdev->shrinker_lock);
524	INIT_LIST_HEAD(&pfdev->shrinker_list);
525
526	err = panfrost_device_init(pfdev);
527	if (err) {
528		if (err != -EPROBE_DEFER)
529			dev_err(&pdev->dev, "Fatal error during GPU init\n");
530		goto err_out0;
531	}
532
533	err = panfrost_devfreq_init(pfdev);
534	if (err) {
535		if (err != -EPROBE_DEFER)
536			dev_err(&pdev->dev, "Fatal error during devfreq init\n");
537		goto err_out1;
538	}
539
540	pm_runtime_set_active(pfdev->dev);
541	pm_runtime_mark_last_busy(pfdev->dev);
542	pm_runtime_enable(pfdev->dev);
543	pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */
544	pm_runtime_use_autosuspend(pfdev->dev);
545
546	/*
547	 * Register the DRM device with the core and the connectors with
548	 * sysfs
549	 */
550	err = drm_dev_register(ddev, 0);
551	if (err < 0)
552		goto err_out2;
553
554	panfrost_gem_shrinker_init(ddev);
555
556	return 0;
557
558err_out2:
559	pm_runtime_disable(pfdev->dev);
560	panfrost_devfreq_fini(pfdev);
561err_out1:
562	panfrost_device_fini(pfdev);
563err_out0:
564	drm_dev_put(ddev);
565	return err;
566}
567
568static int panfrost_remove(struct platform_device *pdev)
569{
570	struct panfrost_device *pfdev = platform_get_drvdata(pdev);
571	struct drm_device *ddev = pfdev->ddev;
572
573	drm_dev_unregister(ddev);
574	panfrost_gem_shrinker_cleanup(ddev);
575
576	pm_runtime_get_sync(pfdev->dev);
577	panfrost_devfreq_fini(pfdev);
578	panfrost_device_fini(pfdev);
579	pm_runtime_put_sync_suspend(pfdev->dev);
580	pm_runtime_disable(pfdev->dev);
581
582	drm_dev_put(ddev);
583	return 0;
584}
585
586static const struct of_device_id dt_match[] = {
587	{ .compatible = "arm,mali-t604" },
588	{ .compatible = "arm,mali-t624" },
589	{ .compatible = "arm,mali-t628" },
590	{ .compatible = "arm,mali-t720" },
591	{ .compatible = "arm,mali-t760" },
592	{ .compatible = "arm,mali-t820" },
593	{ .compatible = "arm,mali-t830" },
594	{ .compatible = "arm,mali-t860" },
595	{ .compatible = "arm,mali-t880" },
596	{}
597};
598MODULE_DEVICE_TABLE(of, dt_match);
599
600static const struct dev_pm_ops panfrost_pm_ops = {
601	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
602	SET_RUNTIME_PM_OPS(panfrost_device_suspend, panfrost_device_resume, NULL)
603};
604
605static struct platform_driver panfrost_driver = {
606	.probe		= panfrost_probe,
607	.remove		= panfrost_remove,
608	.driver		= {
609		.name	= "panfrost",
610		.pm	= &panfrost_pm_ops,
611		.of_match_table = dt_match,
612	},
613};
614module_platform_driver(panfrost_driver);
615
616MODULE_AUTHOR("Panfrost Project Developers");
617MODULE_DESCRIPTION("Panfrost DRM Driver");
618MODULE_LICENSE("GPL v2");