Loading...
Note: File does not exist in v3.15.
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2019 Collabora Ltd */
3
4#include <drm/drm_file.h>
5#include <drm/drm_gem_shmem_helper.h>
6#include <drm/panfrost_drm.h>
7#include <linux/completion.h>
8#include <linux/iopoll.h>
9#include <linux/pm_runtime.h>
10#include <linux/slab.h>
11#include <linux/uaccess.h>
12
13#include "panfrost_device.h"
14#include "panfrost_features.h"
15#include "panfrost_gem.h"
16#include "panfrost_issues.h"
17#include "panfrost_job.h"
18#include "panfrost_mmu.h"
19#include "panfrost_perfcnt.h"
20#include "panfrost_regs.h"
21
22#define COUNTERS_PER_BLOCK 64
23#define BYTES_PER_COUNTER 4
24#define BLOCKS_PER_COREGROUP 8
25#define V4_SHADERS_PER_COREGROUP 4
26
27struct panfrost_perfcnt {
28 struct panfrost_gem_object *bo;
29 size_t bosize;
30 void *buf;
31 struct panfrost_file_priv *user;
32 struct mutex lock;
33 struct completion dump_comp;
34};
35
36void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev)
37{
38 complete(&pfdev->perfcnt->dump_comp);
39}
40
41void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev)
42{
43 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES);
44}
45
46static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev)
47{
48 u64 gpuva;
49 int ret;
50
51 reinit_completion(&pfdev->perfcnt->dump_comp);
52 gpuva = pfdev->perfcnt->bo->node.start << PAGE_SHIFT;
53 gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva);
54 gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32);
55 gpu_write(pfdev, GPU_INT_CLEAR,
56 GPU_IRQ_CLEAN_CACHES_COMPLETED |
57 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
58 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE);
59 ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp,
60 msecs_to_jiffies(1000));
61 if (!ret)
62 ret = -ETIMEDOUT;
63 else if (ret > 0)
64 ret = 0;
65
66 return ret;
67}
68
69static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev,
70 struct panfrost_file_priv *user,
71 unsigned int counterset)
72{
73 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
74 struct drm_gem_shmem_object *bo;
75 u32 cfg;
76 int ret;
77
78 if (user == perfcnt->user)
79 return 0;
80 else if (perfcnt->user)
81 return -EBUSY;
82
83 ret = pm_runtime_get_sync(pfdev->dev);
84 if (ret < 0)
85 return ret;
86
87 bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize);
88 if (IS_ERR(bo))
89 return PTR_ERR(bo);
90
91 perfcnt->bo = to_panfrost_bo(&bo->base);
92
93 /* Map the perfcnt buf in the address space attached to file_priv. */
94 ret = panfrost_mmu_map(perfcnt->bo);
95 if (ret)
96 goto err_put_bo;
97
98 perfcnt->buf = drm_gem_shmem_vmap(&bo->base);
99 if (IS_ERR(perfcnt->buf)) {
100 ret = PTR_ERR(perfcnt->buf);
101 goto err_put_bo;
102 }
103
104 /*
105 * Invalidate the cache and clear the counters to start from a fresh
106 * state.
107 */
108 reinit_completion(&pfdev->perfcnt->dump_comp);
109 gpu_write(pfdev, GPU_INT_CLEAR,
110 GPU_IRQ_CLEAN_CACHES_COMPLETED |
111 GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
112 gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR);
113 gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES);
114 ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp,
115 msecs_to_jiffies(1000));
116 if (!ret) {
117 ret = -ETIMEDOUT;
118 goto err_vunmap;
119 }
120
121 perfcnt->user = user;
122
123 /*
124 * Always use address space 0 for now.
125 * FIXME: this needs to be updated when we start using different
126 * address space.
127 */
128 cfg = GPU_PERFCNT_CFG_AS(0) |
129 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL);
130
131 /*
132 * Bifrost GPUs have 2 set of counters, but we're only interested by
133 * the first one for now.
134 */
135 if (panfrost_model_is_bifrost(pfdev))
136 cfg |= GPU_PERFCNT_CFG_SETSEL(counterset);
137
138 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff);
139 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff);
140 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff);
141
142 /*
143 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW
144 * counters.
145 */
146 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
147 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
148 else
149 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
150
151 gpu_write(pfdev, GPU_PERFCNT_CFG, cfg);
152
153 if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
154 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
155
156 return 0;
157
158err_vunmap:
159 drm_gem_shmem_vunmap(&perfcnt->bo->base.base, perfcnt->buf);
160err_put_bo:
161 drm_gem_object_put_unlocked(&bo->base);
162 return ret;
163}
164
165static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev,
166 struct panfrost_file_priv *user)
167{
168 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
169
170 if (user != perfcnt->user)
171 return -EINVAL;
172
173 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0);
174 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0);
175 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0);
176 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
177 gpu_write(pfdev, GPU_PERFCNT_CFG,
178 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
179
180 perfcnt->user = NULL;
181 drm_gem_shmem_vunmap(&perfcnt->bo->base.base, perfcnt->buf);
182 perfcnt->buf = NULL;
183 drm_gem_object_put_unlocked(&perfcnt->bo->base.base);
184 perfcnt->bo = NULL;
185 pm_runtime_mark_last_busy(pfdev->dev);
186 pm_runtime_put_autosuspend(pfdev->dev);
187
188 return 0;
189}
190
191int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data,
192 struct drm_file *file_priv)
193{
194 struct panfrost_file_priv *pfile = file_priv->driver_priv;
195 struct panfrost_device *pfdev = dev->dev_private;
196 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
197 struct drm_panfrost_perfcnt_enable *req = data;
198 int ret;
199
200 ret = panfrost_unstable_ioctl_check();
201 if (ret)
202 return ret;
203
204 /* Only Bifrost GPUs have 2 set of counters. */
205 if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0))
206 return -EINVAL;
207
208 mutex_lock(&perfcnt->lock);
209 if (req->enable)
210 ret = panfrost_perfcnt_enable_locked(pfdev, pfile,
211 req->counterset);
212 else
213 ret = panfrost_perfcnt_disable_locked(pfdev, pfile);
214 mutex_unlock(&perfcnt->lock);
215
216 return ret;
217}
218
219int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data,
220 struct drm_file *file_priv)
221{
222 struct panfrost_device *pfdev = dev->dev_private;
223 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
224 struct drm_panfrost_perfcnt_dump *req = data;
225 void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr;
226 int ret;
227
228 ret = panfrost_unstable_ioctl_check();
229 if (ret)
230 return ret;
231
232 mutex_lock(&perfcnt->lock);
233 if (perfcnt->user != file_priv->driver_priv) {
234 ret = -EINVAL;
235 goto out;
236 }
237
238 ret = panfrost_perfcnt_dump_locked(pfdev);
239 if (ret)
240 goto out;
241
242 if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize))
243 ret = -EFAULT;
244
245out:
246 mutex_unlock(&perfcnt->lock);
247
248 return ret;
249}
250
251void panfrost_perfcnt_close(struct panfrost_file_priv *pfile)
252{
253 struct panfrost_device *pfdev = pfile->pfdev;
254 struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
255
256 pm_runtime_get_sync(pfdev->dev);
257 mutex_lock(&perfcnt->lock);
258 if (perfcnt->user == pfile)
259 panfrost_perfcnt_disable_locked(pfdev, pfile);
260 mutex_unlock(&perfcnt->lock);
261 pm_runtime_mark_last_busy(pfdev->dev);
262 pm_runtime_put_autosuspend(pfdev->dev);
263}
264
265int panfrost_perfcnt_init(struct panfrost_device *pfdev)
266{
267 struct panfrost_perfcnt *perfcnt;
268 size_t size;
269
270 if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) {
271 unsigned int ncoregroups;
272
273 ncoregroups = hweight64(pfdev->features.l2_present);
274 size = ncoregroups * BLOCKS_PER_COREGROUP *
275 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
276 } else {
277 unsigned int nl2c, ncores;
278
279 /*
280 * TODO: define a macro to extract the number of l2 caches from
281 * mem_features.
282 */
283 nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1;
284
285 /*
286 * shader_present might be sparse, but the counters layout
287 * forces to dump unused regions too, hence the fls64() call
288 * instead of hweight64().
289 */
290 ncores = fls64(pfdev->features.shader_present);
291
292 /*
293 * There's always one JM and one Tiler block, hence the '+ 2'
294 * here.
295 */
296 size = (nl2c + ncores + 2) *
297 COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
298 }
299
300 perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL);
301 if (!perfcnt)
302 return -ENOMEM;
303
304 perfcnt->bosize = size;
305
306 /* Start with everything disabled. */
307 gpu_write(pfdev, GPU_PERFCNT_CFG,
308 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
309 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
310 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
311 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
312 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
313
314 init_completion(&perfcnt->dump_comp);
315 mutex_init(&perfcnt->lock);
316 pfdev->perfcnt = perfcnt;
317
318 return 0;
319}
320
321void panfrost_perfcnt_fini(struct panfrost_device *pfdev)
322{
323 /* Disable everything before leaving. */
324 gpu_write(pfdev, GPU_PERFCNT_CFG,
325 GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
326 gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
327 gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
328 gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
329 gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
330}