Loading...
Note: File does not exist in v3.1.
1/* SPDX-License-Identifier: GPL-2.0 or MIT */
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4/* Copyright 2023 Collabora ltd. */
5
6#ifndef __PANTHOR_DEVICE_H__
7#define __PANTHOR_DEVICE_H__
8
9#include <linux/atomic.h>
10#include <linux/io-pgtable.h>
11#include <linux/regulator/consumer.h>
12#include <linux/sched.h>
13#include <linux/spinlock.h>
14
15#include <drm/drm_device.h>
16#include <drm/drm_mm.h>
17#include <drm/gpu_scheduler.h>
18#include <drm/panthor_drm.h>
19
20struct panthor_csf;
21struct panthor_csf_ctx;
22struct panthor_device;
23struct panthor_gpu;
24struct panthor_group_pool;
25struct panthor_heap_pool;
26struct panthor_job;
27struct panthor_mmu;
28struct panthor_fw;
29struct panthor_perfcnt;
30struct panthor_vm;
31struct panthor_vm_pool;
32
33/**
34 * enum panthor_device_pm_state - PM state
35 */
36enum panthor_device_pm_state {
37 /** @PANTHOR_DEVICE_PM_STATE_SUSPENDED: Device is suspended. */
38 PANTHOR_DEVICE_PM_STATE_SUSPENDED = 0,
39
40 /** @PANTHOR_DEVICE_PM_STATE_RESUMING: Device is being resumed. */
41 PANTHOR_DEVICE_PM_STATE_RESUMING,
42
43 /** @PANTHOR_DEVICE_PM_STATE_ACTIVE: Device is active. */
44 PANTHOR_DEVICE_PM_STATE_ACTIVE,
45
46 /** @PANTHOR_DEVICE_PM_STATE_SUSPENDING: Device is being suspended. */
47 PANTHOR_DEVICE_PM_STATE_SUSPENDING,
48};
49
50/**
51 * struct panthor_irq - IRQ data
52 *
53 * Used to automate IRQ handling for the 3 different IRQs we have in this driver.
54 */
55struct panthor_irq {
56 /** @ptdev: Panthor device */
57 struct panthor_device *ptdev;
58
59 /** @irq: IRQ number. */
60 int irq;
61
62 /** @mask: Current mask being applied to xxx_INT_MASK. */
63 u32 mask;
64
65 /** @suspended: Set to true when the IRQ is suspended. */
66 atomic_t suspended;
67};
68
69/**
70 * enum panthor_device_profiling_mode - Profiling state
71 */
72enum panthor_device_profiling_flags {
73 /** @PANTHOR_DEVICE_PROFILING_DISABLED: Profiling is disabled. */
74 PANTHOR_DEVICE_PROFILING_DISABLED = 0,
75
76 /** @PANTHOR_DEVICE_PROFILING_CYCLES: Sampling job cycles. */
77 PANTHOR_DEVICE_PROFILING_CYCLES = BIT(0),
78
79 /** @PANTHOR_DEVICE_PROFILING_TIMESTAMP: Sampling job timestamp. */
80 PANTHOR_DEVICE_PROFILING_TIMESTAMP = BIT(1),
81
82 /** @PANTHOR_DEVICE_PROFILING_ALL: Sampling everything. */
83 PANTHOR_DEVICE_PROFILING_ALL =
84 PANTHOR_DEVICE_PROFILING_CYCLES |
85 PANTHOR_DEVICE_PROFILING_TIMESTAMP,
86};
87
88/**
89 * struct panthor_device - Panthor device
90 */
91struct panthor_device {
92 /** @base: Base drm_device. */
93 struct drm_device base;
94
95 /** @phys_addr: Physical address of the iomem region. */
96 phys_addr_t phys_addr;
97
98 /** @iomem: CPU mapping of the IOMEM region. */
99 void __iomem *iomem;
100
101 /** @clks: GPU clocks. */
102 struct {
103 /** @core: Core clock. */
104 struct clk *core;
105
106 /** @stacks: Stacks clock. This clock is optional. */
107 struct clk *stacks;
108
109 /** @coregroup: Core group clock. This clock is optional. */
110 struct clk *coregroup;
111 } clks;
112
113 /** @coherent: True if the CPU/GPU are memory coherent. */
114 bool coherent;
115
116 /** @gpu_info: GPU information. */
117 struct drm_panthor_gpu_info gpu_info;
118
119 /** @csif_info: Command stream interface information. */
120 struct drm_panthor_csif_info csif_info;
121
122 /** @gpu: GPU management data. */
123 struct panthor_gpu *gpu;
124
125 /** @fw: FW management data. */
126 struct panthor_fw *fw;
127
128 /** @mmu: MMU management data. */
129 struct panthor_mmu *mmu;
130
131 /** @scheduler: Scheduler management data. */
132 struct panthor_scheduler *scheduler;
133
134 /** @devfreq: Device frequency scaling management data. */
135 struct panthor_devfreq *devfreq;
136
137 /** @unplug: Device unplug related fields. */
138 struct {
139 /** @lock: Lock used to serialize unplug operations. */
140 struct mutex lock;
141
142 /**
143 * @done: Completion object signaled when the unplug
144 * operation is done.
145 */
146 struct completion done;
147 } unplug;
148
149 /** @reset: Reset related fields. */
150 struct {
151 /** @wq: Ordered worqueud used to schedule reset operations. */
152 struct workqueue_struct *wq;
153
154 /** @work: Reset work. */
155 struct work_struct work;
156
157 /** @pending: Set to true if a reset is pending. */
158 atomic_t pending;
159 } reset;
160
161 /** @pm: Power management related data. */
162 struct {
163 /** @state: Power state. */
164 atomic_t state;
165
166 /**
167 * @mmio_lock: Lock protecting MMIO userspace CPU mappings.
168 *
169 * This is needed to ensure we map the dummy IO pages when
170 * the device is being suspended, and the real IO pages when
171 * the device is being resumed. We can't just do with the
172 * state atomicity to deal with this race.
173 */
174 struct mutex mmio_lock;
175
176 /**
177 * @dummy_latest_flush: Dummy LATEST_FLUSH page.
178 *
179 * Used to replace the real LATEST_FLUSH page when the GPU
180 * is suspended.
181 */
182 struct page *dummy_latest_flush;
183 } pm;
184
185 /** @profile_mask: User-set profiling flags for job accounting. */
186 u32 profile_mask;
187
188 /** @current_frequency: Device clock frequency at present. Set by DVFS*/
189 unsigned long current_frequency;
190
191 /** @fast_rate: Maximum device clock frequency. Set by DVFS */
192 unsigned long fast_rate;
193};
194
195struct panthor_gpu_usage {
196 u64 time;
197 u64 cycles;
198};
199
200/**
201 * struct panthor_file - Panthor file
202 */
203struct panthor_file {
204 /** @ptdev: Device attached to this file. */
205 struct panthor_device *ptdev;
206
207 /** @vms: VM pool attached to this file. */
208 struct panthor_vm_pool *vms;
209
210 /** @groups: Scheduling group pool attached to this file. */
211 struct panthor_group_pool *groups;
212
213 /** @stats: cycle and timestamp measures for job execution. */
214 struct panthor_gpu_usage stats;
215};
216
217int panthor_device_init(struct panthor_device *ptdev);
218void panthor_device_unplug(struct panthor_device *ptdev);
219
220/**
221 * panthor_device_schedule_reset() - Schedules a reset operation
222 */
223static inline void panthor_device_schedule_reset(struct panthor_device *ptdev)
224{
225 if (!atomic_cmpxchg(&ptdev->reset.pending, 0, 1) &&
226 atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE)
227 queue_work(ptdev->reset.wq, &ptdev->reset.work);
228}
229
230/**
231 * panthor_device_reset_is_pending() - Checks if a reset is pending.
232 *
233 * Return: true if a reset is pending, false otherwise.
234 */
235static inline bool panthor_device_reset_is_pending(struct panthor_device *ptdev)
236{
237 return atomic_read(&ptdev->reset.pending) != 0;
238}
239
240int panthor_device_mmap_io(struct panthor_device *ptdev,
241 struct vm_area_struct *vma);
242
243int panthor_device_resume(struct device *dev);
244int panthor_device_suspend(struct device *dev);
245
246enum drm_panthor_exception_type {
247 DRM_PANTHOR_EXCEPTION_OK = 0x00,
248 DRM_PANTHOR_EXCEPTION_TERMINATED = 0x04,
249 DRM_PANTHOR_EXCEPTION_KABOOM = 0x05,
250 DRM_PANTHOR_EXCEPTION_EUREKA = 0x06,
251 DRM_PANTHOR_EXCEPTION_ACTIVE = 0x08,
252 DRM_PANTHOR_EXCEPTION_CS_RES_TERM = 0x0f,
253 DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT = 0x3f,
254 DRM_PANTHOR_EXCEPTION_CS_CONFIG_FAULT = 0x40,
255 DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE = 0x41,
256 DRM_PANTHOR_EXCEPTION_CS_ENDPOINT_FAULT = 0x44,
257 DRM_PANTHOR_EXCEPTION_CS_BUS_FAULT = 0x48,
258 DRM_PANTHOR_EXCEPTION_CS_INSTR_INVALID = 0x49,
259 DRM_PANTHOR_EXCEPTION_CS_CALL_STACK_OVERFLOW = 0x4a,
260 DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT = 0x4b,
261 DRM_PANTHOR_EXCEPTION_INSTR_INVALID_PC = 0x50,
262 DRM_PANTHOR_EXCEPTION_INSTR_INVALID_ENC = 0x51,
263 DRM_PANTHOR_EXCEPTION_INSTR_BARRIER_FAULT = 0x55,
264 DRM_PANTHOR_EXCEPTION_DATA_INVALID_FAULT = 0x58,
265 DRM_PANTHOR_EXCEPTION_TILE_RANGE_FAULT = 0x59,
266 DRM_PANTHOR_EXCEPTION_ADDR_RANGE_FAULT = 0x5a,
267 DRM_PANTHOR_EXCEPTION_IMPRECISE_FAULT = 0x5b,
268 DRM_PANTHOR_EXCEPTION_OOM = 0x60,
269 DRM_PANTHOR_EXCEPTION_CSF_FW_INTERNAL_ERROR = 0x68,
270 DRM_PANTHOR_EXCEPTION_CSF_RES_EVICTION_TIMEOUT = 0x69,
271 DRM_PANTHOR_EXCEPTION_GPU_BUS_FAULT = 0x80,
272 DRM_PANTHOR_EXCEPTION_GPU_SHAREABILITY_FAULT = 0x88,
273 DRM_PANTHOR_EXCEPTION_SYS_SHAREABILITY_FAULT = 0x89,
274 DRM_PANTHOR_EXCEPTION_GPU_CACHEABILITY_FAULT = 0x8a,
275 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_0 = 0xc0,
276 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_1 = 0xc1,
277 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_2 = 0xc2,
278 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_3 = 0xc3,
279 DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_4 = 0xc4,
280 DRM_PANTHOR_EXCEPTION_PERM_FAULT_0 = 0xc8,
281 DRM_PANTHOR_EXCEPTION_PERM_FAULT_1 = 0xc9,
282 DRM_PANTHOR_EXCEPTION_PERM_FAULT_2 = 0xca,
283 DRM_PANTHOR_EXCEPTION_PERM_FAULT_3 = 0xcb,
284 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_1 = 0xd9,
285 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_2 = 0xda,
286 DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_3 = 0xdb,
287 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_IN = 0xe0,
288 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT0 = 0xe4,
289 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT1 = 0xe5,
290 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT2 = 0xe6,
291 DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT3 = 0xe7,
292 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_0 = 0xe8,
293 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_1 = 0xe9,
294 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_2 = 0xea,
295 DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_3 = 0xeb,
296};
297
298/**
299 * panthor_exception_is_fault() - Checks if an exception is a fault.
300 *
301 * Return: true if the exception is a fault, false otherwise.
302 */
303static inline bool
304panthor_exception_is_fault(u32 exception_code)
305{
306 return exception_code > DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT;
307}
308
309const char *panthor_exception_name(struct panthor_device *ptdev,
310 u32 exception_code);
311
312/**
313 * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt
314 * registration function.
315 *
316 * The boiler-plate to gracefully deal with shared interrupts is
317 * auto-generated. All you have to do is call PANTHOR_IRQ_HANDLER()
318 * just after the actual handler. The handler prototype is:
319 *
320 * void (*handler)(struct panthor_device *, u32 status);
321 */
322#define PANTHOR_IRQ_HANDLER(__name, __reg_prefix, __handler) \
323static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data) \
324{ \
325 struct panthor_irq *pirq = data; \
326 struct panthor_device *ptdev = pirq->ptdev; \
327 \
328 if (atomic_read(&pirq->suspended)) \
329 return IRQ_NONE; \
330 if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT)) \
331 return IRQ_NONE; \
332 \
333 gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0); \
334 return IRQ_WAKE_THREAD; \
335} \
336 \
337static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *data) \
338{ \
339 struct panthor_irq *pirq = data; \
340 struct panthor_device *ptdev = pirq->ptdev; \
341 irqreturn_t ret = IRQ_NONE; \
342 \
343 while (true) { \
344 u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask; \
345 \
346 if (!status) \
347 break; \
348 \
349 gpu_write(ptdev, __reg_prefix ## _INT_CLEAR, status); \
350 \
351 __handler(ptdev, status); \
352 ret = IRQ_HANDLED; \
353 } \
354 \
355 if (!atomic_read(&pirq->suspended)) \
356 gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \
357 \
358 return ret; \
359} \
360 \
361static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) \
362{ \
363 pirq->mask = 0; \
364 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \
365 synchronize_irq(pirq->irq); \
366 atomic_set(&pirq->suspended, true); \
367} \
368 \
369static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq, u32 mask) \
370{ \
371 atomic_set(&pirq->suspended, false); \
372 pirq->mask = mask; \
373 gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, mask); \
374 gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, mask); \
375} \
376 \
377static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev, \
378 struct panthor_irq *pirq, \
379 int irq, u32 mask) \
380{ \
381 pirq->ptdev = ptdev; \
382 pirq->irq = irq; \
383 panthor_ ## __name ## _irq_resume(pirq, mask); \
384 \
385 return devm_request_threaded_irq(ptdev->base.dev, irq, \
386 panthor_ ## __name ## _irq_raw_handler, \
387 panthor_ ## __name ## _irq_threaded_handler, \
388 IRQF_SHARED, KBUILD_MODNAME "-" # __name, \
389 pirq); \
390}
391
392extern struct workqueue_struct *panthor_cleanup_wq;
393
394#endif