1// SPDX-License-Identifier: GPL-2.0+
  2/* Copyright (C) 2014-2018 Broadcom */
  3
  4/**
  5 * DOC: Interrupt management for the V3D engine
  6 *
  7 * When we take a bin, render, TFU done, or CSD done interrupt, we
  8 * need to signal the fence for that job so that the scheduler can
  9 * queue up the next one and unblock any waiters.
 10 *
 11 * When we take the binner out of memory interrupt, we need to
 12 * allocate some new memory and pass it to the binner so that the
 13 * current job can make progress.
 14 */
 15
 16#include <linux/platform_device.h>
 17
 18#include "v3d_drv.h"
 19#include "v3d_regs.h"
 20#include "v3d_trace.h"
 21
 22#define V3D_CORE_IRQS ((u32)(V3D_INT_OUTOMEM |	\
 23			     V3D_INT_FLDONE |	\
 24			     V3D_INT_FRDONE |	\
 25			     V3D_INT_CSDDONE |	\
 26			     V3D_INT_GMPV))
 27
 28#define V3D_HUB_IRQS ((u32)(V3D_HUB_INT_MMU_WRV |	\
 29			    V3D_HUB_INT_MMU_PTI |	\
 30			    V3D_HUB_INT_MMU_CAP |	\
 31			    V3D_HUB_INT_TFUC))
 32
 33static irqreturn_t
 34v3d_hub_irq(int irq, void *arg);
 35
 36static void
 37v3d_overflow_mem_work(struct work_struct *work)
 38{
 39	struct v3d_dev *v3d =
 40		container_of(work, struct v3d_dev, overflow_mem_work);
 41	struct drm_device *dev = &v3d->drm;
 42	struct v3d_bo *bo = v3d_bo_create(dev, NULL /* XXX: GMP */, 256 * 1024);
 43	struct drm_gem_object *obj;
 44	unsigned long irqflags;
 45
 46	if (IS_ERR(bo)) {
 47		DRM_ERROR("Couldn't allocate binner overflow mem\n");
 48		return;
 49	}
 50	obj = &bo->base.base;
 51
 52	/* We lost a race, and our work task came in after the bin job
 53	 * completed and exited.  This can happen because the HW
 54	 * signals OOM before it's fully OOM, so the binner might just
 55	 * barely complete.
 56	 *
 57	 * If we lose the race and our work task comes in after a new
 58	 * bin job got scheduled, that's fine.  We'll just give them
 59	 * some binner pool anyway.
 60	 */
 61	spin_lock_irqsave(&v3d->job_lock, irqflags);
 62	if (!v3d->bin_job) {
 63		spin_unlock_irqrestore(&v3d->job_lock, irqflags);
 64		goto out;
 65	}
 66
 67	drm_gem_object_get(obj);
 68	list_add_tail(&bo->unref_head, &v3d->bin_job->render->unref_list);
 69	spin_unlock_irqrestore(&v3d->job_lock, irqflags);
 70
 71	V3D_CORE_WRITE(0, V3D_PTB_BPOA, bo->node.start << PAGE_SHIFT);
 72	V3D_CORE_WRITE(0, V3D_PTB_BPOS, obj->size);
 73
 74out:
 75	drm_gem_object_put(obj);
 76}
 77
 78static irqreturn_t
 79v3d_irq(int irq, void *arg)
 80{
 81	struct v3d_dev *v3d = arg;
 82	u32 intsts;
 83	irqreturn_t status = IRQ_NONE;
 84
 85	intsts = V3D_CORE_READ(0, V3D_CTL_INT_STS);
 86
 87	/* Acknowledge the interrupts we're handling here. */
 88	V3D_CORE_WRITE(0, V3D_CTL_INT_CLR, intsts);
 89
 90	if (intsts & V3D_INT_OUTOMEM) {
 91		/* Note that the OOM status is edge signaled, so the
 92		 * interrupt won't happen again until the we actually
 93		 * add more memory.  Also, as of V3D 4.1, FLDONE won't
 94		 * be reported until any OOM state has been cleared.
 95		 */
 96		schedule_work(&v3d->overflow_mem_work);
 97		status = IRQ_HANDLED;
 98	}
 99
100	if (intsts & V3D_INT_FLDONE) {
101		struct v3d_fence *fence =
102			to_v3d_fence(v3d->bin_job->base.irq_fence);
103
104		trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
105		dma_fence_signal(&fence->base);
106		status = IRQ_HANDLED;
107	}
108
109	if (intsts & V3D_INT_FRDONE) {
110		struct v3d_fence *fence =
111			to_v3d_fence(v3d->render_job->base.irq_fence);
112
113		trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
114		dma_fence_signal(&fence->base);
115		status = IRQ_HANDLED;
116	}
117
118	if (intsts & V3D_INT_CSDDONE) {
119		struct v3d_fence *fence =
120			to_v3d_fence(v3d->csd_job->base.irq_fence);
121
122		trace_v3d_csd_irq(&v3d->drm, fence->seqno);
123		dma_fence_signal(&fence->base);
124		status = IRQ_HANDLED;
125	}
126
127	/* We shouldn't be triggering these if we have GMP in
128	 * always-allowed mode.
129	 */
130	if (intsts & V3D_INT_GMPV)
131		dev_err(v3d->drm.dev, "GMP violation\n");
132
133	/* V3D 4.2 wires the hub and core IRQs together, so if we &
134	 * didn't see the common one then check hub for MMU IRQs.
135	 */
136	if (v3d->single_irq_line && status == IRQ_NONE)
137		return v3d_hub_irq(irq, arg);
138
139	return status;
140}
141
142static irqreturn_t
143v3d_hub_irq(int irq, void *arg)
144{
145	struct v3d_dev *v3d = arg;
146	u32 intsts;
147	irqreturn_t status = IRQ_NONE;
148
149	intsts = V3D_READ(V3D_HUB_INT_STS);
150
151	/* Acknowledge the interrupts we're handling here. */
152	V3D_WRITE(V3D_HUB_INT_CLR, intsts);
153
154	if (intsts & V3D_HUB_INT_TFUC) {
155		struct v3d_fence *fence =
156			to_v3d_fence(v3d->tfu_job->base.irq_fence);
157
158		trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
159		dma_fence_signal(&fence->base);
160		status = IRQ_HANDLED;
161	}
162
163	if (intsts & (V3D_HUB_INT_MMU_WRV |
164		      V3D_HUB_INT_MMU_PTI |
165		      V3D_HUB_INT_MMU_CAP)) {
166		u32 axi_id = V3D_READ(V3D_MMU_VIO_ID);
167		u64 vio_addr = ((u64)V3D_READ(V3D_MMU_VIO_ADDR) <<
168				(v3d->va_width - 32));
169		static const char *const v3d41_axi_ids[] = {
170			"L2T",
171			"PTB",
172			"PSE",
173			"TLB",
174			"CLE",
175			"TFU",
176			"MMU",
177			"GMP",
178		};
179		const char *client = "?";
180
181		V3D_WRITE(V3D_MMU_CTL,
182			  V3D_READ(V3D_MMU_CTL) & (V3D_MMU_CTL_CAP_EXCEEDED |
183						   V3D_MMU_CTL_PT_INVALID |
184						   V3D_MMU_CTL_WRITE_VIOLATION));
185
186		if (v3d->ver >= 41) {
187			axi_id = axi_id >> 5;
188			if (axi_id < ARRAY_SIZE(v3d41_axi_ids))
189				client = v3d41_axi_ids[axi_id];
190		}
191
192		dev_err(v3d->drm.dev, "MMU error from client %s (%d) at 0x%llx%s%s%s\n",
193			client, axi_id, (long long)vio_addr,
194			((intsts & V3D_HUB_INT_MMU_WRV) ?
195			 ", write violation" : ""),
196			((intsts & V3D_HUB_INT_MMU_PTI) ?
197			 ", pte invalid" : ""),
198			((intsts & V3D_HUB_INT_MMU_CAP) ?
199			 ", cap exceeded" : ""));
200		status = IRQ_HANDLED;
201	}
202
203	return status;
204}
205
206int
207v3d_irq_init(struct v3d_dev *v3d)
208{
209	int irq1, ret, core;
210
211	INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
212
213	/* Clear any pending interrupts someone might have left around
214	 * for us.
215	 */
216	for (core = 0; core < v3d->cores; core++)
217		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS);
218	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS);
219
220	irq1 = platform_get_irq(v3d_to_pdev(v3d), 1);
221	if (irq1 == -EPROBE_DEFER)
222		return irq1;
223	if (irq1 > 0) {
224		ret = devm_request_irq(v3d->drm.dev, irq1,
225				       v3d_irq, IRQF_SHARED,
226				       "v3d_core0", v3d);
227		if (ret)
228			goto fail;
229		ret = devm_request_irq(v3d->drm.dev,
230				       platform_get_irq(v3d_to_pdev(v3d), 0),
231				       v3d_hub_irq, IRQF_SHARED,
232				       "v3d_hub", v3d);
233		if (ret)
234			goto fail;
235	} else {
236		v3d->single_irq_line = true;
237
238		ret = devm_request_irq(v3d->drm.dev,
239				       platform_get_irq(v3d_to_pdev(v3d), 0),
240				       v3d_irq, IRQF_SHARED,
241				       "v3d", v3d);
242		if (ret)
243			goto fail;
244	}
245
246	v3d_irq_enable(v3d);
247	return 0;
248
249fail:
250	if (ret != -EPROBE_DEFER)
251		dev_err(v3d->drm.dev, "IRQ setup failed: %d\n", ret);
252	return ret;
253}
254
255void
256v3d_irq_enable(struct v3d_dev *v3d)
257{
258	int core;
259
260	/* Enable our set of interrupts, masking out any others. */
261	for (core = 0; core < v3d->cores; core++) {
262		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS);
263		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_CLR, V3D_CORE_IRQS);
264	}
265
266	V3D_WRITE(V3D_HUB_INT_MSK_SET, ~V3D_HUB_IRQS);
267	V3D_WRITE(V3D_HUB_INT_MSK_CLR, V3D_HUB_IRQS);
268}
269
270void
271v3d_irq_disable(struct v3d_dev *v3d)
272{
273	int core;
274
275	/* Disable all interrupts. */
276	for (core = 0; core < v3d->cores; core++)
277		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~0);
278	V3D_WRITE(V3D_HUB_INT_MSK_SET, ~0);
279
280	/* Clear any pending interrupts we might have left. */
281	for (core = 0; core < v3d->cores; core++)
282		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS);
283	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS);
284
285	cancel_work_sync(&v3d->overflow_mem_work);
286}
287
288/** Reinitializes interrupt registers when a GPU reset is performed. */
289void v3d_irq_reset(struct v3d_dev *v3d)
290{
291	v3d_irq_enable(v3d);
292}