1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2019 Intel Corporation
  4 */
  5
  6#include "gen7_renderclear.h"
  7#include "i915_drv.h"
  8#include "intel_gpu_commands.h"
  9
 10#define GT3_INLINE_DATA_DELAYS 0x1E00
 11#define batch_advance(Y, CS) GEM_BUG_ON((Y)->end != (CS))
 12
 13struct cb_kernel {
 14	const void *data;
 15	u32 size;
 16};
 17
 18#define CB_KERNEL(name) { .data = (name), .size = sizeof(name) }
 19
 20#include "ivb_clear_kernel.c"
 21static const struct cb_kernel cb_kernel_ivb = CB_KERNEL(ivb_clear_kernel);
 22
 23#include "hsw_clear_kernel.c"
 24static const struct cb_kernel cb_kernel_hsw = CB_KERNEL(hsw_clear_kernel);
 25
 26struct batch_chunk {
 27	struct i915_vma *vma;
 28	u32 offset;
 29	u32 *start;
 30	u32 *end;
 31	u32 max_items;
 32};
 33
 34struct batch_vals {
 35	u32 max_threads;
 36	u32 state_start;
 37	u32 surface_start;
 38	u32 surface_height;
 39	u32 surface_width;
 40	u32 size;
 41};
 42
 43static int num_primitives(const struct batch_vals *bv)
 44{
 45	/*
 46	 * We need to saturate the GPU with work in order to dispatch
 47	 * a shader on every HW thread, and clear the thread-local registers.
 48	 * In short, we have to dispatch work faster than the shaders can
 49	 * run in order to fill the EU and occupy each HW thread.
 50	 */
 51	return bv->max_threads;
 52}
 53
 54static void
 55batch_get_defaults(struct drm_i915_private *i915, struct batch_vals *bv)
 56{
 57	if (IS_HASWELL(i915)) {
 58		switch (INTEL_INFO(i915)->gt) {
 59		default:
 60		case 1:
 61			bv->max_threads = 70;
 62			break;
 63		case 2:
 64			bv->max_threads = 140;
 65			break;
 66		case 3:
 67			bv->max_threads = 280;
 68			break;
 69		}
 70		bv->surface_height = 16 * 16;
 71		bv->surface_width = 32 * 2 * 16;
 72	} else {
 73		switch (INTEL_INFO(i915)->gt) {
 74		default:
 75		case 1: /* including vlv */
 76			bv->max_threads = 36;
 77			break;
 78		case 2:
 79			bv->max_threads = 128;
 80			break;
 81		}
 82		bv->surface_height = 16 * 8;
 83		bv->surface_width = 32 * 16;
 84	}
 85	bv->state_start = round_up(SZ_1K + num_primitives(bv) * 64, SZ_4K);
 86	bv->surface_start = bv->state_start + SZ_4K;
 87	bv->size = bv->surface_start + bv->surface_height * bv->surface_width;
 88}
 89
 90static void batch_init(struct batch_chunk *bc,
 91		       struct i915_vma *vma,
 92		       u32 *start, u32 offset, u32 max_bytes)
 93{
 94	bc->vma = vma;
 95	bc->offset = offset;
 96	bc->start = start + bc->offset / sizeof(*bc->start);
 97	bc->end = bc->start;
 98	bc->max_items = max_bytes / sizeof(*bc->start);
 99}
100
101static u32 batch_offset(const struct batch_chunk *bc, u32 *cs)
102{
103	return (cs - bc->start) * sizeof(*bc->start) + bc->offset;
104}
105
106static u32 batch_addr(const struct batch_chunk *bc)
107{
108	return bc->vma->node.start;
109}
110
111static void batch_add(struct batch_chunk *bc, const u32 d)
112{
113	GEM_BUG_ON((bc->end - bc->start) >= bc->max_items);
114	*bc->end++ = d;
115}
116
117static u32 *batch_alloc_items(struct batch_chunk *bc, u32 align, u32 items)
118{
119	u32 *map;
120
121	if (align) {
122		u32 *end = PTR_ALIGN(bc->end, align);
123
124		memset32(bc->end, 0, end - bc->end);
125		bc->end = end;
126	}
127
128	map = bc->end;
129	bc->end += items;
130
131	return map;
132}
133
134static u32 *batch_alloc_bytes(struct batch_chunk *bc, u32 align, u32 bytes)
135{
136	GEM_BUG_ON(!IS_ALIGNED(bytes, sizeof(*bc->start)));
137	return batch_alloc_items(bc, align, bytes / sizeof(*bc->start));
138}
139
140static u32
141gen7_fill_surface_state(struct batch_chunk *state,
142			const u32 dst_offset,
143			const struct batch_vals *bv)
144{
145	u32 surface_h = bv->surface_height;
146	u32 surface_w = bv->surface_width;
147	u32 *cs = batch_alloc_items(state, 32, 8);
148	u32 offset = batch_offset(state, cs);
149
150#define SURFACE_2D 1
151#define SURFACEFORMAT_B8G8R8A8_UNORM 0x0C0
152#define RENDER_CACHE_READ_WRITE 1
153
154	*cs++ = SURFACE_2D << 29 |
155		(SURFACEFORMAT_B8G8R8A8_UNORM << 18) |
156		(RENDER_CACHE_READ_WRITE << 8);
157
158	*cs++ = batch_addr(state) + dst_offset;
159
160	*cs++ = ((surface_h / 4 - 1) << 16) | (surface_w / 4 - 1);
161	*cs++ = surface_w;
162	*cs++ = 0;
163	*cs++ = 0;
164	*cs++ = 0;
165#define SHADER_CHANNELS(r, g, b, a) \
166	(((r) << 25) | ((g) << 22) | ((b) << 19) | ((a) << 16))
167	*cs++ = SHADER_CHANNELS(4, 5, 6, 7);
168	batch_advance(state, cs);
169
170	return offset;
171}
172
173static u32
174gen7_fill_binding_table(struct batch_chunk *state,
175			const struct batch_vals *bv)
176{
177	u32 surface_start =
178		gen7_fill_surface_state(state, bv->surface_start, bv);
179	u32 *cs = batch_alloc_items(state, 32, 8);
180	u32 offset = batch_offset(state, cs);
181
182	*cs++ = surface_start - state->offset;
183	*cs++ = 0;
184	*cs++ = 0;
185	*cs++ = 0;
186	*cs++ = 0;
187	*cs++ = 0;
188	*cs++ = 0;
189	*cs++ = 0;
190	batch_advance(state, cs);
191
192	return offset;
193}
194
195static u32
196gen7_fill_kernel_data(struct batch_chunk *state,
197		      const u32 *data,
198		      const u32 size)
199{
200	return batch_offset(state,
201			    memcpy(batch_alloc_bytes(state, 64, size),
202				   data, size));
203}
204
205static u32
206gen7_fill_interface_descriptor(struct batch_chunk *state,
207			       const struct batch_vals *bv,
208			       const struct cb_kernel *kernel,
209			       unsigned int count)
210{
211	u32 kernel_offset =
212		gen7_fill_kernel_data(state, kernel->data, kernel->size);
213	u32 binding_table = gen7_fill_binding_table(state, bv);
214	u32 *cs = batch_alloc_items(state, 32, 8 * count);
215	u32 offset = batch_offset(state, cs);
216
217	*cs++ = kernel_offset;
218	*cs++ = (1 << 7) | (1 << 13);
219	*cs++ = 0;
220	*cs++ = (binding_table - state->offset) | 1;
221	*cs++ = 0;
222	*cs++ = 0;
223	*cs++ = 0;
224	*cs++ = 0;
225
226	/* 1 - 63dummy idds */
227	memset32(cs, 0x00, (count - 1) * 8);
228	batch_advance(state, cs + (count - 1) * 8);
229
230	return offset;
231}
232
233static void
234gen7_emit_state_base_address(struct batch_chunk *batch,
235			     u32 surface_state_base)
236{
237	u32 *cs = batch_alloc_items(batch, 0, 10);
238
239	*cs++ = STATE_BASE_ADDRESS | (10 - 2);
240	/* general */
241	*cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
242	/* surface */
243	*cs++ = (batch_addr(batch) + surface_state_base) | BASE_ADDRESS_MODIFY;
244	/* dynamic */
245	*cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
246	/* indirect */
247	*cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
248	/* instruction */
249	*cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY;
250
251	/* general/dynamic/indirect/instruction access Bound */
252	*cs++ = 0;
253	*cs++ = BASE_ADDRESS_MODIFY;
254	*cs++ = 0;
255	*cs++ = BASE_ADDRESS_MODIFY;
256	batch_advance(batch, cs);
257}
258
259static void
260gen7_emit_vfe_state(struct batch_chunk *batch,
261		    const struct batch_vals *bv,
262		    u32 urb_size, u32 curbe_size,
263		    u32 mode)
264{
265	u32 threads = bv->max_threads - 1;
266	u32 *cs = batch_alloc_items(batch, 32, 8);
267
268	*cs++ = MEDIA_VFE_STATE | (8 - 2);
269
270	/* scratch buffer */
271	*cs++ = 0;
272
273	/* number of threads & urb entries for GPGPU vs Media Mode */
274	*cs++ = threads << 16 | 1 << 8 | mode << 2;
275
276	*cs++ = 0;
277
278	/* urb entry size & curbe size in 256 bits unit */
279	*cs++ = urb_size << 16 | curbe_size;
280
281	/* scoreboard */
282	*cs++ = 0;
283	*cs++ = 0;
284	*cs++ = 0;
285	batch_advance(batch, cs);
286}
287
288static void
289gen7_emit_interface_descriptor_load(struct batch_chunk *batch,
290				    const u32 interface_descriptor,
291				    unsigned int count)
292{
293	u32 *cs = batch_alloc_items(batch, 8, 4);
294
295	*cs++ = MEDIA_INTERFACE_DESCRIPTOR_LOAD | (4 - 2);
296	*cs++ = 0;
297	*cs++ = count * 8 * sizeof(*cs);
298
299	/*
300	 * interface descriptor address - it is relative to the dynamics base
301	 * address
302	 */
303	*cs++ = interface_descriptor;
304	batch_advance(batch, cs);
305}
306
307static void
308gen7_emit_media_object(struct batch_chunk *batch,
309		       unsigned int media_object_index)
310{
311	unsigned int x_offset = (media_object_index % 16) * 64;
312	unsigned int y_offset = (media_object_index / 16) * 16;
313	unsigned int pkt = 6 + 3;
314	u32 *cs;
315
316	cs = batch_alloc_items(batch, 8, pkt);
317
318	*cs++ = MEDIA_OBJECT | (pkt - 2);
319
320	/* interface descriptor offset */
321	*cs++ = 0;
322
323	/* without indirect data */
324	*cs++ = 0;
325	*cs++ = 0;
326
327	/* scoreboard */
328	*cs++ = 0;
329	*cs++ = 0;
330
331	/* inline */
332	*cs++ = y_offset << 16 | x_offset;
333	*cs++ = 0;
334	*cs++ = GT3_INLINE_DATA_DELAYS;
335
336	batch_advance(batch, cs);
337}
338
339static void gen7_emit_pipeline_flush(struct batch_chunk *batch)
340{
341	u32 *cs = batch_alloc_items(batch, 0, 4);
342
343	*cs++ = GFX_OP_PIPE_CONTROL(4);
344	*cs++ = PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
345		PIPE_CONTROL_DEPTH_CACHE_FLUSH |
346		PIPE_CONTROL_DC_FLUSH_ENABLE |
347		PIPE_CONTROL_CS_STALL;
348	*cs++ = 0;
349	*cs++ = 0;
350
351	batch_advance(batch, cs);
352}
353
354static void gen7_emit_pipeline_invalidate(struct batch_chunk *batch)
355{
356	u32 *cs = batch_alloc_items(batch, 0, 10);
357
358	/* ivb: Stall before STATE_CACHE_INVALIDATE */
359	*cs++ = GFX_OP_PIPE_CONTROL(5);
360	*cs++ = PIPE_CONTROL_STALL_AT_SCOREBOARD |
361		PIPE_CONTROL_CS_STALL;
362	*cs++ = 0;
363	*cs++ = 0;
364	*cs++ = 0;
365
366	*cs++ = GFX_OP_PIPE_CONTROL(5);
367	*cs++ = PIPE_CONTROL_STATE_CACHE_INVALIDATE;
368	*cs++ = 0;
369	*cs++ = 0;
370	*cs++ = 0;
371
372	batch_advance(batch, cs);
373}
374
375static void emit_batch(struct i915_vma * const vma,
376		       u32 *start,
377		       const struct batch_vals *bv)
378{
379	struct drm_i915_private *i915 = vma->vm->i915;
380	const unsigned int desc_count = 1;
381	const unsigned int urb_size = 1;
382	struct batch_chunk cmds, state;
383	u32 descriptors;
384	unsigned int i;
385
386	batch_init(&cmds, vma, start, 0, bv->state_start);
387	batch_init(&state, vma, start, bv->state_start, SZ_4K);
388
389	descriptors = gen7_fill_interface_descriptor(&state, bv,
390						     IS_HASWELL(i915) ?
391						     &cb_kernel_hsw :
392						     &cb_kernel_ivb,
393						     desc_count);
394
395	/* Reset inherited context registers */
396	gen7_emit_pipeline_flush(&cmds);
397	gen7_emit_pipeline_invalidate(&cmds);
398	batch_add(&cmds, MI_LOAD_REGISTER_IMM(2));
399	batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_0_GEN7));
400	batch_add(&cmds, 0xffff0000 |
401			((IS_IVB_GT1(i915) || IS_VALLEYVIEW(i915)) ?
402			 HIZ_RAW_STALL_OPT_DISABLE :
403			 0));
404	batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_1));
405	batch_add(&cmds, 0xffff0000 | PIXEL_SUBSPAN_COLLECT_OPT_DISABLE);
406	gen7_emit_pipeline_invalidate(&cmds);
407	gen7_emit_pipeline_flush(&cmds);
408
409	/* Switch to the media pipeline and our base address */
410	gen7_emit_pipeline_invalidate(&cmds);
411	batch_add(&cmds, PIPELINE_SELECT | PIPELINE_SELECT_MEDIA);
412	batch_add(&cmds, MI_NOOP);
413	gen7_emit_pipeline_invalidate(&cmds);
414
415	gen7_emit_pipeline_flush(&cmds);
416	gen7_emit_state_base_address(&cmds, descriptors);
417	gen7_emit_pipeline_invalidate(&cmds);
418
419	/* Set the clear-residual kernel state */
420	gen7_emit_vfe_state(&cmds, bv, urb_size - 1, 0, 0);
421	gen7_emit_interface_descriptor_load(&cmds, descriptors, desc_count);
422
423	/* Execute the kernel on all HW threads */
424	for (i = 0; i < num_primitives(bv); i++)
425		gen7_emit_media_object(&cmds, i);
426
427	batch_add(&cmds, MI_BATCH_BUFFER_END);
428}
429
430int gen7_setup_clear_gpr_bb(struct intel_engine_cs * const engine,
431			    struct i915_vma * const vma)
432{
433	struct batch_vals bv;
434	u32 *batch;
435
436	batch_get_defaults(engine->i915, &bv);
437	if (!vma)
438		return bv.size;
439
440	GEM_BUG_ON(vma->obj->base.size < bv.size);
441
442	batch = i915_gem_object_pin_map(vma->obj, I915_MAP_WC);
443	if (IS_ERR(batch))
444		return PTR_ERR(batch);
445
446	emit_batch(vma, memset(batch, 0, bv.size), &bv);
447
448	i915_gem_object_flush_map(vma->obj);
449	__i915_gem_object_release_map(vma->obj);
450
451	return 0;
452}