1/*
  2 * Copyright (c) 2015 Intel Corporation
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions: *
 10 * The above copyright notice and this permission notice (including the next
 11 * paragraph) shall be included in all copies or substantial portions of the
 12 * Software.
 13 *
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 20 * SOFTWARE.
 21 */
 22
 23#include "i915_drv.h"
 24
 25#include "intel_engine.h"
 26#include "intel_gt.h"
 27#include "intel_mocs.h"
 28#include "intel_lrc.h"
 29#include "intel_ring.h"
 30
 31/* structures required */
 32struct drm_i915_mocs_entry {
 33	u32 control_value;
 34	u16 l3cc_value;
 35	u16 used;
 36};
 37
 38struct drm_i915_mocs_table {
 39	unsigned int size;
 40	unsigned int n_entries;
 41	const struct drm_i915_mocs_entry *table;
 42};
 43
 44/* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */
 45#define _LE_CACHEABILITY(value)	((value) << 0)
 46#define _LE_TGT_CACHE(value)	((value) << 2)
 47#define LE_LRUM(value)		((value) << 4)
 48#define LE_AOM(value)		((value) << 6)
 49#define LE_RSC(value)		((value) << 7)
 50#define LE_SCC(value)		((value) << 8)
 51#define LE_PFM(value)		((value) << 11)
 52#define LE_SCF(value)		((value) << 14)
 53#define LE_COS(value)		((value) << 15)
 54#define LE_SSE(value)		((value) << 17)
 55
 56/* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */
 57#define L3_ESC(value)		((value) << 0)
 58#define L3_SCC(value)		((value) << 1)
 59#define _L3_CACHEABILITY(value)	((value) << 4)
 60
 61/* Helper defines */
 62#define GEN9_NUM_MOCS_ENTRIES	62  /* 62 out of 64 - 63 & 64 are reserved. */
 63#define GEN11_NUM_MOCS_ENTRIES	64  /* 63-64 are reserved, but configured. */
 64
 65/* (e)LLC caching options */
 66/*
 67 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means
 68 * the same as LE_UC
 69 */
 70#define LE_0_PAGETABLE		_LE_CACHEABILITY(0)
 71#define LE_1_UC			_LE_CACHEABILITY(1)
 72#define LE_2_WT			_LE_CACHEABILITY(2)
 73#define LE_3_WB			_LE_CACHEABILITY(3)
 74
 75/* Target cache */
 76#define LE_TC_0_PAGETABLE	_LE_TGT_CACHE(0)
 77#define LE_TC_1_LLC		_LE_TGT_CACHE(1)
 78#define LE_TC_2_LLC_ELLC	_LE_TGT_CACHE(2)
 79#define LE_TC_3_LLC_ELLC_ALT	_LE_TGT_CACHE(3)
 80
 81/* L3 caching options */
 82#define L3_0_DIRECT		_L3_CACHEABILITY(0)
 83#define L3_1_UC			_L3_CACHEABILITY(1)
 84#define L3_2_RESERVED		_L3_CACHEABILITY(2)
 85#define L3_3_WB			_L3_CACHEABILITY(3)
 86
 87#define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \
 88	[__idx] = { \
 89		.control_value = __control_value, \
 90		.l3cc_value = __l3cc_value, \
 91		.used = 1, \
 92	}
 93
 94/*
 95 * MOCS tables
 96 *
 97 * These are the MOCS tables that are programmed across all the rings.
 98 * The control value is programmed to all the rings that support the
 99 * MOCS registers. While the l3cc_values are only programmed to the
100 * LNCFCMOCS0 - LNCFCMOCS32 registers.
101 *
102 * These tables are intended to be kept reasonably consistent across
103 * HW platforms, and for ICL+, be identical across OSes. To achieve
104 * that, for Icelake and above, list of entries is published as part
105 * of bspec.
106 *
107 * Entries not part of the following tables are undefined as far as
108 * userspace is concerned and shouldn't be relied upon.  For Gen < 12
109 * they will be initialized to PTE. Gen >= 12 onwards don't have a setting for
110 * PTE and will be initialized to an invalid value.
111 *
112 * The last two entries are reserved by the hardware. For ICL+ they
113 * should be initialized according to bspec and never used, for older
114 * platforms they should never be written to.
115 *
116 * NOTE: These tables are part of bspec and defined as part of hardware
117 *       interface for ICL+. For older platforms, they are part of kernel
118 *       ABI. It is expected that, for specific hardware platform, existing
119 *       entries will remain constant and the table will only be updated by
120 *       adding new entries, filling unused positions.
121 */
122#define GEN9_MOCS_ENTRIES \
123	MOCS_ENTRY(I915_MOCS_UNCACHED, \
124		   LE_1_UC | LE_TC_2_LLC_ELLC, \
125		   L3_1_UC), \
126	MOCS_ENTRY(I915_MOCS_PTE, \
127		   LE_0_PAGETABLE | LE_TC_2_LLC_ELLC | LE_LRUM(3), \
128		   L3_3_WB)
129
130static const struct drm_i915_mocs_entry skl_mocs_table[] = {
131	GEN9_MOCS_ENTRIES,
132	MOCS_ENTRY(I915_MOCS_CACHED,
133		   LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3),
134		   L3_3_WB)
135};
136
137/* NOTE: the LE_TGT_CACHE is not used on Broxton */
138static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
139	GEN9_MOCS_ENTRIES,
140	MOCS_ENTRY(I915_MOCS_CACHED,
141		   LE_1_UC | LE_TC_2_LLC_ELLC | LE_LRUM(3),
142		   L3_3_WB)
143};
144
145#define GEN11_MOCS_ENTRIES \
146	/* Entries 0 and 1 are defined per-platform */ \
147	/* Base - L3 + LLC */ \
148	MOCS_ENTRY(2, \
149		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
150		   L3_3_WB), \
151	/* Base - Uncached */ \
152	MOCS_ENTRY(3, \
153		   LE_1_UC | LE_TC_1_LLC, \
154		   L3_1_UC), \
155	/* Base - L3 */ \
156	MOCS_ENTRY(4, \
157		   LE_1_UC | LE_TC_1_LLC, \
158		   L3_3_WB), \
159	/* Base - LLC */ \
160	MOCS_ENTRY(5, \
161		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
162		   L3_1_UC), \
163	/* Age 0 - LLC */ \
164	MOCS_ENTRY(6, \
165		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
166		   L3_1_UC), \
167	/* Age 0 - L3 + LLC */ \
168	MOCS_ENTRY(7, \
169		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
170		   L3_3_WB), \
171	/* Age: Don't Chg. - LLC */ \
172	MOCS_ENTRY(8, \
173		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
174		   L3_1_UC), \
175	/* Age: Don't Chg. - L3 + LLC */ \
176	MOCS_ENTRY(9, \
177		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
178		   L3_3_WB), \
179	/* No AOM - LLC */ \
180	MOCS_ENTRY(10, \
181		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
182		   L3_1_UC), \
183	/* No AOM - L3 + LLC */ \
184	MOCS_ENTRY(11, \
185		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
186		   L3_3_WB), \
187	/* No AOM; Age 0 - LLC */ \
188	MOCS_ENTRY(12, \
189		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
190		   L3_1_UC), \
191	/* No AOM; Age 0 - L3 + LLC */ \
192	MOCS_ENTRY(13, \
193		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
194		   L3_3_WB), \
195	/* No AOM; Age:DC - LLC */ \
196	MOCS_ENTRY(14, \
197		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
198		   L3_1_UC), \
199	/* No AOM; Age:DC - L3 + LLC */ \
200	MOCS_ENTRY(15, \
201		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
202		   L3_3_WB), \
203	/* Self-Snoop - L3 + LLC */ \
204	MOCS_ENTRY(18, \
205		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \
206		   L3_3_WB), \
207	/* Skip Caching - L3 + LLC(12.5%) */ \
208	MOCS_ENTRY(19, \
209		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \
210		   L3_3_WB), \
211	/* Skip Caching - L3 + LLC(25%) */ \
212	MOCS_ENTRY(20, \
213		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \
214		   L3_3_WB), \
215	/* Skip Caching - L3 + LLC(50%) */ \
216	MOCS_ENTRY(21, \
217		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \
218		   L3_3_WB), \
219	/* Skip Caching - L3 + LLC(75%) */ \
220	MOCS_ENTRY(22, \
221		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \
222		   L3_3_WB), \
223	/* Skip Caching - L3 + LLC(87.5%) */ \
224	MOCS_ENTRY(23, \
225		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \
226		   L3_3_WB), \
227	/* HW Reserved - SW program but never use */ \
228	MOCS_ENTRY(62, \
229		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
230		   L3_1_UC), \
231	/* HW Reserved - SW program but never use */ \
232	MOCS_ENTRY(63, \
233		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
234		   L3_1_UC)
235
236static const struct drm_i915_mocs_entry tgl_mocs_table[] = {
237	/* Base - Error (Reserved for Non-Use) */
238	MOCS_ENTRY(0, 0x0, 0x0),
239	/* Base - Reserved */
240	MOCS_ENTRY(1, 0x0, 0x0),
241
242	GEN11_MOCS_ENTRIES,
243
244	/* Implicitly enable L1 - HDC:L1 + L3 + LLC */
245	MOCS_ENTRY(48,
246		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
247		   L3_3_WB),
248	/* Implicitly enable L1 - HDC:L1 + L3 */
249	MOCS_ENTRY(49,
250		   LE_1_UC | LE_TC_1_LLC,
251		   L3_3_WB),
252	/* Implicitly enable L1 - HDC:L1 + LLC */
253	MOCS_ENTRY(50,
254		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
255		   L3_1_UC),
256	/* Implicitly enable L1 - HDC:L1 */
257	MOCS_ENTRY(51,
258		   LE_1_UC | LE_TC_1_LLC,
259		   L3_1_UC),
260	/* HW Special Case (CCS) */
261	MOCS_ENTRY(60,
262		   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
263		   L3_1_UC),
264	/* HW Special Case (Displayable) */
265	MOCS_ENTRY(61,
266		   LE_1_UC | LE_TC_1_LLC,
267		   L3_3_WB),
268};
269
270static const struct drm_i915_mocs_entry icl_mocs_table[] = {
271	/* Base - Uncached (Deprecated) */
272	MOCS_ENTRY(I915_MOCS_UNCACHED,
273		   LE_1_UC | LE_TC_1_LLC,
274		   L3_1_UC),
275	/* Base - L3 + LeCC:PAT (Deprecated) */
276	MOCS_ENTRY(I915_MOCS_PTE,
277		   LE_0_PAGETABLE | LE_TC_1_LLC,
278		   L3_3_WB),
279
280	GEN11_MOCS_ENTRIES
281};
282
283enum {
284	HAS_GLOBAL_MOCS = BIT(0),
285	HAS_ENGINE_MOCS = BIT(1),
286	HAS_RENDER_L3CC = BIT(2),
287};
288
289static bool has_l3cc(const struct drm_i915_private *i915)
290{
291	return true;
292}
293
294static bool has_global_mocs(const struct drm_i915_private *i915)
295{
296	return HAS_GLOBAL_MOCS_REGISTERS(i915);
297}
298
299static bool has_mocs(const struct drm_i915_private *i915)
300{
301	return !IS_DGFX(i915);
302}
303
304static unsigned int get_mocs_settings(const struct drm_i915_private *i915,
305				      struct drm_i915_mocs_table *table)
306{
307	unsigned int flags;
308
309	if (INTEL_GEN(i915) >= 12) {
310		table->size  = ARRAY_SIZE(tgl_mocs_table);
311		table->table = tgl_mocs_table;
312		table->n_entries = GEN11_NUM_MOCS_ENTRIES;
313	} else if (IS_GEN(i915, 11)) {
314		table->size  = ARRAY_SIZE(icl_mocs_table);
315		table->table = icl_mocs_table;
316		table->n_entries = GEN11_NUM_MOCS_ENTRIES;
317	} else if (IS_GEN9_BC(i915) || IS_CANNONLAKE(i915)) {
318		table->size  = ARRAY_SIZE(skl_mocs_table);
319		table->n_entries = GEN9_NUM_MOCS_ENTRIES;
320		table->table = skl_mocs_table;
321	} else if (IS_GEN9_LP(i915)) {
322		table->size  = ARRAY_SIZE(broxton_mocs_table);
323		table->n_entries = GEN9_NUM_MOCS_ENTRIES;
324		table->table = broxton_mocs_table;
325	} else {
326		drm_WARN_ONCE(&i915->drm, INTEL_GEN(i915) >= 9,
327			      "Platform that should have a MOCS table does not.\n");
328		return 0;
329	}
330
331	if (GEM_DEBUG_WARN_ON(table->size > table->n_entries))
332		return 0;
333
334	/* WaDisableSkipCaching:skl,bxt,kbl,glk */
335	if (IS_GEN(i915, 9)) {
336		int i;
337
338		for (i = 0; i < table->size; i++)
339			if (GEM_DEBUG_WARN_ON(table->table[i].l3cc_value &
340					      (L3_ESC(1) | L3_SCC(0x7))))
341				return 0;
342	}
343
344	flags = 0;
345	if (has_mocs(i915)) {
346		if (has_global_mocs(i915))
347			flags |= HAS_GLOBAL_MOCS;
348		else
349			flags |= HAS_ENGINE_MOCS;
350	}
351	if (has_l3cc(i915))
352		flags |= HAS_RENDER_L3CC;
353
354	return flags;
355}
356
357/*
358 * Get control_value from MOCS entry taking into account when it's not used:
359 * I915_MOCS_PTE's value is returned in this case.
360 */
361static u32 get_entry_control(const struct drm_i915_mocs_table *table,
362			     unsigned int index)
363{
364	if (index < table->size && table->table[index].used)
365		return table->table[index].control_value;
366
367	return table->table[I915_MOCS_PTE].control_value;
368}
369
370#define for_each_mocs(mocs, t, i) \
371	for (i = 0; \
372	     i < (t)->n_entries ? (mocs = get_entry_control((t), i)), 1 : 0;\
373	     i++)
374
375static void __init_mocs_table(struct intel_uncore *uncore,
376			      const struct drm_i915_mocs_table *table,
377			      u32 addr)
378{
379	unsigned int i;
380	u32 mocs;
381
382	for_each_mocs(mocs, table, i)
383		intel_uncore_write_fw(uncore, _MMIO(addr + i * 4), mocs);
384}
385
386static u32 mocs_offset(const struct intel_engine_cs *engine)
387{
388	static const u32 offset[] = {
389		[RCS0]  =  __GEN9_RCS0_MOCS0,
390		[VCS0]  =  __GEN9_VCS0_MOCS0,
391		[VCS1]  =  __GEN9_VCS1_MOCS0,
392		[VECS0] =  __GEN9_VECS0_MOCS0,
393		[BCS0]  =  __GEN9_BCS0_MOCS0,
394		[VCS2]  = __GEN11_VCS2_MOCS0,
395	};
396
397	GEM_BUG_ON(engine->id >= ARRAY_SIZE(offset));
398	return offset[engine->id];
399}
400
401static void init_mocs_table(struct intel_engine_cs *engine,
402			    const struct drm_i915_mocs_table *table)
403{
404	__init_mocs_table(engine->uncore, table, mocs_offset(engine));
405}
406
407/*
408 * Get l3cc_value from MOCS entry taking into account when it's not used:
409 * I915_MOCS_PTE's value is returned in this case.
410 */
411static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table,
412			  unsigned int index)
413{
414	if (index < table->size && table->table[index].used)
415		return table->table[index].l3cc_value;
416
417	return table->table[I915_MOCS_PTE].l3cc_value;
418}
419
420static inline u32 l3cc_combine(u16 low, u16 high)
421{
422	return low | (u32)high << 16;
423}
424
425#define for_each_l3cc(l3cc, t, i) \
426	for (i = 0; \
427	     i < ((t)->n_entries + 1) / 2 ? \
428	     (l3cc = l3cc_combine(get_entry_l3cc((t), 2 * i), \
429				  get_entry_l3cc((t), 2 * i + 1))), 1 : \
430	     0; \
431	     i++)
432
433static void init_l3cc_table(struct intel_engine_cs *engine,
434			    const struct drm_i915_mocs_table *table)
435{
436	struct intel_uncore *uncore = engine->uncore;
437	unsigned int i;
438	u32 l3cc;
439
440	for_each_l3cc(l3cc, table, i)
441		intel_uncore_write_fw(uncore, GEN9_LNCFCMOCS(i), l3cc);
442}
443
444void intel_mocs_init_engine(struct intel_engine_cs *engine)
445{
446	struct drm_i915_mocs_table table;
447	unsigned int flags;
448
449	/* Called under a blanket forcewake */
450	assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
451
452	flags = get_mocs_settings(engine->i915, &table);
453	if (!flags)
454		return;
455
456	/* Platforms with global MOCS do not need per-engine initialization. */
457	if (flags & HAS_ENGINE_MOCS)
458		init_mocs_table(engine, &table);
459
460	if (flags & HAS_RENDER_L3CC && engine->class == RENDER_CLASS)
461		init_l3cc_table(engine, &table);
462}
463
464static u32 global_mocs_offset(void)
465{
466	return i915_mmio_reg_offset(GEN12_GLOBAL_MOCS(0));
467}
468
469void intel_mocs_init(struct intel_gt *gt)
470{
471	struct drm_i915_mocs_table table;
472	unsigned int flags;
473
474	/*
475	 * LLC and eDRAM control values are not applicable to dgfx
476	 */
477	flags = get_mocs_settings(gt->i915, &table);
478	if (flags & HAS_GLOBAL_MOCS)
479		__init_mocs_table(gt->uncore, &table, global_mocs_offset());
480}
481
482#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
483#include "selftest_mocs.c"
484#endif