Loading...
Note: File does not exist in v6.2.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2023 Intel Corporation
4 */
5
6#include "xe_pat.h"
7
8#include <uapi/drm/xe_drm.h>
9
10#include <generated/xe_wa_oob.h>
11
12#include "regs/xe_reg_defs.h"
13#include "xe_assert.h"
14#include "xe_device.h"
15#include "xe_force_wake.h"
16#include "xe_gt.h"
17#include "xe_gt_mcr.h"
18#include "xe_mmio.h"
19#include "xe_sriov.h"
20#include "xe_wa.h"
21
22#define _PAT_ATS 0x47fc
23#define _PAT_INDEX(index) _PICK_EVEN_2RANGES(index, 8, \
24 0x4800, 0x4804, \
25 0x4848, 0x484c)
26#define _PAT_PTA 0x4820
27
28#define XE2_NO_PROMOTE REG_BIT(10)
29#define XE2_COMP_EN REG_BIT(9)
30#define XE2_L3_CLOS REG_GENMASK(7, 6)
31#define XE2_L3_POLICY REG_GENMASK(5, 4)
32#define XE2_L4_POLICY REG_GENMASK(3, 2)
33#define XE2_COH_MODE REG_GENMASK(1, 0)
34
35#define XELPG_L4_POLICY_MASK REG_GENMASK(3, 2)
36#define XELPG_PAT_3_UC REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 3)
37#define XELPG_PAT_1_WT REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 1)
38#define XELPG_PAT_0_WB REG_FIELD_PREP(XELPG_L4_POLICY_MASK, 0)
39#define XELPG_INDEX_COH_MODE_MASK REG_GENMASK(1, 0)
40#define XELPG_3_COH_2W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 3)
41#define XELPG_2_COH_1W REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 2)
42#define XELPG_0_COH_NON REG_FIELD_PREP(XELPG_INDEX_COH_MODE_MASK, 0)
43
44#define XEHPC_CLOS_LEVEL_MASK REG_GENMASK(3, 2)
45#define XEHPC_PAT_CLOS(x) REG_FIELD_PREP(XEHPC_CLOS_LEVEL_MASK, x)
46
47#define XELP_MEM_TYPE_MASK REG_GENMASK(1, 0)
48#define XELP_PAT_WB REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 3)
49#define XELP_PAT_WT REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 2)
50#define XELP_PAT_WC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 1)
51#define XELP_PAT_UC REG_FIELD_PREP(XELP_MEM_TYPE_MASK, 0)
52
53static const char *XELP_MEM_TYPE_STR_MAP[] = { "UC", "WC", "WT", "WB" };
54
55struct xe_pat_ops {
56 void (*program_graphics)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
57 int n_entries);
58 void (*program_media)(struct xe_gt *gt, const struct xe_pat_table_entry table[],
59 int n_entries);
60 void (*dump)(struct xe_gt *gt, struct drm_printer *p);
61};
62
63static const struct xe_pat_table_entry xelp_pat_table[] = {
64 [0] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
65 [1] = { XELP_PAT_WC, XE_COH_NONE },
66 [2] = { XELP_PAT_WT, XE_COH_NONE },
67 [3] = { XELP_PAT_UC, XE_COH_NONE },
68};
69
70static const struct xe_pat_table_entry xehpc_pat_table[] = {
71 [0] = { XELP_PAT_UC, XE_COH_NONE },
72 [1] = { XELP_PAT_WC, XE_COH_NONE },
73 [2] = { XELP_PAT_WT, XE_COH_NONE },
74 [3] = { XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
75 [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE },
76 [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
77 [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE },
78 [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_AT_LEAST_1WAY },
79};
80
81static const struct xe_pat_table_entry xelpg_pat_table[] = {
82 [0] = { XELPG_PAT_0_WB, XE_COH_NONE },
83 [1] = { XELPG_PAT_1_WT, XE_COH_NONE },
84 [2] = { XELPG_PAT_3_UC, XE_COH_NONE },
85 [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_AT_LEAST_1WAY },
86 [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_AT_LEAST_1WAY },
87};
88
89/*
90 * The Xe2 table is getting large/complicated so it's easier to review if
91 * provided in a form that exactly matches the bspec's formatting. The meaning
92 * of the fields here are:
93 * - no_promote: 0=promotable, 1=no promote
94 * - comp_en: 0=disable, 1=enable
95 * - l3clos: L3 class of service (0-3)
96 * - l3_policy: 0=WB, 1=XD ("WB - Transient Display"), 3=UC
97 * - l4_policy: 0=WB, 1=WT, 3=UC
98 * - coh_mode: 0=no snoop, 2=1-way coherent, 3=2-way coherent
99 *
100 * Reserved entries should be programmed with the maximum caching, minimum
101 * coherency (which matches an all-0's encoding), so we can just omit them
102 * in the table.
103 *
104 * Note: There is an implicit assumption in the driver that compression and
105 * coh_1way+ are mutually exclusive. If this is ever not true then userptr
106 * and imported dma-buf from external device will have uncleared ccs state.
107 */
108#define XE2_PAT(no_promote, comp_en, l3clos, l3_policy, l4_policy, __coh_mode) \
109 { \
110 .value = (no_promote ? XE2_NO_PROMOTE : 0) | \
111 (comp_en ? XE2_COMP_EN : 0) | \
112 REG_FIELD_PREP(XE2_L3_CLOS, l3clos) | \
113 REG_FIELD_PREP(XE2_L3_POLICY, l3_policy) | \
114 REG_FIELD_PREP(XE2_L4_POLICY, l4_policy) | \
115 REG_FIELD_PREP(XE2_COH_MODE, __coh_mode), \
116 .coh_mode = (BUILD_BUG_ON_ZERO(__coh_mode && comp_en) || __coh_mode) ? \
117 XE_COH_AT_LEAST_1WAY : XE_COH_NONE \
118 }
119
120static const struct xe_pat_table_entry xe2_pat_table[] = {
121 [ 0] = XE2_PAT( 0, 0, 0, 0, 3, 0 ),
122 [ 1] = XE2_PAT( 0, 0, 0, 0, 3, 2 ),
123 [ 2] = XE2_PAT( 0, 0, 0, 0, 3, 3 ),
124 [ 3] = XE2_PAT( 0, 0, 0, 3, 3, 0 ),
125 [ 4] = XE2_PAT( 0, 0, 0, 3, 0, 2 ),
126 [ 5] = XE2_PAT( 0, 0, 0, 3, 3, 2 ),
127 [ 6] = XE2_PAT( 1, 0, 0, 1, 3, 0 ),
128 [ 7] = XE2_PAT( 0, 0, 0, 3, 0, 3 ),
129 [ 8] = XE2_PAT( 0, 0, 0, 3, 0, 0 ),
130 [ 9] = XE2_PAT( 0, 1, 0, 0, 3, 0 ),
131 [10] = XE2_PAT( 0, 1, 0, 3, 0, 0 ),
132 [11] = XE2_PAT( 1, 1, 0, 1, 3, 0 ),
133 [12] = XE2_PAT( 0, 1, 0, 3, 3, 0 ),
134 [13] = XE2_PAT( 0, 0, 0, 0, 0, 0 ),
135 [14] = XE2_PAT( 0, 1, 0, 0, 0, 0 ),
136 [15] = XE2_PAT( 1, 1, 0, 1, 1, 0 ),
137 /* 16..19 are reserved; leave set to all 0's */
138 [20] = XE2_PAT( 0, 0, 1, 0, 3, 0 ),
139 [21] = XE2_PAT( 0, 1, 1, 0, 3, 0 ),
140 [22] = XE2_PAT( 0, 0, 1, 0, 3, 2 ),
141 [23] = XE2_PAT( 0, 0, 1, 0, 3, 3 ),
142 [24] = XE2_PAT( 0, 0, 2, 0, 3, 0 ),
143 [25] = XE2_PAT( 0, 1, 2, 0, 3, 0 ),
144 [26] = XE2_PAT( 0, 0, 2, 0, 3, 2 ),
145 [27] = XE2_PAT( 0, 0, 2, 0, 3, 3 ),
146 [28] = XE2_PAT( 0, 0, 3, 0, 3, 0 ),
147 [29] = XE2_PAT( 0, 1, 3, 0, 3, 0 ),
148 [30] = XE2_PAT( 0, 0, 3, 0, 3, 2 ),
149 [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ),
150};
151
152/* Special PAT values programmed outside the main table */
153static const struct xe_pat_table_entry xe2_pat_ats = XE2_PAT( 0, 0, 0, 0, 3, 3 );
154static const struct xe_pat_table_entry xe2_pat_pta = XE2_PAT( 0, 0, 0, 0, 3, 0 );
155
156u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index)
157{
158 WARN_ON(pat_index >= xe->pat.n_entries);
159 return xe->pat.table[pat_index].coh_mode;
160}
161
162static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
163 int n_entries)
164{
165 for (int i = 0; i < n_entries; i++) {
166 struct xe_reg reg = XE_REG(_PAT_INDEX(i));
167
168 xe_mmio_write32(>->mmio, reg, table[i].value);
169 }
170}
171
172static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry table[],
173 int n_entries)
174{
175 for (int i = 0; i < n_entries; i++) {
176 struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i));
177
178 xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value);
179 }
180}
181
182static void xelp_dump(struct xe_gt *gt, struct drm_printer *p)
183{
184 struct xe_device *xe = gt_to_xe(gt);
185 unsigned int fw_ref;
186 int i;
187
188 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
189 if (!fw_ref)
190 return;
191
192 drm_printf(p, "PAT table:\n");
193
194 for (i = 0; i < xe->pat.n_entries; i++) {
195 u32 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i)));
196 u8 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
197
198 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
199 XELP_MEM_TYPE_STR_MAP[mem_type], pat);
200 }
201
202 xe_force_wake_put(gt_to_fw(gt), fw_ref);
203}
204
205static const struct xe_pat_ops xelp_pat_ops = {
206 .program_graphics = program_pat,
207 .dump = xelp_dump,
208};
209
210static void xehp_dump(struct xe_gt *gt, struct drm_printer *p)
211{
212 struct xe_device *xe = gt_to_xe(gt);
213 unsigned int fw_ref;
214 int i;
215
216 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
217 if (!fw_ref)
218 return;
219
220 drm_printf(p, "PAT table:\n");
221
222 for (i = 0; i < xe->pat.n_entries; i++) {
223 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
224 u8 mem_type;
225
226 mem_type = REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat);
227
228 drm_printf(p, "PAT[%2d] = %s (%#8x)\n", i,
229 XELP_MEM_TYPE_STR_MAP[mem_type], pat);
230 }
231
232 xe_force_wake_put(gt_to_fw(gt), fw_ref);
233}
234
235static const struct xe_pat_ops xehp_pat_ops = {
236 .program_graphics = program_pat_mcr,
237 .dump = xehp_dump,
238};
239
240static void xehpc_dump(struct xe_gt *gt, struct drm_printer *p)
241{
242 struct xe_device *xe = gt_to_xe(gt);
243 unsigned int fw_ref;
244 int i;
245
246 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
247 if (!fw_ref)
248 return;
249
250 drm_printf(p, "PAT table:\n");
251
252 for (i = 0; i < xe->pat.n_entries; i++) {
253 u32 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
254
255 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
256 REG_FIELD_GET(XELP_MEM_TYPE_MASK, pat),
257 REG_FIELD_GET(XEHPC_CLOS_LEVEL_MASK, pat), pat);
258 }
259
260 xe_force_wake_put(gt_to_fw(gt), fw_ref);
261}
262
263static const struct xe_pat_ops xehpc_pat_ops = {
264 .program_graphics = program_pat_mcr,
265 .dump = xehpc_dump,
266};
267
268static void xelpg_dump(struct xe_gt *gt, struct drm_printer *p)
269{
270 struct xe_device *xe = gt_to_xe(gt);
271 unsigned int fw_ref;
272 int i;
273
274 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
275 if (!fw_ref)
276 return;
277
278 drm_printf(p, "PAT table:\n");
279
280 for (i = 0; i < xe->pat.n_entries; i++) {
281 u32 pat;
282
283 if (xe_gt_is_media_type(gt))
284 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i)));
285 else
286 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
287
288 drm_printf(p, "PAT[%2d] = [ %u, %u ] (%#8x)\n", i,
289 REG_FIELD_GET(XELPG_L4_POLICY_MASK, pat),
290 REG_FIELD_GET(XELPG_INDEX_COH_MODE_MASK, pat), pat);
291 }
292
293 xe_force_wake_put(gt_to_fw(gt), fw_ref);
294}
295
296/*
297 * SAMedia register offsets are adjusted by the write methods and they target
298 * registers that are not MCR, while for normal GT they are MCR
299 */
300static const struct xe_pat_ops xelpg_pat_ops = {
301 .program_graphics = program_pat,
302 .program_media = program_pat_mcr,
303 .dump = xelpg_dump,
304};
305
306static void xe2lpg_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
307 int n_entries)
308{
309 program_pat_mcr(gt, table, n_entries);
310 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe2_pat_ats.value);
311
312 if (IS_DGFX(gt_to_xe(gt)))
313 xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), xe2_pat_pta.value);
314}
315
316static void xe2lpm_program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[],
317 int n_entries)
318{
319 program_pat(gt, table, n_entries);
320 xe_mmio_write32(>->mmio, XE_REG(_PAT_ATS), xe2_pat_ats.value);
321
322 if (IS_DGFX(gt_to_xe(gt)))
323 xe_mmio_write32(>->mmio, XE_REG(_PAT_PTA), xe2_pat_pta.value);
324}
325
326static void xe2_dump(struct xe_gt *gt, struct drm_printer *p)
327{
328 struct xe_device *xe = gt_to_xe(gt);
329 unsigned int fw_ref;
330 u32 pat;
331 int i;
332
333 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
334 if (!fw_ref)
335 return;
336
337 drm_printf(p, "PAT table:\n");
338
339 for (i = 0; i < xe->pat.n_entries; i++) {
340 if (xe_gt_is_media_type(gt))
341 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i)));
342 else
343 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i)));
344
345 drm_printf(p, "PAT[%2d] = [ %u, %u, %u, %u, %u, %u ] (%#8x)\n", i,
346 !!(pat & XE2_NO_PROMOTE),
347 !!(pat & XE2_COMP_EN),
348 REG_FIELD_GET(XE2_L3_CLOS, pat),
349 REG_FIELD_GET(XE2_L3_POLICY, pat),
350 REG_FIELD_GET(XE2_L4_POLICY, pat),
351 REG_FIELD_GET(XE2_COH_MODE, pat),
352 pat);
353 }
354
355 /*
356 * Also print PTA_MODE, which describes how the hardware accesses
357 * PPGTT entries.
358 */
359 if (xe_gt_is_media_type(gt))
360 pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_PTA));
361 else
362 pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_PTA));
363
364 drm_printf(p, "Page Table Access:\n");
365 drm_printf(p, "PTA_MODE= [ %u, %u, %u, %u, %u, %u ] (%#8x)\n",
366 !!(pat & XE2_NO_PROMOTE),
367 !!(pat & XE2_COMP_EN),
368 REG_FIELD_GET(XE2_L3_CLOS, pat),
369 REG_FIELD_GET(XE2_L3_POLICY, pat),
370 REG_FIELD_GET(XE2_L4_POLICY, pat),
371 REG_FIELD_GET(XE2_COH_MODE, pat),
372 pat);
373
374 xe_force_wake_put(gt_to_fw(gt), fw_ref);
375}
376
377static const struct xe_pat_ops xe2_pat_ops = {
378 .program_graphics = xe2lpg_program_pat,
379 .program_media = xe2lpm_program_pat,
380 .dump = xe2_dump,
381};
382
383void xe_pat_init_early(struct xe_device *xe)
384{
385 if (GRAPHICS_VER(xe) == 30 || GRAPHICS_VER(xe) == 20) {
386 xe->pat.ops = &xe2_pat_ops;
387 xe->pat.table = xe2_pat_table;
388
389 /* Wa_16023588340. XXX: Should use XE_WA */
390 if (GRAPHICS_VERx100(xe) == 2001)
391 xe->pat.n_entries = 28; /* Disable CLOS3 */
392 else
393 xe->pat.n_entries = ARRAY_SIZE(xe2_pat_table);
394
395 xe->pat.idx[XE_CACHE_NONE] = 3;
396 xe->pat.idx[XE_CACHE_WT] = 15;
397 xe->pat.idx[XE_CACHE_WB] = 2;
398 xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = 12; /*Applicable on xe2 and beyond */
399 } else if (xe->info.platform == XE_METEORLAKE) {
400 xe->pat.ops = &xelpg_pat_ops;
401 xe->pat.table = xelpg_pat_table;
402 xe->pat.n_entries = ARRAY_SIZE(xelpg_pat_table);
403 xe->pat.idx[XE_CACHE_NONE] = 2;
404 xe->pat.idx[XE_CACHE_WT] = 1;
405 xe->pat.idx[XE_CACHE_WB] = 3;
406 } else if (xe->info.platform == XE_PVC) {
407 xe->pat.ops = &xehpc_pat_ops;
408 xe->pat.table = xehpc_pat_table;
409 xe->pat.n_entries = ARRAY_SIZE(xehpc_pat_table);
410 xe->pat.idx[XE_CACHE_NONE] = 0;
411 xe->pat.idx[XE_CACHE_WT] = 2;
412 xe->pat.idx[XE_CACHE_WB] = 3;
413 } else if (xe->info.platform == XE_DG2) {
414 /*
415 * Table is the same as previous platforms, but programming
416 * method has changed.
417 */
418 xe->pat.ops = &xehp_pat_ops;
419 xe->pat.table = xelp_pat_table;
420 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table);
421 xe->pat.idx[XE_CACHE_NONE] = 3;
422 xe->pat.idx[XE_CACHE_WT] = 2;
423 xe->pat.idx[XE_CACHE_WB] = 0;
424 } else if (GRAPHICS_VERx100(xe) <= 1210) {
425 WARN_ON_ONCE(!IS_DGFX(xe) && !xe->info.has_llc);
426 xe->pat.ops = &xelp_pat_ops;
427 xe->pat.table = xelp_pat_table;
428 xe->pat.n_entries = ARRAY_SIZE(xelp_pat_table);
429 xe->pat.idx[XE_CACHE_NONE] = 3;
430 xe->pat.idx[XE_CACHE_WT] = 2;
431 xe->pat.idx[XE_CACHE_WB] = 0;
432 } else {
433 /*
434 * Going forward we expect to need new PAT settings for most
435 * new platforms; failure to provide a new table can easily
436 * lead to subtle, hard-to-debug problems. If none of the
437 * conditions above match the platform we're running on we'll
438 * raise an error rather than trying to silently inherit the
439 * most recent platform's behavior.
440 */
441 drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n",
442 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100);
443 }
444
445 /* VFs can't program nor dump PAT settings */
446 if (IS_SRIOV_VF(xe))
447 xe->pat.ops = NULL;
448
449 xe_assert(xe, !xe->pat.ops || xe->pat.ops->dump);
450 xe_assert(xe, !xe->pat.ops || xe->pat.ops->program_graphics);
451 xe_assert(xe, !xe->pat.ops || MEDIA_VER(xe) < 13 || xe->pat.ops->program_media);
452}
453
454void xe_pat_init(struct xe_gt *gt)
455{
456 struct xe_device *xe = gt_to_xe(gt);
457
458 if (!xe->pat.ops)
459 return;
460
461 if (xe_gt_is_media_type(gt))
462 xe->pat.ops->program_media(gt, xe->pat.table, xe->pat.n_entries);
463 else
464 xe->pat.ops->program_graphics(gt, xe->pat.table, xe->pat.n_entries);
465}
466
467void xe_pat_dump(struct xe_gt *gt, struct drm_printer *p)
468{
469 struct xe_device *xe = gt_to_xe(gt);
470
471 if (!xe->pat.ops)
472 return;
473
474 xe->pat.ops->dump(gt, p);
475}