Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_query.h"
7
8#include <linux/nospec.h>
9#include <linux/sched/clock.h>
10
11#include <drm/ttm/ttm_placement.h>
12#include <drm/xe_drm.h>
13
14#include "regs/xe_engine_regs.h"
15#include "xe_bo.h"
16#include "xe_device.h"
17#include "xe_exec_queue.h"
18#include "xe_ggtt.h"
19#include "xe_gt.h"
20#include "xe_guc_hwconfig.h"
21#include "xe_macros.h"
22#include "xe_mmio.h"
23#include "xe_ttm_vram_mgr.h"
24
25static const u16 xe_to_user_engine_class[] = {
26 [XE_ENGINE_CLASS_RENDER] = DRM_XE_ENGINE_CLASS_RENDER,
27 [XE_ENGINE_CLASS_COPY] = DRM_XE_ENGINE_CLASS_COPY,
28 [XE_ENGINE_CLASS_VIDEO_DECODE] = DRM_XE_ENGINE_CLASS_VIDEO_DECODE,
29 [XE_ENGINE_CLASS_VIDEO_ENHANCE] = DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE,
30 [XE_ENGINE_CLASS_COMPUTE] = DRM_XE_ENGINE_CLASS_COMPUTE,
31};
32
33static const enum xe_engine_class user_to_xe_engine_class[] = {
34 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER,
35 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY,
36 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE,
37 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE,
38 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE,
39};
40
41static size_t calc_hw_engine_info_size(struct xe_device *xe)
42{
43 struct xe_hw_engine *hwe;
44 enum xe_hw_engine_id id;
45 struct xe_gt *gt;
46 u8 gt_id;
47 int i = 0;
48
49 for_each_gt(gt, xe, gt_id)
50 for_each_hw_engine(hwe, gt, id) {
51 if (xe_hw_engine_is_reserved(hwe))
52 continue;
53 i++;
54 }
55
56 return sizeof(struct drm_xe_query_engines) +
57 i * sizeof(struct drm_xe_engine);
58}
59
60typedef u64 (*__ktime_func_t)(void);
61static __ktime_func_t __clock_id_to_func(clockid_t clk_id)
62{
63 /*
64 * Use logic same as the perf subsystem to allow user to select the
65 * reference clock id to be used for timestamps.
66 */
67 switch (clk_id) {
68 case CLOCK_MONOTONIC:
69 return &ktime_get_ns;
70 case CLOCK_MONOTONIC_RAW:
71 return &ktime_get_raw_ns;
72 case CLOCK_REALTIME:
73 return &ktime_get_real_ns;
74 case CLOCK_BOOTTIME:
75 return &ktime_get_boottime_ns;
76 case CLOCK_TAI:
77 return &ktime_get_clocktai_ns;
78 default:
79 return NULL;
80 }
81}
82
83static void
84__read_timestamps(struct xe_gt *gt,
85 struct xe_reg lower_reg,
86 struct xe_reg upper_reg,
87 u64 *engine_ts,
88 u64 *cpu_ts,
89 u64 *cpu_delta,
90 __ktime_func_t cpu_clock)
91{
92 u32 upper, lower, old_upper, loop = 0;
93
94 upper = xe_mmio_read32(gt, upper_reg);
95 do {
96 *cpu_delta = local_clock();
97 *cpu_ts = cpu_clock();
98 lower = xe_mmio_read32(gt, lower_reg);
99 *cpu_delta = local_clock() - *cpu_delta;
100 old_upper = upper;
101 upper = xe_mmio_read32(gt, upper_reg);
102 } while (upper != old_upper && loop++ < 2);
103
104 *engine_ts = (u64)upper << 32 | lower;
105}
106
107static int
108query_engine_cycles(struct xe_device *xe,
109 struct drm_xe_device_query *query)
110{
111 struct drm_xe_query_engine_cycles __user *query_ptr;
112 struct drm_xe_engine_class_instance *eci;
113 struct drm_xe_query_engine_cycles resp;
114 size_t size = sizeof(resp);
115 __ktime_func_t cpu_clock;
116 struct xe_hw_engine *hwe;
117 struct xe_gt *gt;
118
119 if (query->size == 0) {
120 query->size = size;
121 return 0;
122 } else if (XE_IOCTL_DBG(xe, query->size != size)) {
123 return -EINVAL;
124 }
125
126 query_ptr = u64_to_user_ptr(query->data);
127 if (copy_from_user(&resp, query_ptr, size))
128 return -EFAULT;
129
130 cpu_clock = __clock_id_to_func(resp.clockid);
131 if (!cpu_clock)
132 return -EINVAL;
133
134 eci = &resp.eci;
135 if (eci->gt_id >= XE_MAX_GT_PER_TILE)
136 return -EINVAL;
137
138 gt = xe_device_get_gt(xe, eci->gt_id);
139 if (!gt)
140 return -EINVAL;
141
142 if (eci->engine_class >= ARRAY_SIZE(user_to_xe_engine_class))
143 return -EINVAL;
144
145 hwe = xe_gt_hw_engine(gt, user_to_xe_engine_class[eci->engine_class],
146 eci->engine_instance, true);
147 if (!hwe)
148 return -EINVAL;
149
150 xe_device_mem_access_get(xe);
151 xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
152
153 __read_timestamps(gt,
154 RING_TIMESTAMP(hwe->mmio_base),
155 RING_TIMESTAMP_UDW(hwe->mmio_base),
156 &resp.engine_cycles,
157 &resp.cpu_timestamp,
158 &resp.cpu_delta,
159 cpu_clock);
160
161 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
162 xe_device_mem_access_put(xe);
163 resp.width = 36;
164
165 /* Only write to the output fields of user query */
166 if (put_user(resp.cpu_timestamp, &query_ptr->cpu_timestamp))
167 return -EFAULT;
168
169 if (put_user(resp.cpu_delta, &query_ptr->cpu_delta))
170 return -EFAULT;
171
172 if (put_user(resp.engine_cycles, &query_ptr->engine_cycles))
173 return -EFAULT;
174
175 if (put_user(resp.width, &query_ptr->width))
176 return -EFAULT;
177
178 return 0;
179}
180
181static int query_engines(struct xe_device *xe,
182 struct drm_xe_device_query *query)
183{
184 size_t size = calc_hw_engine_info_size(xe);
185 struct drm_xe_query_engines __user *query_ptr =
186 u64_to_user_ptr(query->data);
187 struct drm_xe_query_engines *engines;
188 struct xe_hw_engine *hwe;
189 enum xe_hw_engine_id id;
190 struct xe_gt *gt;
191 u8 gt_id;
192 int i = 0;
193
194 if (query->size == 0) {
195 query->size = size;
196 return 0;
197 } else if (XE_IOCTL_DBG(xe, query->size != size)) {
198 return -EINVAL;
199 }
200
201 engines = kzalloc(size, GFP_KERNEL);
202 if (!engines)
203 return -ENOMEM;
204
205 for_each_gt(gt, xe, gt_id)
206 for_each_hw_engine(hwe, gt, id) {
207 if (xe_hw_engine_is_reserved(hwe))
208 continue;
209
210 engines->engines[i].instance.engine_class =
211 xe_to_user_engine_class[hwe->class];
212 engines->engines[i].instance.engine_instance =
213 hwe->logical_instance;
214 engines->engines[i].instance.gt_id = gt->info.id;
215
216 i++;
217 }
218
219 engines->num_engines = i;
220
221 if (copy_to_user(query_ptr, engines, size)) {
222 kfree(engines);
223 return -EFAULT;
224 }
225 kfree(engines);
226
227 return 0;
228}
229
230static size_t calc_mem_regions_size(struct xe_device *xe)
231{
232 u32 num_managers = 1;
233 int i;
234
235 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i)
236 if (ttm_manager_type(&xe->ttm, i))
237 num_managers++;
238
239 return offsetof(struct drm_xe_query_mem_regions, mem_regions[num_managers]);
240}
241
242static int query_mem_regions(struct xe_device *xe,
243 struct drm_xe_device_query *query)
244{
245 size_t size = calc_mem_regions_size(xe);
246 struct drm_xe_query_mem_regions *mem_regions;
247 struct drm_xe_query_mem_regions __user *query_ptr =
248 u64_to_user_ptr(query->data);
249 struct ttm_resource_manager *man;
250 int ret, i;
251
252 if (query->size == 0) {
253 query->size = size;
254 return 0;
255 } else if (XE_IOCTL_DBG(xe, query->size != size)) {
256 return -EINVAL;
257 }
258
259 mem_regions = kzalloc(size, GFP_KERNEL);
260 if (XE_IOCTL_DBG(xe, !mem_regions))
261 return -ENOMEM;
262
263 man = ttm_manager_type(&xe->ttm, XE_PL_TT);
264 mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM;
265 /*
266 * The instance needs to be a unique number that represents the index
267 * in the placement mask used at xe_gem_create_ioctl() for the
268 * xe_bo_create() placement.
269 */
270 mem_regions->mem_regions[0].instance = 0;
271 mem_regions->mem_regions[0].min_page_size = PAGE_SIZE;
272 mem_regions->mem_regions[0].total_size = man->size << PAGE_SHIFT;
273 if (perfmon_capable())
274 mem_regions->mem_regions[0].used = ttm_resource_manager_usage(man);
275 mem_regions->num_mem_regions = 1;
276
277 for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) {
278 man = ttm_manager_type(&xe->ttm, i);
279 if (man) {
280 mem_regions->mem_regions[mem_regions->num_mem_regions].mem_class =
281 DRM_XE_MEM_REGION_CLASS_VRAM;
282 mem_regions->mem_regions[mem_regions->num_mem_regions].instance =
283 mem_regions->num_mem_regions;
284 mem_regions->mem_regions[mem_regions->num_mem_regions].min_page_size =
285 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ?
286 SZ_64K : PAGE_SIZE;
287 mem_regions->mem_regions[mem_regions->num_mem_regions].total_size =
288 man->size;
289
290 if (perfmon_capable()) {
291 xe_ttm_vram_get_used(man,
292 &mem_regions->mem_regions
293 [mem_regions->num_mem_regions].used,
294 &mem_regions->mem_regions
295 [mem_regions->num_mem_regions].cpu_visible_used);
296 }
297
298 mem_regions->mem_regions[mem_regions->num_mem_regions].cpu_visible_size =
299 xe_ttm_vram_get_cpu_visible_size(man);
300 mem_regions->num_mem_regions++;
301 }
302 }
303
304 if (!copy_to_user(query_ptr, mem_regions, size))
305 ret = 0;
306 else
307 ret = -ENOSPC;
308
309 kfree(mem_regions);
310 return ret;
311}
312
313static int query_config(struct xe_device *xe, struct drm_xe_device_query *query)
314{
315 const u32 num_params = DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1;
316 size_t size =
317 sizeof(struct drm_xe_query_config) + num_params * sizeof(u64);
318 struct drm_xe_query_config __user *query_ptr =
319 u64_to_user_ptr(query->data);
320 struct drm_xe_query_config *config;
321
322 if (query->size == 0) {
323 query->size = size;
324 return 0;
325 } else if (XE_IOCTL_DBG(xe, query->size != size)) {
326 return -EINVAL;
327 }
328
329 config = kzalloc(size, GFP_KERNEL);
330 if (!config)
331 return -ENOMEM;
332
333 config->num_params = num_params;
334 config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] =
335 xe->info.devid | (xe->info.revid << 16);
336 if (xe_device_get_root_tile(xe)->mem.vram.usable_size)
337 config->info[DRM_XE_QUERY_CONFIG_FLAGS] =
338 DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM;
339 config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] =
340 xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K;
341 config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits;
342 config->info[DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] =
343 xe_exec_queue_device_get_max_priority(xe);
344
345 if (copy_to_user(query_ptr, config, size)) {
346 kfree(config);
347 return -EFAULT;
348 }
349 kfree(config);
350
351 return 0;
352}
353
354static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query)
355{
356 struct xe_gt *gt;
357 size_t size = sizeof(struct drm_xe_query_gt_list) +
358 xe->info.gt_count * sizeof(struct drm_xe_gt);
359 struct drm_xe_query_gt_list __user *query_ptr =
360 u64_to_user_ptr(query->data);
361 struct drm_xe_query_gt_list *gt_list;
362 u8 id;
363
364 if (query->size == 0) {
365 query->size = size;
366 return 0;
367 } else if (XE_IOCTL_DBG(xe, query->size != size)) {
368 return -EINVAL;
369 }
370
371 gt_list = kzalloc(size, GFP_KERNEL);
372 if (!gt_list)
373 return -ENOMEM;
374
375 gt_list->num_gt = xe->info.gt_count;
376
377 for_each_gt(gt, xe, id) {
378 if (xe_gt_is_media_type(gt))
379 gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MEDIA;
380 else
381 gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MAIN;
382 gt_list->gt_list[id].tile_id = gt_to_tile(gt)->id;
383 gt_list->gt_list[id].gt_id = gt->info.id;
384 gt_list->gt_list[id].reference_clock = gt->info.reference_clock;
385 /*
386 * The mem_regions indexes in the mask below need to
387 * directly identify the struct
388 * drm_xe_query_mem_regions' instance constructed at
389 * query_mem_regions()
390 *
391 * For our current platforms:
392 * Bit 0 -> System Memory
393 * Bit 1 -> VRAM0 on Tile0
394 * Bit 2 -> VRAM1 on Tile1
395 * However the uAPI is generic and it's userspace's
396 * responsibility to check the mem_class, without any
397 * assumption.
398 */
399 if (!IS_DGFX(xe))
400 gt_list->gt_list[id].near_mem_regions = 0x1;
401 else
402 gt_list->gt_list[id].near_mem_regions =
403 BIT(gt_to_tile(gt)->id) << 1;
404 gt_list->gt_list[id].far_mem_regions = xe->info.mem_region_mask ^
405 gt_list->gt_list[id].near_mem_regions;
406 }
407
408 if (copy_to_user(query_ptr, gt_list, size)) {
409 kfree(gt_list);
410 return -EFAULT;
411 }
412 kfree(gt_list);
413
414 return 0;
415}
416
417static int query_hwconfig(struct xe_device *xe,
418 struct drm_xe_device_query *query)
419{
420 struct xe_gt *gt = xe_root_mmio_gt(xe);
421 size_t size = xe_guc_hwconfig_size(>->uc.guc);
422 void __user *query_ptr = u64_to_user_ptr(query->data);
423 void *hwconfig;
424
425 if (query->size == 0) {
426 query->size = size;
427 return 0;
428 } else if (XE_IOCTL_DBG(xe, query->size != size)) {
429 return -EINVAL;
430 }
431
432 hwconfig = kzalloc(size, GFP_KERNEL);
433 if (!hwconfig)
434 return -ENOMEM;
435
436 xe_device_mem_access_get(xe);
437 xe_guc_hwconfig_copy(>->uc.guc, hwconfig);
438 xe_device_mem_access_put(xe);
439
440 if (copy_to_user(query_ptr, hwconfig, size)) {
441 kfree(hwconfig);
442 return -EFAULT;
443 }
444 kfree(hwconfig);
445
446 return 0;
447}
448
449static size_t calc_topo_query_size(struct xe_device *xe)
450{
451 return xe->info.gt_count *
452 (3 * sizeof(struct drm_xe_query_topology_mask) +
453 sizeof_field(struct xe_gt, fuse_topo.g_dss_mask) +
454 sizeof_field(struct xe_gt, fuse_topo.c_dss_mask) +
455 sizeof_field(struct xe_gt, fuse_topo.eu_mask_per_dss));
456}
457
458static int copy_mask(void __user **ptr,
459 struct drm_xe_query_topology_mask *topo,
460 void *mask, size_t mask_size)
461{
462 topo->num_bytes = mask_size;
463
464 if (copy_to_user(*ptr, topo, sizeof(*topo)))
465 return -EFAULT;
466 *ptr += sizeof(topo);
467
468 if (copy_to_user(*ptr, mask, mask_size))
469 return -EFAULT;
470 *ptr += mask_size;
471
472 return 0;
473}
474
475static int query_gt_topology(struct xe_device *xe,
476 struct drm_xe_device_query *query)
477{
478 void __user *query_ptr = u64_to_user_ptr(query->data);
479 size_t size = calc_topo_query_size(xe);
480 struct drm_xe_query_topology_mask topo;
481 struct xe_gt *gt;
482 int id;
483
484 if (query->size == 0) {
485 query->size = size;
486 return 0;
487 } else if (XE_IOCTL_DBG(xe, query->size != size)) {
488 return -EINVAL;
489 }
490
491 for_each_gt(gt, xe, id) {
492 int err;
493
494 topo.gt_id = id;
495
496 topo.type = DRM_XE_TOPO_DSS_GEOMETRY;
497 err = copy_mask(&query_ptr, &topo, gt->fuse_topo.g_dss_mask,
498 sizeof(gt->fuse_topo.g_dss_mask));
499 if (err)
500 return err;
501
502 topo.type = DRM_XE_TOPO_DSS_COMPUTE;
503 err = copy_mask(&query_ptr, &topo, gt->fuse_topo.c_dss_mask,
504 sizeof(gt->fuse_topo.c_dss_mask));
505 if (err)
506 return err;
507
508 topo.type = DRM_XE_TOPO_EU_PER_DSS;
509 err = copy_mask(&query_ptr, &topo,
510 gt->fuse_topo.eu_mask_per_dss,
511 sizeof(gt->fuse_topo.eu_mask_per_dss));
512 if (err)
513 return err;
514 }
515
516 return 0;
517}
518
519static int
520query_uc_fw_version(struct xe_device *xe, struct drm_xe_device_query *query)
521{
522 struct drm_xe_query_uc_fw_version __user *query_ptr = u64_to_user_ptr(query->data);
523 size_t size = sizeof(struct drm_xe_query_uc_fw_version);
524 struct drm_xe_query_uc_fw_version resp;
525 struct xe_uc_fw_version *version = NULL;
526
527 if (query->size == 0) {
528 query->size = size;
529 return 0;
530 } else if (XE_IOCTL_DBG(xe, query->size != size)) {
531 return -EINVAL;
532 }
533
534 if (copy_from_user(&resp, query_ptr, size))
535 return -EFAULT;
536
537 if (XE_IOCTL_DBG(xe, resp.pad || resp.pad2 || resp.reserved))
538 return -EINVAL;
539
540 switch (resp.uc_type) {
541 case XE_QUERY_UC_TYPE_GUC_SUBMISSION: {
542 struct xe_guc *guc = &xe->tiles[0].primary_gt->uc.guc;
543
544 version = &guc->fw.versions.found[XE_UC_FW_VER_COMPATIBILITY];
545 break;
546 }
547 default:
548 return -EINVAL;
549 }
550
551 resp.branch_ver = 0;
552 resp.major_ver = version->major;
553 resp.minor_ver = version->minor;
554 resp.patch_ver = version->patch;
555
556 if (copy_to_user(query_ptr, &resp, size))
557 return -EFAULT;
558
559 return 0;
560}
561
562static int (* const xe_query_funcs[])(struct xe_device *xe,
563 struct drm_xe_device_query *query) = {
564 query_engines,
565 query_mem_regions,
566 query_config,
567 query_gt_list,
568 query_hwconfig,
569 query_gt_topology,
570 query_engine_cycles,
571 query_uc_fw_version,
572};
573
574int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
575{
576 struct xe_device *xe = to_xe_device(dev);
577 struct drm_xe_device_query *query = data;
578 u32 idx;
579
580 if (XE_IOCTL_DBG(xe, query->extensions) ||
581 XE_IOCTL_DBG(xe, query->reserved[0] || query->reserved[1]))
582 return -EINVAL;
583
584 if (XE_IOCTL_DBG(xe, query->query >= ARRAY_SIZE(xe_query_funcs)))
585 return -EINVAL;
586
587 idx = array_index_nospec(query->query, ARRAY_SIZE(xe_query_funcs));
588 if (XE_IOCTL_DBG(xe, !xe_query_funcs[idx]))
589 return -EINVAL;
590
591 return xe_query_funcs[idx](xe, query);
592}