Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2023-2024 Intel Corporation
4 */
5
6#include <linux/anon_inodes.h>
7#include <linux/delay.h>
8#include <linux/nospec.h>
9#include <linux/poll.h>
10
11#include <drm/drm_drv.h>
12#include <drm/drm_managed.h>
13#include <uapi/drm/xe_drm.h>
14
15#include "abi/guc_actions_slpc_abi.h"
16#include "instructions/xe_mi_commands.h"
17#include "regs/xe_engine_regs.h"
18#include "regs/xe_gt_regs.h"
19#include "regs/xe_lrc_layout.h"
20#include "regs/xe_oa_regs.h"
21#include "xe_assert.h"
22#include "xe_bb.h"
23#include "xe_bo.h"
24#include "xe_device.h"
25#include "xe_exec_queue.h"
26#include "xe_force_wake.h"
27#include "xe_gt.h"
28#include "xe_gt_mcr.h"
29#include "xe_gt_printk.h"
30#include "xe_guc_pc.h"
31#include "xe_lrc.h"
32#include "xe_macros.h"
33#include "xe_mmio.h"
34#include "xe_oa.h"
35#include "xe_observation.h"
36#include "xe_pm.h"
37#include "xe_sched_job.h"
38#include "xe_sriov.h"
39#include "xe_sync.h"
40
41#define DEFAULT_POLL_FREQUENCY_HZ 200
42#define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ)
43#define XE_OA_UNIT_INVALID U32_MAX
44
45enum xe_oa_submit_deps {
46 XE_OA_SUBMIT_NO_DEPS,
47 XE_OA_SUBMIT_ADD_DEPS,
48};
49
50enum xe_oa_user_extn_from {
51 XE_OA_USER_EXTN_FROM_OPEN,
52 XE_OA_USER_EXTN_FROM_CONFIG,
53};
54
55struct xe_oa_reg {
56 struct xe_reg addr;
57 u32 value;
58};
59
60struct xe_oa_config {
61 struct xe_oa *oa;
62
63 char uuid[UUID_STRING_LEN + 1];
64 int id;
65
66 const struct xe_oa_reg *regs;
67 u32 regs_len;
68
69 struct attribute_group sysfs_metric;
70 struct attribute *attrs[2];
71 struct kobj_attribute sysfs_metric_id;
72
73 struct kref ref;
74 struct rcu_head rcu;
75};
76
77struct xe_oa_open_param {
78 struct xe_file *xef;
79 u32 oa_unit_id;
80 bool sample;
81 u32 metric_set;
82 enum xe_oa_format_name oa_format;
83 int period_exponent;
84 bool disabled;
85 int exec_queue_id;
86 int engine_instance;
87 struct xe_exec_queue *exec_q;
88 struct xe_hw_engine *hwe;
89 bool no_preempt;
90 struct drm_xe_sync __user *syncs_user;
91 int num_syncs;
92 struct xe_sync_entry *syncs;
93 size_t oa_buffer_size;
94 int wait_num_reports;
95};
96
97struct xe_oa_config_bo {
98 struct llist_node node;
99
100 struct xe_oa_config *oa_config;
101 struct xe_bb *bb;
102};
103
104struct xe_oa_fence {
105 /* @base: dma fence base */
106 struct dma_fence base;
107 /* @lock: lock for the fence */
108 spinlock_t lock;
109 /* @work: work to signal @base */
110 struct delayed_work work;
111 /* @cb: callback to schedule @work */
112 struct dma_fence_cb cb;
113};
114
115#define DRM_FMT(x) DRM_XE_OA_FMT_TYPE_##x
116
117static const struct xe_oa_format oa_formats[] = {
118 [XE_OA_FORMAT_C4_B8] = { 7, 64, DRM_FMT(OAG) },
119 [XE_OA_FORMAT_A12] = { 0, 64, DRM_FMT(OAG) },
120 [XE_OA_FORMAT_A12_B8_C8] = { 2, 128, DRM_FMT(OAG) },
121 [XE_OA_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256, DRM_FMT(OAG) },
122 [XE_OAR_FORMAT_A32u40_A4u32_B8_C8] = { 5, 256, DRM_FMT(OAR) },
123 [XE_OA_FORMAT_A24u40_A14u32_B8_C8] = { 5, 256, DRM_FMT(OAG) },
124 [XE_OAC_FORMAT_A24u64_B8_C8] = { 1, 320, DRM_FMT(OAC), HDR_64_BIT },
125 [XE_OAC_FORMAT_A22u32_R2u32_B8_C8] = { 2, 192, DRM_FMT(OAC), HDR_64_BIT },
126 [XE_OAM_FORMAT_MPEC8u64_B8_C8] = { 1, 192, DRM_FMT(OAM_MPEC), HDR_64_BIT },
127 [XE_OAM_FORMAT_MPEC8u32_B8_C8] = { 2, 128, DRM_FMT(OAM_MPEC), HDR_64_BIT },
128 [XE_OA_FORMAT_PEC64u64] = { 1, 576, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
129 [XE_OA_FORMAT_PEC64u64_B8_C8] = { 1, 640, DRM_FMT(PEC), HDR_64_BIT, 1, 1 },
130 [XE_OA_FORMAT_PEC64u32] = { 1, 320, DRM_FMT(PEC), HDR_64_BIT },
131 [XE_OA_FORMAT_PEC32u64_G1] = { 5, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
132 [XE_OA_FORMAT_PEC32u32_G1] = { 5, 192, DRM_FMT(PEC), HDR_64_BIT },
133 [XE_OA_FORMAT_PEC32u64_G2] = { 6, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
134 [XE_OA_FORMAT_PEC32u32_G2] = { 6, 192, DRM_FMT(PEC), HDR_64_BIT },
135 [XE_OA_FORMAT_PEC36u64_G1_32_G2_4] = { 3, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
136 [XE_OA_FORMAT_PEC36u64_G1_4_G2_32] = { 4, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
137};
138
139static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head)
140{
141 return tail >= head ? tail - head :
142 tail + stream->oa_buffer.circ_size - head;
143}
144
145static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n)
146{
147 return ptr + n >= stream->oa_buffer.circ_size ?
148 ptr + n - stream->oa_buffer.circ_size : ptr + n;
149}
150
151static void xe_oa_config_release(struct kref *ref)
152{
153 struct xe_oa_config *oa_config =
154 container_of(ref, typeof(*oa_config), ref);
155
156 kfree(oa_config->regs);
157
158 kfree_rcu(oa_config, rcu);
159}
160
161static void xe_oa_config_put(struct xe_oa_config *oa_config)
162{
163 if (!oa_config)
164 return;
165
166 kref_put(&oa_config->ref, xe_oa_config_release);
167}
168
169static struct xe_oa_config *xe_oa_config_get(struct xe_oa_config *oa_config)
170{
171 return kref_get_unless_zero(&oa_config->ref) ? oa_config : NULL;
172}
173
174static struct xe_oa_config *xe_oa_get_oa_config(struct xe_oa *oa, int metrics_set)
175{
176 struct xe_oa_config *oa_config;
177
178 rcu_read_lock();
179 oa_config = idr_find(&oa->metrics_idr, metrics_set);
180 if (oa_config)
181 oa_config = xe_oa_config_get(oa_config);
182 rcu_read_unlock();
183
184 return oa_config;
185}
186
187static void free_oa_config_bo(struct xe_oa_config_bo *oa_bo, struct dma_fence *last_fence)
188{
189 xe_oa_config_put(oa_bo->oa_config);
190 xe_bb_free(oa_bo->bb, last_fence);
191 kfree(oa_bo);
192}
193
194static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream)
195{
196 return &stream->hwe->oa_unit->regs;
197}
198
199static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream)
200{
201 return xe_mmio_read32(&stream->gt->mmio, __oa_regs(stream)->oa_tail_ptr) &
202 OAG_OATAILPTR_MASK;
203}
204
205#define oa_report_header_64bit(__s) \
206 ((__s)->oa_buffer.format->header == HDR_64_BIT)
207
208static u64 oa_report_id(struct xe_oa_stream *stream, void *report)
209{
210 return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
211}
212
213static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report)
214{
215 if (oa_report_header_64bit(stream))
216 *(u64 *)report = 0;
217 else
218 *report = 0;
219}
220
221static u64 oa_timestamp(struct xe_oa_stream *stream, void *report)
222{
223 return oa_report_header_64bit(stream) ?
224 *((u64 *)report + 1) :
225 *((u32 *)report + 1);
226}
227
228static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report)
229{
230 if (oa_report_header_64bit(stream))
231 *(u64 *)&report[2] = 0;
232 else
233 report[1] = 0;
234}
235
236static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream)
237{
238 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
239 u32 tail, hw_tail, partial_report_size, available;
240 int report_size = stream->oa_buffer.format->size;
241 unsigned long flags;
242
243 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
244
245 hw_tail = xe_oa_hw_tail_read(stream);
246 hw_tail -= gtt_offset;
247
248 /*
249 * The tail pointer increases in 64 byte (cacheline size), not in report_size
250 * increments. Also report size may not be a power of 2. Compute potential
251 * partially landed report in OA buffer.
252 */
253 partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail);
254 partial_report_size %= report_size;
255
256 /* Subtract partial amount off the tail */
257 hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size);
258
259 tail = hw_tail;
260
261 /*
262 * Walk the stream backward until we find a report with report id and timestamp
263 * not 0. We can't tell whether a report has fully landed in memory before the
264 * report id and timestamp of the following report have landed.
265 *
266 * This is assuming that the writes of the OA unit land in memory in the order
267 * they were written. If not : (╯°□°）╯︵ ┻━┻
268 */
269 while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) {
270 void *report = stream->oa_buffer.vaddr + tail;
271
272 if (oa_report_id(stream, report) || oa_timestamp(stream, report))
273 break;
274
275 tail = xe_oa_circ_diff(stream, tail, report_size);
276 }
277
278 if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size)
279 drm_dbg(&stream->oa->xe->drm,
280 "unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n",
281 stream->oa_buffer.head, tail, hw_tail);
282
283 stream->oa_buffer.tail = tail;
284
285 available = xe_oa_circ_diff(stream, stream->oa_buffer.tail, stream->oa_buffer.head);
286 stream->pollin = available >= stream->wait_num_reports * report_size;
287
288 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
289
290 return stream->pollin;
291}
292
293static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer)
294{
295 struct xe_oa_stream *stream =
296 container_of(hrtimer, typeof(*stream), poll_check_timer);
297
298 if (xe_oa_buffer_check_unlocked(stream))
299 wake_up(&stream->poll_wq);
300
301 hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns));
302
303 return HRTIMER_RESTART;
304}
305
306static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf,
307 size_t count, size_t *offset, const u8 *report)
308{
309 int report_size = stream->oa_buffer.format->size;
310 int report_size_partial;
311 u8 *oa_buf_end;
312
313 if ((count - *offset) < report_size)
314 return -ENOSPC;
315
316 buf += *offset;
317
318 oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;
319 report_size_partial = oa_buf_end - report;
320
321 if (report_size_partial < report_size) {
322 if (copy_to_user(buf, report, report_size_partial))
323 return -EFAULT;
324 buf += report_size_partial;
325
326 if (copy_to_user(buf, stream->oa_buffer.vaddr,
327 report_size - report_size_partial))
328 return -EFAULT;
329 } else if (copy_to_user(buf, report, report_size)) {
330 return -EFAULT;
331 }
332
333 *offset += report_size;
334
335 return 0;
336}
337
338static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf,
339 size_t count, size_t *offset)
340{
341 int report_size = stream->oa_buffer.format->size;
342 u8 *oa_buf_base = stream->oa_buffer.vaddr;
343 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
344 size_t start_offset = *offset;
345 unsigned long flags;
346 u32 head, tail;
347 int ret = 0;
348
349 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
350 head = stream->oa_buffer.head;
351 tail = stream->oa_buffer.tail;
352 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
353
354 xe_assert(stream->oa->xe,
355 head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size);
356
357 for (; xe_oa_circ_diff(stream, tail, head);
358 head = xe_oa_circ_incr(stream, head, report_size)) {
359 u8 *report = oa_buf_base + head;
360
361 ret = xe_oa_append_report(stream, buf, count, offset, report);
362 if (ret)
363 break;
364
365 if (!(stream->oa_buffer.circ_size % report_size)) {
366 /* Clear out report id and timestamp to detect unlanded reports */
367 oa_report_id_clear(stream, (void *)report);
368 oa_timestamp_clear(stream, (void *)report);
369 } else {
370 u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;
371 u32 part = oa_buf_end - report;
372
373 /* Zero out the entire report */
374 if (report_size <= part) {
375 memset(report, 0, report_size);
376 } else {
377 memset(report, 0, part);
378 memset(oa_buf_base, 0, report_size - part);
379 }
380 }
381 }
382
383 if (start_offset != *offset) {
384 struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr;
385
386 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
387 xe_mmio_write32(&stream->gt->mmio, oaheadptr,
388 (head + gtt_offset) & OAG_OAHEADPTR_MASK);
389 stream->oa_buffer.head = head;
390 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
391 }
392
393 return ret;
394}
395
396static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream)
397{
398 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
399 int size_exponent = __ffs(stream->oa_buffer.bo->size);
400 u32 oa_buf = gtt_offset | OAG_OABUFFER_MEMORY_SELECT;
401 struct xe_mmio *mmio = &stream->gt->mmio;
402 unsigned long flags;
403
404 /*
405 * If oa buffer size is more than 16MB (exponent greater than 24), the
406 * oa buffer size field is multiplied by 8 in xe_oa_enable_metric_set.
407 */
408 oa_buf |= REG_FIELD_PREP(OABUFFER_SIZE_MASK,
409 size_exponent > 24 ? size_exponent - 20 : size_exponent - 17);
410
411 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
412
413 xe_mmio_write32(mmio, __oa_regs(stream)->oa_status, 0);
414 xe_mmio_write32(mmio, __oa_regs(stream)->oa_head_ptr,
415 gtt_offset & OAG_OAHEADPTR_MASK);
416 stream->oa_buffer.head = 0;
417 /*
418 * PRM says: "This MMIO must be set before the OATAILPTR register and after the
419 * OAHEADPTR register. This is to enable proper functionality of the overflow bit".
420 */
421 xe_mmio_write32(mmio, __oa_regs(stream)->oa_buffer, oa_buf);
422 xe_mmio_write32(mmio, __oa_regs(stream)->oa_tail_ptr,
423 gtt_offset & OAG_OATAILPTR_MASK);
424
425 /* Mark that we need updated tail pointer to read from */
426 stream->oa_buffer.tail = 0;
427
428 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
429
430 /* Zero out the OA buffer since we rely on zero report id and timestamp fields */
431 memset(stream->oa_buffer.vaddr, 0, stream->oa_buffer.bo->size);
432}
433
434static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask)
435{
436 return ((format->counter_select << (ffs(counter_sel_mask) - 1)) & counter_sel_mask) |
437 REG_FIELD_PREP(OA_OACONTROL_REPORT_BC_MASK, format->bc_report) |
438 REG_FIELD_PREP(OA_OACONTROL_COUNTER_SIZE_MASK, format->counter_size);
439}
440
441static u32 __oa_ccs_select(struct xe_oa_stream *stream)
442{
443 u32 val;
444
445 if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE)
446 return 0;
447
448 val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance);
449 xe_assert(stream->oa->xe,
450 REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance);
451 return val;
452}
453
454static u32 __oactrl_used_bits(struct xe_oa_stream *stream)
455{
456 return stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG ?
457 OAG_OACONTROL_USED_BITS : OAM_OACONTROL_USED_BITS;
458}
459
460static void xe_oa_enable(struct xe_oa_stream *stream)
461{
462 const struct xe_oa_format *format = stream->oa_buffer.format;
463 const struct xe_oa_regs *regs;
464 u32 val;
465
466 /*
467 * BSpec: 46822: Bit 0. Even if stream->sample is 0, for OAR to function, the OA
468 * buffer must be correctly initialized
469 */
470 xe_oa_init_oa_buffer(stream);
471
472 regs = __oa_regs(stream);
473 val = __format_to_oactrl(format, regs->oa_ctrl_counter_select_mask) |
474 __oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE;
475
476 if (GRAPHICS_VER(stream->oa->xe) >= 20 &&
477 stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG)
478 val |= OAG_OACONTROL_OA_PES_DISAG_EN;
479
480 xe_mmio_rmw32(&stream->gt->mmio, regs->oa_ctrl, __oactrl_used_bits(stream), val);
481}
482
483static void xe_oa_disable(struct xe_oa_stream *stream)
484{
485 struct xe_mmio *mmio = &stream->gt->mmio;
486
487 xe_mmio_rmw32(mmio, __oa_regs(stream)->oa_ctrl, __oactrl_used_bits(stream), 0);
488 if (xe_mmio_wait32(mmio, __oa_regs(stream)->oa_ctrl,
489 OAG_OACONTROL_OA_COUNTER_ENABLE, 0, 50000, NULL, false))
490 drm_err(&stream->oa->xe->drm,
491 "wait for OA to be disabled timed out\n");
492
493 if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) {
494 /* <= XE_METEORLAKE except XE_PVC */
495 xe_mmio_write32(mmio, OA_TLB_INV_CR, 1);
496 if (xe_mmio_wait32(mmio, OA_TLB_INV_CR, 1, 0, 50000, NULL, false))
497 drm_err(&stream->oa->xe->drm,
498 "wait for OA tlb invalidate timed out\n");
499 }
500}
501
502static int xe_oa_wait_unlocked(struct xe_oa_stream *stream)
503{
504 /* We might wait indefinitely if periodic sampling is not enabled */
505 if (!stream->periodic)
506 return -EINVAL;
507
508 return wait_event_interruptible(stream->poll_wq,
509 xe_oa_buffer_check_unlocked(stream));
510}
511
512#define OASTATUS_RELEVANT_BITS (OASTATUS_MMIO_TRG_Q_FULL | OASTATUS_COUNTER_OVERFLOW | \
513 OASTATUS_BUFFER_OVERFLOW | OASTATUS_REPORT_LOST)
514
515static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf,
516 size_t count, size_t *offset)
517{
518 /* Only clear our bits to avoid side-effects */
519 stream->oa_status = xe_mmio_rmw32(&stream->gt->mmio, __oa_regs(stream)->oa_status,
520 OASTATUS_RELEVANT_BITS, 0);
521 /*
522 * Signal to userspace that there is non-zero OA status to read via
523 * @DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl
524 */
525 if (stream->oa_status & OASTATUS_RELEVANT_BITS)
526 return -EIO;
527
528 return xe_oa_append_reports(stream, buf, count, offset);
529}
530
531static ssize_t xe_oa_read(struct file *file, char __user *buf,
532 size_t count, loff_t *ppos)
533{
534 struct xe_oa_stream *stream = file->private_data;
535 size_t offset = 0;
536 int ret;
537
538 /* Can't read from disabled streams */
539 if (!stream->enabled || !stream->sample)
540 return -EINVAL;
541
542 if (!(file->f_flags & O_NONBLOCK)) {
543 do {
544 ret = xe_oa_wait_unlocked(stream);
545 if (ret)
546 return ret;
547
548 mutex_lock(&stream->stream_lock);
549 ret = __xe_oa_read(stream, buf, count, &offset);
550 mutex_unlock(&stream->stream_lock);
551 } while (!offset && !ret);
552 } else {
553 mutex_lock(&stream->stream_lock);
554 ret = __xe_oa_read(stream, buf, count, &offset);
555 mutex_unlock(&stream->stream_lock);
556 }
557
558 /*
559 * Typically we clear pollin here in order to wait for the new hrtimer callback
560 * before unblocking. The exception to this is if __xe_oa_read returns -ENOSPC,
561 * which means that more OA data is available than could fit in the user provided
562 * buffer. In this case we want the next poll() call to not block.
563 *
564 * Also in case of -EIO, we have already waited for data before returning
565 * -EIO, so need to wait again
566 */
567 if (ret != -ENOSPC && ret != -EIO)
568 stream->pollin = false;
569
570 /* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, -EINVAL, ... */
571 return offset ?: (ret ?: -EAGAIN);
572}
573
574static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream,
575 struct file *file, poll_table *wait)
576{
577 __poll_t events = 0;
578
579 poll_wait(file, &stream->poll_wq, wait);
580
581 /*
582 * We don't explicitly check whether there's something to read here since this
583 * path may be hot depending on what else userspace is polling, or on the timeout
584 * in use. We rely on hrtimer xe_oa_poll_check_timer_cb to notify us when there
585 * are samples to read
586 */
587 if (stream->pollin)
588 events |= EPOLLIN;
589
590 return events;
591}
592
593static __poll_t xe_oa_poll(struct file *file, poll_table *wait)
594{
595 struct xe_oa_stream *stream = file->private_data;
596 __poll_t ret;
597
598 mutex_lock(&stream->stream_lock);
599 ret = xe_oa_poll_locked(stream, file, wait);
600 mutex_unlock(&stream->stream_lock);
601
602 return ret;
603}
604
605static void xe_oa_lock_vma(struct xe_exec_queue *q)
606{
607 if (q->vm) {
608 down_read(&q->vm->lock);
609 xe_vm_lock(q->vm, false);
610 }
611}
612
613static void xe_oa_unlock_vma(struct xe_exec_queue *q)
614{
615 if (q->vm) {
616 xe_vm_unlock(q->vm);
617 up_read(&q->vm->lock);
618 }
619}
620
621static struct dma_fence *xe_oa_submit_bb(struct xe_oa_stream *stream, enum xe_oa_submit_deps deps,
622 struct xe_bb *bb)
623{
624 struct xe_exec_queue *q = stream->exec_q ?: stream->k_exec_q;
625 struct xe_sched_job *job;
626 struct dma_fence *fence;
627 int err = 0;
628
629 xe_oa_lock_vma(q);
630
631 job = xe_bb_create_job(q, bb);
632 if (IS_ERR(job)) {
633 err = PTR_ERR(job);
634 goto exit;
635 }
636 job->ggtt = true;
637
638 if (deps == XE_OA_SUBMIT_ADD_DEPS) {
639 for (int i = 0; i < stream->num_syncs && !err; i++)
640 err = xe_sync_entry_add_deps(&stream->syncs[i], job);
641 if (err) {
642 drm_dbg(&stream->oa->xe->drm, "xe_sync_entry_add_deps err %d\n", err);
643 goto err_put_job;
644 }
645 }
646
647 xe_sched_job_arm(job);
648 fence = dma_fence_get(&job->drm.s_fence->finished);
649 xe_sched_job_push(job);
650
651 xe_oa_unlock_vma(q);
652
653 return fence;
654err_put_job:
655 xe_sched_job_put(job);
656exit:
657 xe_oa_unlock_vma(q);
658 return ERR_PTR(err);
659}
660
661static void write_cs_mi_lri(struct xe_bb *bb, const struct xe_oa_reg *reg_data, u32 n_regs)
662{
663 u32 i;
664
665#define MI_LOAD_REGISTER_IMM_MAX_REGS (126)
666
667 for (i = 0; i < n_regs; i++) {
668 if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) {
669 u32 n_lri = min_t(u32, n_regs - i,
670 MI_LOAD_REGISTER_IMM_MAX_REGS);
671
672 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(n_lri);
673 }
674 bb->cs[bb->len++] = reg_data[i].addr.addr;
675 bb->cs[bb->len++] = reg_data[i].value;
676 }
677}
678
679static int num_lri_dwords(int num_regs)
680{
681 int count = 0;
682
683 if (num_regs > 0) {
684 count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS);
685 count += num_regs * 2;
686 }
687
688 return count;
689}
690
691static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream)
692{
693 xe_bo_unpin_map_no_vm(stream->oa_buffer.bo);
694}
695
696static void xe_oa_free_configs(struct xe_oa_stream *stream)
697{
698 struct xe_oa_config_bo *oa_bo, *tmp;
699
700 xe_oa_config_put(stream->oa_config);
701 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
702 free_oa_config_bo(oa_bo, stream->last_fence);
703 dma_fence_put(stream->last_fence);
704}
705
706static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri, u32 count)
707{
708 struct dma_fence *fence;
709 struct xe_bb *bb;
710 int err;
711
712 bb = xe_bb_new(stream->gt, 2 * count + 1, false);
713 if (IS_ERR(bb)) {
714 err = PTR_ERR(bb);
715 goto exit;
716 }
717
718 write_cs_mi_lri(bb, reg_lri, count);
719
720 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_NO_DEPS, bb);
721 if (IS_ERR(fence)) {
722 err = PTR_ERR(fence);
723 goto free_bb;
724 }
725 xe_bb_free(bb, fence);
726 dma_fence_put(fence);
727
728 return 0;
729free_bb:
730 xe_bb_free(bb, NULL);
731exit:
732 return err;
733}
734
735static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable)
736{
737 const struct xe_oa_format *format = stream->oa_buffer.format;
738 u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) |
739 (enable ? OAR_OACONTROL_COUNTER_ENABLE : 0);
740
741 struct xe_oa_reg reg_lri[] = {
742 {
743 OACTXCONTROL(stream->hwe->mmio_base),
744 enable ? OA_COUNTER_RESUME : 0,
745 },
746 {
747 OAR_OACONTROL,
748 oacontrol,
749 },
750 {
751 RING_CONTEXT_CONTROL(stream->hwe->mmio_base),
752 _MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE,
753 enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0)
754 },
755 };
756
757 return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri));
758}
759
760static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable)
761{
762 const struct xe_oa_format *format = stream->oa_buffer.format;
763 u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) |
764 (enable ? OAR_OACONTROL_COUNTER_ENABLE : 0);
765 struct xe_oa_reg reg_lri[] = {
766 {
767 OACTXCONTROL(stream->hwe->mmio_base),
768 enable ? OA_COUNTER_RESUME : 0,
769 },
770 {
771 OAC_OACONTROL,
772 oacontrol
773 },
774 {
775 RING_CONTEXT_CONTROL(stream->hwe->mmio_base),
776 _MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE,
777 enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0) |
778 _MASKED_FIELD(CTX_CTRL_RUN_ALONE, enable ? CTX_CTRL_RUN_ALONE : 0),
779 },
780 };
781
782 /* Set ccs select to enable programming of OAC_OACONTROL */
783 xe_mmio_write32(&stream->gt->mmio, __oa_regs(stream)->oa_ctrl,
784 __oa_ccs_select(stream));
785
786 return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri));
787}
788
789static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable)
790{
791 switch (stream->hwe->class) {
792 case XE_ENGINE_CLASS_RENDER:
793 return xe_oa_configure_oar_context(stream, enable);
794 case XE_ENGINE_CLASS_COMPUTE:
795 return xe_oa_configure_oac_context(stream, enable);
796 default:
797 /* Video engines do not support MI_REPORT_PERF_COUNT */
798 return 0;
799 }
800}
801
802#define HAS_OA_BPC_REPORTING(xe) (GRAPHICS_VERx100(xe) >= 1255)
803
804static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable)
805{
806 return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_MMIO_TRG,
807 enable && stream && stream->sample ?
808 0 : OAG_OA_DEBUG_DISABLE_MMIO_TRG);
809}
810
811static void xe_oa_disable_metric_set(struct xe_oa_stream *stream)
812{
813 struct xe_mmio *mmio = &stream->gt->mmio;
814 u32 sqcnt1;
815
816 /*
817 * Wa_1508761755:xehpsdv, dg2
818 * Enable thread stall DOP gating and EU DOP gating.
819 */
820 if (stream->oa->xe->info.platform == XE_DG2) {
821 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,
822 _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE));
823 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,
824 _MASKED_BIT_DISABLE(DISABLE_DOP_GATING));
825 }
826
827 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug,
828 oag_configure_mmio_trigger(stream, false));
829
830 /* disable the context save/restore or OAR counters */
831 if (stream->exec_q)
832 xe_oa_configure_oa_context(stream, false);
833
834 /* Make sure we disable noa to save power. */
835 xe_mmio_rmw32(mmio, RPM_CONFIG1, GT_NOA_ENABLE, 0);
836
837 sqcnt1 = SQCNT1_PMON_ENABLE |
838 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0);
839
840 /* Reset PMON Enable to save power. */
841 xe_mmio_rmw32(mmio, XELPMP_SQCNT1, sqcnt1, 0);
842}
843
844static void xe_oa_stream_destroy(struct xe_oa_stream *stream)
845{
846 struct xe_oa_unit *u = stream->hwe->oa_unit;
847 struct xe_gt *gt = stream->hwe->gt;
848
849 if (WARN_ON(stream != u->exclusive_stream))
850 return;
851
852 WRITE_ONCE(u->exclusive_stream, NULL);
853
854 mutex_destroy(&stream->stream_lock);
855
856 xe_oa_disable_metric_set(stream);
857 xe_exec_queue_put(stream->k_exec_q);
858
859 xe_oa_free_oa_buffer(stream);
860
861 xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
862 xe_pm_runtime_put(stream->oa->xe);
863
864 /* Wa_1509372804:pvc: Unset the override of GUCRC mode to enable rc6 */
865 if (stream->override_gucrc)
866 xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(>->uc.guc.pc));
867
868 xe_oa_free_configs(stream);
869 xe_file_put(stream->xef);
870}
871
872static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size)
873{
874 struct xe_bo *bo;
875
876 bo = xe_bo_create_pin_map(stream->oa->xe, stream->gt->tile, NULL,
877 size, ttm_bo_type_kernel,
878 XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT);
879 if (IS_ERR(bo))
880 return PTR_ERR(bo);
881
882 stream->oa_buffer.bo = bo;
883 /* mmap implementation requires OA buffer to be in system memory */
884 xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0);
885 stream->oa_buffer.vaddr = bo->vmap.vaddr;
886 return 0;
887}
888
889static struct xe_oa_config_bo *
890__xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)
891{
892 struct xe_oa_config_bo *oa_bo;
893 size_t config_length;
894 struct xe_bb *bb;
895
896 oa_bo = kzalloc(sizeof(*oa_bo), GFP_KERNEL);
897 if (!oa_bo)
898 return ERR_PTR(-ENOMEM);
899
900 config_length = num_lri_dwords(oa_config->regs_len);
901 config_length = ALIGN(sizeof(u32) * config_length, XE_PAGE_SIZE) / sizeof(u32);
902
903 bb = xe_bb_new(stream->gt, config_length, false);
904 if (IS_ERR(bb))
905 goto err_free;
906
907 write_cs_mi_lri(bb, oa_config->regs, oa_config->regs_len);
908
909 oa_bo->bb = bb;
910 oa_bo->oa_config = xe_oa_config_get(oa_config);
911 llist_add(&oa_bo->node, &stream->oa_config_bos);
912
913 return oa_bo;
914err_free:
915 kfree(oa_bo);
916 return ERR_CAST(bb);
917}
918
919static struct xe_oa_config_bo *
920xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)
921{
922 struct xe_oa_config_bo *oa_bo;
923
924 /* Look for the buffer in the already allocated BOs attached to the stream */
925 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
926 if (oa_bo->oa_config == oa_config &&
927 memcmp(oa_bo->oa_config->uuid, oa_config->uuid,
928 sizeof(oa_config->uuid)) == 0)
929 goto out;
930 }
931
932 oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config);
933out:
934 return oa_bo;
935}
936
937static void xe_oa_update_last_fence(struct xe_oa_stream *stream, struct dma_fence *fence)
938{
939 dma_fence_put(stream->last_fence);
940 stream->last_fence = dma_fence_get(fence);
941}
942
943static void xe_oa_fence_work_fn(struct work_struct *w)
944{
945 struct xe_oa_fence *ofence = container_of(w, typeof(*ofence), work.work);
946
947 /* Signal fence to indicate new OA configuration is active */
948 dma_fence_signal(&ofence->base);
949 dma_fence_put(&ofence->base);
950}
951
952static void xe_oa_config_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
953{
954 /* Additional empirical delay needed for NOA programming after registers are written */
955#define NOA_PROGRAM_ADDITIONAL_DELAY_US 500
956
957 struct xe_oa_fence *ofence = container_of(cb, typeof(*ofence), cb);
958
959 INIT_DELAYED_WORK(&ofence->work, xe_oa_fence_work_fn);
960 queue_delayed_work(system_unbound_wq, &ofence->work,
961 usecs_to_jiffies(NOA_PROGRAM_ADDITIONAL_DELAY_US));
962 dma_fence_put(fence);
963}
964
965static const char *xe_oa_get_driver_name(struct dma_fence *fence)
966{
967 return "xe_oa";
968}
969
970static const char *xe_oa_get_timeline_name(struct dma_fence *fence)
971{
972 return "unbound";
973}
974
975static const struct dma_fence_ops xe_oa_fence_ops = {
976 .get_driver_name = xe_oa_get_driver_name,
977 .get_timeline_name = xe_oa_get_timeline_name,
978};
979
980static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config)
981{
982#define NOA_PROGRAM_ADDITIONAL_DELAY_US 500
983 struct xe_oa_config_bo *oa_bo;
984 struct xe_oa_fence *ofence;
985 int i, err, num_signal = 0;
986 struct dma_fence *fence;
987
988 ofence = kzalloc(sizeof(*ofence), GFP_KERNEL);
989 if (!ofence) {
990 err = -ENOMEM;
991 goto exit;
992 }
993
994 oa_bo = xe_oa_alloc_config_buffer(stream, config);
995 if (IS_ERR(oa_bo)) {
996 err = PTR_ERR(oa_bo);
997 goto exit;
998 }
999
1000 /* Emit OA configuration batch */
1001 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_ADD_DEPS, oa_bo->bb);
1002 if (IS_ERR(fence)) {
1003 err = PTR_ERR(fence);
1004 goto exit;
1005 }
1006
1007 /* Point of no return: initialize and set fence to signal */
1008 spin_lock_init(&ofence->lock);
1009 dma_fence_init(&ofence->base, &xe_oa_fence_ops, &ofence->lock, 0, 0);
1010
1011 for (i = 0; i < stream->num_syncs; i++) {
1012 if (stream->syncs[i].flags & DRM_XE_SYNC_FLAG_SIGNAL)
1013 num_signal++;
1014 xe_sync_entry_signal(&stream->syncs[i], &ofence->base);
1015 }
1016
1017 /* Additional dma_fence_get in case we dma_fence_wait */
1018 if (!num_signal)
1019 dma_fence_get(&ofence->base);
1020
1021 /* Update last fence too before adding callback */
1022 xe_oa_update_last_fence(stream, fence);
1023
1024 /* Add job fence callback to schedule work to signal ofence->base */
1025 err = dma_fence_add_callback(fence, &ofence->cb, xe_oa_config_cb);
1026 xe_gt_assert(stream->gt, !err || err == -ENOENT);
1027 if (err == -ENOENT)
1028 xe_oa_config_cb(fence, &ofence->cb);
1029
1030 /* If nothing needs to be signaled we wait synchronously */
1031 if (!num_signal) {
1032 dma_fence_wait(&ofence->base, false);
1033 dma_fence_put(&ofence->base);
1034 }
1035
1036 /* Done with syncs */
1037 for (i = 0; i < stream->num_syncs; i++)
1038 xe_sync_entry_cleanup(&stream->syncs[i]);
1039 kfree(stream->syncs);
1040
1041 return 0;
1042exit:
1043 kfree(ofence);
1044 return err;
1045}
1046
1047static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream)
1048{
1049 /* If user didn't require OA reports, ask HW not to emit ctx switch reports */
1050 return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS,
1051 stream->sample ?
1052 0 : OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
1053}
1054
1055static u32 oag_buf_size_select(const struct xe_oa_stream *stream)
1056{
1057 return _MASKED_FIELD(OAG_OA_DEBUG_BUF_SIZE_SELECT,
1058 stream->oa_buffer.bo->size > SZ_16M ?
1059 OAG_OA_DEBUG_BUF_SIZE_SELECT : 0);
1060}
1061
1062static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)
1063{
1064 struct xe_mmio *mmio = &stream->gt->mmio;
1065 u32 oa_debug, sqcnt1;
1066 int ret;
1067
1068 /*
1069 * Wa_1508761755:xehpsdv, dg2
1070 * EU NOA signals behave incorrectly if EU clock gating is enabled.
1071 * Disable thread stall DOP gating and EU DOP gating.
1072 */
1073 if (stream->oa->xe->info.platform == XE_DG2) {
1074 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,
1075 _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
1076 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,
1077 _MASKED_BIT_ENABLE(DISABLE_DOP_GATING));
1078 }
1079
1080 /* Disable clk ratio reports */
1081 oa_debug = OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
1082 OAG_OA_DEBUG_INCLUDE_CLK_RATIO;
1083
1084 if (GRAPHICS_VER(stream->oa->xe) >= 20)
1085 oa_debug |=
1086 /* The three bits below are needed to get PEC counters running */
1087 OAG_OA_DEBUG_START_TRIGGER_SCOPE_CONTROL |
1088 OAG_OA_DEBUG_DISABLE_START_TRG_2_COUNT_QUAL |
1089 OAG_OA_DEBUG_DISABLE_START_TRG_1_COUNT_QUAL;
1090
1091 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug,
1092 _MASKED_BIT_ENABLE(oa_debug) |
1093 oag_report_ctx_switches(stream) |
1094 oag_buf_size_select(stream) |
1095 oag_configure_mmio_trigger(stream, true));
1096
1097 xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ?
1098 (OAG_OAGLBCTXCTRL_COUNTER_RESUME |
1099 OAG_OAGLBCTXCTRL_TIMER_ENABLE |
1100 REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK,
1101 stream->period_exponent)) : 0);
1102
1103 /*
1104 * Initialize Super Queue Internal Cnt Register
1105 * Set PMON Enable in order to collect valid metrics
1106 * Enable bytes per clock reporting
1107 */
1108 sqcnt1 = SQCNT1_PMON_ENABLE |
1109 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0);
1110
1111 xe_mmio_rmw32(mmio, XELPMP_SQCNT1, 0, sqcnt1);
1112
1113 /* Configure OAR/OAC */
1114 if (stream->exec_q) {
1115 ret = xe_oa_configure_oa_context(stream, true);
1116 if (ret)
1117 return ret;
1118 }
1119
1120 return xe_oa_emit_oa_config(stream, stream->oa_config);
1121}
1122
1123static int decode_oa_format(struct xe_oa *oa, u64 fmt, enum xe_oa_format_name *name)
1124{
1125 u32 counter_size = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE, fmt);
1126 u32 counter_sel = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SEL, fmt);
1127 u32 bc_report = FIELD_GET(DRM_XE_OA_FORMAT_MASK_BC_REPORT, fmt);
1128 u32 type = FIELD_GET(DRM_XE_OA_FORMAT_MASK_FMT_TYPE, fmt);
1129 int idx;
1130
1131 for_each_set_bit(idx, oa->format_mask, __XE_OA_FORMAT_MAX) {
1132 const struct xe_oa_format *f = &oa->oa_formats[idx];
1133
1134 if (counter_size == f->counter_size && bc_report == f->bc_report &&
1135 type == f->type && counter_sel == f->counter_select) {
1136 *name = idx;
1137 return 0;
1138 }
1139 }
1140
1141 return -EINVAL;
1142}
1143
1144static int xe_oa_set_prop_oa_unit_id(struct xe_oa *oa, u64 value,
1145 struct xe_oa_open_param *param)
1146{
1147 if (value >= oa->oa_unit_ids) {
1148 drm_dbg(&oa->xe->drm, "OA unit ID out of range %lld\n", value);
1149 return -EINVAL;
1150 }
1151 param->oa_unit_id = value;
1152 return 0;
1153}
1154
1155static int xe_oa_set_prop_sample_oa(struct xe_oa *oa, u64 value,
1156 struct xe_oa_open_param *param)
1157{
1158 param->sample = value;
1159 return 0;
1160}
1161
1162static int xe_oa_set_prop_metric_set(struct xe_oa *oa, u64 value,
1163 struct xe_oa_open_param *param)
1164{
1165 param->metric_set = value;
1166 return 0;
1167}
1168
1169static int xe_oa_set_prop_oa_format(struct xe_oa *oa, u64 value,
1170 struct xe_oa_open_param *param)
1171{
1172 int ret = decode_oa_format(oa, value, ¶m->oa_format);
1173
1174 if (ret) {
1175 drm_dbg(&oa->xe->drm, "Unsupported OA report format %#llx\n", value);
1176 return ret;
1177 }
1178 return 0;
1179}
1180
1181static int xe_oa_set_prop_oa_exponent(struct xe_oa *oa, u64 value,
1182 struct xe_oa_open_param *param)
1183{
1184#define OA_EXPONENT_MAX 31
1185
1186 if (value > OA_EXPONENT_MAX) {
1187 drm_dbg(&oa->xe->drm, "OA timer exponent too high (> %u)\n", OA_EXPONENT_MAX);
1188 return -EINVAL;
1189 }
1190 param->period_exponent = value;
1191 return 0;
1192}
1193
1194static int xe_oa_set_prop_disabled(struct xe_oa *oa, u64 value,
1195 struct xe_oa_open_param *param)
1196{
1197 param->disabled = value;
1198 return 0;
1199}
1200
1201static int xe_oa_set_prop_exec_queue_id(struct xe_oa *oa, u64 value,
1202 struct xe_oa_open_param *param)
1203{
1204 param->exec_queue_id = value;
1205 return 0;
1206}
1207
1208static int xe_oa_set_prop_engine_instance(struct xe_oa *oa, u64 value,
1209 struct xe_oa_open_param *param)
1210{
1211 param->engine_instance = value;
1212 return 0;
1213}
1214
1215static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value,
1216 struct xe_oa_open_param *param)
1217{
1218 param->no_preempt = value;
1219 return 0;
1220}
1221
1222static int xe_oa_set_prop_num_syncs(struct xe_oa *oa, u64 value,
1223 struct xe_oa_open_param *param)
1224{
1225 param->num_syncs = value;
1226 return 0;
1227}
1228
1229static int xe_oa_set_prop_syncs_user(struct xe_oa *oa, u64 value,
1230 struct xe_oa_open_param *param)
1231{
1232 param->syncs_user = u64_to_user_ptr(value);
1233 return 0;
1234}
1235
1236static int xe_oa_set_prop_oa_buffer_size(struct xe_oa *oa, u64 value,
1237 struct xe_oa_open_param *param)
1238{
1239 if (!is_power_of_2(value) || value < SZ_128K || value > SZ_128M) {
1240 drm_dbg(&oa->xe->drm, "OA buffer size invalid %llu\n", value);
1241 return -EINVAL;
1242 }
1243 param->oa_buffer_size = value;
1244 return 0;
1245}
1246
1247static int xe_oa_set_prop_wait_num_reports(struct xe_oa *oa, u64 value,
1248 struct xe_oa_open_param *param)
1249{
1250 if (!value) {
1251 drm_dbg(&oa->xe->drm, "wait_num_reports %llu\n", value);
1252 return -EINVAL;
1253 }
1254 param->wait_num_reports = value;
1255 return 0;
1256}
1257
1258static int xe_oa_set_prop_ret_inval(struct xe_oa *oa, u64 value,
1259 struct xe_oa_open_param *param)
1260{
1261 return -EINVAL;
1262}
1263
1264typedef int (*xe_oa_set_property_fn)(struct xe_oa *oa, u64 value,
1265 struct xe_oa_open_param *param);
1266static const xe_oa_set_property_fn xe_oa_set_property_funcs_open[] = {
1267 [DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_oa_unit_id,
1268 [DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_sample_oa,
1269 [DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set,
1270 [DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_oa_format,
1271 [DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_oa_exponent,
1272 [DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_disabled,
1273 [DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_exec_queue_id,
1274 [DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_engine_instance,
1275 [DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_no_preempt,
1276 [DRM_XE_OA_PROPERTY_NUM_SYNCS] = xe_oa_set_prop_num_syncs,
1277 [DRM_XE_OA_PROPERTY_SYNCS] = xe_oa_set_prop_syncs_user,
1278 [DRM_XE_OA_PROPERTY_OA_BUFFER_SIZE] = xe_oa_set_prop_oa_buffer_size,
1279 [DRM_XE_OA_PROPERTY_WAIT_NUM_REPORTS] = xe_oa_set_prop_wait_num_reports,
1280};
1281
1282static const xe_oa_set_property_fn xe_oa_set_property_funcs_config[] = {
1283 [DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_ret_inval,
1284 [DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_ret_inval,
1285 [DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set,
1286 [DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_ret_inval,
1287 [DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_ret_inval,
1288 [DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_ret_inval,
1289 [DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_ret_inval,
1290 [DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_ret_inval,
1291 [DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_prop_ret_inval,
1292 [DRM_XE_OA_PROPERTY_NUM_SYNCS] = xe_oa_set_prop_num_syncs,
1293 [DRM_XE_OA_PROPERTY_SYNCS] = xe_oa_set_prop_syncs_user,
1294 [DRM_XE_OA_PROPERTY_OA_BUFFER_SIZE] = xe_oa_set_prop_ret_inval,
1295 [DRM_XE_OA_PROPERTY_WAIT_NUM_REPORTS] = xe_oa_set_prop_ret_inval,
1296};
1297
1298static int xe_oa_user_ext_set_property(struct xe_oa *oa, enum xe_oa_user_extn_from from,
1299 u64 extension, struct xe_oa_open_param *param)
1300{
1301 u64 __user *address = u64_to_user_ptr(extension);
1302 struct drm_xe_ext_set_property ext;
1303 int err;
1304 u32 idx;
1305
1306 err = __copy_from_user(&ext, address, sizeof(ext));
1307 if (XE_IOCTL_DBG(oa->xe, err))
1308 return -EFAULT;
1309
1310 BUILD_BUG_ON(ARRAY_SIZE(xe_oa_set_property_funcs_open) !=
1311 ARRAY_SIZE(xe_oa_set_property_funcs_config));
1312
1313 if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs_open)) ||
1314 XE_IOCTL_DBG(oa->xe, ext.pad))
1315 return -EINVAL;
1316
1317 idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs_open));
1318
1319 if (from == XE_OA_USER_EXTN_FROM_CONFIG)
1320 return xe_oa_set_property_funcs_config[idx](oa, ext.value, param);
1321 else
1322 return xe_oa_set_property_funcs_open[idx](oa, ext.value, param);
1323}
1324
1325typedef int (*xe_oa_user_extension_fn)(struct xe_oa *oa, enum xe_oa_user_extn_from from,
1326 u64 extension, struct xe_oa_open_param *param);
1327static const xe_oa_user_extension_fn xe_oa_user_extension_funcs[] = {
1328 [DRM_XE_OA_EXTENSION_SET_PROPERTY] = xe_oa_user_ext_set_property,
1329};
1330
1331#define MAX_USER_EXTENSIONS 16
1332static int xe_oa_user_extensions(struct xe_oa *oa, enum xe_oa_user_extn_from from, u64 extension,
1333 int ext_number, struct xe_oa_open_param *param)
1334{
1335 u64 __user *address = u64_to_user_ptr(extension);
1336 struct drm_xe_user_extension ext;
1337 int err;
1338 u32 idx;
1339
1340 if (XE_IOCTL_DBG(oa->xe, ext_number >= MAX_USER_EXTENSIONS))
1341 return -E2BIG;
1342
1343 err = __copy_from_user(&ext, address, sizeof(ext));
1344 if (XE_IOCTL_DBG(oa->xe, err))
1345 return -EFAULT;
1346
1347 if (XE_IOCTL_DBG(oa->xe, ext.pad) ||
1348 XE_IOCTL_DBG(oa->xe, ext.name >= ARRAY_SIZE(xe_oa_user_extension_funcs)))
1349 return -EINVAL;
1350
1351 idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_oa_user_extension_funcs));
1352 err = xe_oa_user_extension_funcs[idx](oa, from, extension, param);
1353 if (XE_IOCTL_DBG(oa->xe, err))
1354 return err;
1355
1356 if (ext.next_extension)
1357 return xe_oa_user_extensions(oa, from, ext.next_extension, ++ext_number, param);
1358
1359 return 0;
1360}
1361
1362static int xe_oa_parse_syncs(struct xe_oa *oa, struct xe_oa_open_param *param)
1363{
1364 int ret, num_syncs, num_ufence = 0;
1365
1366 if (param->num_syncs && !param->syncs_user) {
1367 drm_dbg(&oa->xe->drm, "num_syncs specified without sync array\n");
1368 ret = -EINVAL;
1369 goto exit;
1370 }
1371
1372 if (param->num_syncs) {
1373 param->syncs = kcalloc(param->num_syncs, sizeof(*param->syncs), GFP_KERNEL);
1374 if (!param->syncs) {
1375 ret = -ENOMEM;
1376 goto exit;
1377 }
1378 }
1379
1380 for (num_syncs = 0; num_syncs < param->num_syncs; num_syncs++) {
1381 ret = xe_sync_entry_parse(oa->xe, param->xef, ¶m->syncs[num_syncs],
1382 ¶m->syncs_user[num_syncs], 0);
1383 if (ret)
1384 goto err_syncs;
1385
1386 if (xe_sync_is_ufence(¶m->syncs[num_syncs]))
1387 num_ufence++;
1388 }
1389
1390 if (XE_IOCTL_DBG(oa->xe, num_ufence > 1)) {
1391 ret = -EINVAL;
1392 goto err_syncs;
1393 }
1394
1395 return 0;
1396
1397err_syncs:
1398 while (num_syncs--)
1399 xe_sync_entry_cleanup(¶m->syncs[num_syncs]);
1400 kfree(param->syncs);
1401exit:
1402 return ret;
1403}
1404
1405static void xe_oa_stream_enable(struct xe_oa_stream *stream)
1406{
1407 stream->pollin = false;
1408
1409 xe_oa_enable(stream);
1410
1411 if (stream->sample)
1412 hrtimer_start(&stream->poll_check_timer,
1413 ns_to_ktime(stream->poll_period_ns),
1414 HRTIMER_MODE_REL_PINNED);
1415}
1416
1417static void xe_oa_stream_disable(struct xe_oa_stream *stream)
1418{
1419 xe_oa_disable(stream);
1420
1421 if (stream->sample)
1422 hrtimer_cancel(&stream->poll_check_timer);
1423}
1424
1425static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream)
1426{
1427 struct xe_exec_queue *q = stream->exec_q;
1428 int ret1, ret2;
1429
1430 /* Best effort recovery: try to revert both to original, irrespective of error */
1431 ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us);
1432 ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us);
1433 if (ret1 || ret2)
1434 goto err;
1435 return 0;
1436err:
1437 drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2);
1438 return ret1 ?: ret2;
1439}
1440
1441static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream)
1442{
1443 struct xe_exec_queue *q = stream->exec_q;
1444 int ret;
1445
1446 /* Setting values to 0 will disable timeslice and preempt_timeout */
1447 ret = q->ops->set_timeslice(q, 0);
1448 if (ret)
1449 goto err;
1450
1451 ret = q->ops->set_preempt_timeout(q, 0);
1452 if (ret)
1453 goto err;
1454
1455 return 0;
1456err:
1457 xe_oa_enable_preempt_timeslice(stream);
1458 drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret);
1459 return ret;
1460}
1461
1462static int xe_oa_enable_locked(struct xe_oa_stream *stream)
1463{
1464 if (stream->enabled)
1465 return 0;
1466
1467 if (stream->no_preempt) {
1468 int ret = xe_oa_disable_preempt_timeslice(stream);
1469
1470 if (ret)
1471 return ret;
1472 }
1473
1474 xe_oa_stream_enable(stream);
1475
1476 stream->enabled = true;
1477 return 0;
1478}
1479
1480static int xe_oa_disable_locked(struct xe_oa_stream *stream)
1481{
1482 int ret = 0;
1483
1484 if (!stream->enabled)
1485 return 0;
1486
1487 xe_oa_stream_disable(stream);
1488
1489 if (stream->no_preempt)
1490 ret = xe_oa_enable_preempt_timeslice(stream);
1491
1492 stream->enabled = false;
1493 return ret;
1494}
1495
1496static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg)
1497{
1498 struct xe_oa_open_param param = {};
1499 long ret = stream->oa_config->id;
1500 struct xe_oa_config *config;
1501 int err;
1502
1503 err = xe_oa_user_extensions(stream->oa, XE_OA_USER_EXTN_FROM_CONFIG, arg, 0, ¶m);
1504 if (err)
1505 return err;
1506
1507 config = xe_oa_get_oa_config(stream->oa, param.metric_set);
1508 if (!config)
1509 return -ENODEV;
1510
1511 param.xef = stream->xef;
1512 err = xe_oa_parse_syncs(stream->oa, ¶m);
1513 if (err)
1514 goto err_config_put;
1515
1516 stream->num_syncs = param.num_syncs;
1517 stream->syncs = param.syncs;
1518
1519 err = xe_oa_emit_oa_config(stream, config);
1520 if (!err) {
1521 config = xchg(&stream->oa_config, config);
1522 drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n",
1523 stream->oa_config->uuid);
1524 }
1525
1526err_config_put:
1527 xe_oa_config_put(config);
1528
1529 return err ?: ret;
1530}
1531
1532static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg)
1533{
1534 struct drm_xe_oa_stream_status status = {};
1535 void __user *uaddr = (void __user *)arg;
1536
1537 /* Map from register to uapi bits */
1538 if (stream->oa_status & OASTATUS_REPORT_LOST)
1539 status.oa_status |= DRM_XE_OASTATUS_REPORT_LOST;
1540 if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW)
1541 status.oa_status |= DRM_XE_OASTATUS_BUFFER_OVERFLOW;
1542 if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW)
1543 status.oa_status |= DRM_XE_OASTATUS_COUNTER_OVERFLOW;
1544 if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL)
1545 status.oa_status |= DRM_XE_OASTATUS_MMIO_TRG_Q_FULL;
1546
1547 if (copy_to_user(uaddr, &status, sizeof(status)))
1548 return -EFAULT;
1549
1550 return 0;
1551}
1552
1553static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg)
1554{
1555 struct drm_xe_oa_stream_info info = { .oa_buf_size = stream->oa_buffer.bo->size, };
1556 void __user *uaddr = (void __user *)arg;
1557
1558 if (copy_to_user(uaddr, &info, sizeof(info)))
1559 return -EFAULT;
1560
1561 return 0;
1562}
1563
1564static long xe_oa_ioctl_locked(struct xe_oa_stream *stream,
1565 unsigned int cmd,
1566 unsigned long arg)
1567{
1568 switch (cmd) {
1569 case DRM_XE_OBSERVATION_IOCTL_ENABLE:
1570 return xe_oa_enable_locked(stream);
1571 case DRM_XE_OBSERVATION_IOCTL_DISABLE:
1572 return xe_oa_disable_locked(stream);
1573 case DRM_XE_OBSERVATION_IOCTL_CONFIG:
1574 return xe_oa_config_locked(stream, arg);
1575 case DRM_XE_OBSERVATION_IOCTL_STATUS:
1576 return xe_oa_status_locked(stream, arg);
1577 case DRM_XE_OBSERVATION_IOCTL_INFO:
1578 return xe_oa_info_locked(stream, arg);
1579 }
1580
1581 return -EINVAL;
1582}
1583
1584static long xe_oa_ioctl(struct file *file,
1585 unsigned int cmd,
1586 unsigned long arg)
1587{
1588 struct xe_oa_stream *stream = file->private_data;
1589 long ret;
1590
1591 mutex_lock(&stream->stream_lock);
1592 ret = xe_oa_ioctl_locked(stream, cmd, arg);
1593 mutex_unlock(&stream->stream_lock);
1594
1595 return ret;
1596}
1597
1598static void xe_oa_destroy_locked(struct xe_oa_stream *stream)
1599{
1600 if (stream->enabled)
1601 xe_oa_disable_locked(stream);
1602
1603 xe_oa_stream_destroy(stream);
1604
1605 if (stream->exec_q)
1606 xe_exec_queue_put(stream->exec_q);
1607
1608 kfree(stream);
1609}
1610
1611static int xe_oa_release(struct inode *inode, struct file *file)
1612{
1613 struct xe_oa_stream *stream = file->private_data;
1614 struct xe_gt *gt = stream->gt;
1615
1616 xe_pm_runtime_get(gt_to_xe(gt));
1617 mutex_lock(>->oa.gt_lock);
1618 xe_oa_destroy_locked(stream);
1619 mutex_unlock(>->oa.gt_lock);
1620 xe_pm_runtime_put(gt_to_xe(gt));
1621
1622 /* Release the reference the OA stream kept on the driver */
1623 drm_dev_put(>_to_xe(gt)->drm);
1624
1625 return 0;
1626}
1627
1628static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma)
1629{
1630 struct xe_oa_stream *stream = file->private_data;
1631 struct xe_bo *bo = stream->oa_buffer.bo;
1632 unsigned long start = vma->vm_start;
1633 int i, ret;
1634
1635 if (xe_observation_paranoid && !perfmon_capable()) {
1636 drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n");
1637 return -EACCES;
1638 }
1639
1640 /* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */
1641 if (vma->vm_end - vma->vm_start != stream->oa_buffer.bo->size) {
1642 drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n");
1643 return -EINVAL;
1644 }
1645
1646 /*
1647 * Only support VM_READ, enforce MAP_PRIVATE by checking for
1648 * VM_MAYSHARE, don't copy the vma on fork
1649 */
1650 if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_SHARED | VM_MAYSHARE)) {
1651 drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n");
1652 return -EINVAL;
1653 }
1654 vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY,
1655 VM_MAYWRITE | VM_MAYEXEC);
1656
1657 xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma));
1658 for (i = 0; i < bo->ttm.ttm->num_pages; i++) {
1659 ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]),
1660 PAGE_SIZE, vma->vm_page_prot);
1661 if (ret)
1662 break;
1663
1664 start += PAGE_SIZE;
1665 }
1666
1667 return ret;
1668}
1669
1670static const struct file_operations xe_oa_fops = {
1671 .owner = THIS_MODULE,
1672 .release = xe_oa_release,
1673 .poll = xe_oa_poll,
1674 .read = xe_oa_read,
1675 .unlocked_ioctl = xe_oa_ioctl,
1676 .mmap = xe_oa_mmap,
1677};
1678
1679static bool engine_supports_mi_query(struct xe_hw_engine *hwe)
1680{
1681 return hwe->class == XE_ENGINE_CLASS_RENDER ||
1682 hwe->class == XE_ENGINE_CLASS_COMPUTE;
1683}
1684
1685static bool xe_oa_find_reg_in_lri(u32 *state, u32 reg, u32 *offset, u32 end)
1686{
1687 u32 idx = *offset;
1688 u32 len = min(MI_LRI_LEN(state[idx]) + idx, end);
1689 bool found = false;
1690
1691 idx++;
1692 for (; idx < len; idx += 2) {
1693 if (state[idx] == reg) {
1694 found = true;
1695 break;
1696 }
1697 }
1698
1699 *offset = idx;
1700 return found;
1701}
1702
1703#define IS_MI_LRI_CMD(x) (REG_FIELD_GET(MI_OPCODE, (x)) == \
1704 REG_FIELD_GET(MI_OPCODE, MI_LOAD_REGISTER_IMM))
1705
1706static u32 xe_oa_context_image_offset(struct xe_oa_stream *stream, u32 reg)
1707{
1708 struct xe_lrc *lrc = stream->exec_q->lrc[0];
1709 u32 len = (xe_gt_lrc_size(stream->gt, stream->hwe->class) +
1710 lrc->ring.size) / sizeof(u32);
1711 u32 offset = xe_lrc_regs_offset(lrc) / sizeof(u32);
1712 u32 *state = (u32 *)lrc->bo->vmap.vaddr;
1713
1714 if (drm_WARN_ON(&stream->oa->xe->drm, !state))
1715 return U32_MAX;
1716
1717 for (; offset < len; ) {
1718 if (IS_MI_LRI_CMD(state[offset])) {
1719 /*
1720 * We expect reg-value pairs in MI_LRI command, so
1721 * MI_LRI_LEN() should be even
1722 */
1723 drm_WARN_ON(&stream->oa->xe->drm,
1724 MI_LRI_LEN(state[offset]) & 0x1);
1725
1726 if (xe_oa_find_reg_in_lri(state, reg, &offset, len))
1727 break;
1728 } else {
1729 offset++;
1730 }
1731 }
1732
1733 return offset < len ? offset : U32_MAX;
1734}
1735
1736static int xe_oa_set_ctx_ctrl_offset(struct xe_oa_stream *stream)
1737{
1738 struct xe_reg reg = OACTXCONTROL(stream->hwe->mmio_base);
1739 u32 offset = stream->oa->ctx_oactxctrl_offset[stream->hwe->class];
1740
1741 /* Do this only once. Failure is stored as offset of U32_MAX */
1742 if (offset)
1743 goto exit;
1744
1745 offset = xe_oa_context_image_offset(stream, reg.addr);
1746 stream->oa->ctx_oactxctrl_offset[stream->hwe->class] = offset;
1747
1748 drm_dbg(&stream->oa->xe->drm, "%s oa ctx control at 0x%08x dword offset\n",
1749 stream->hwe->name, offset);
1750exit:
1751 return offset && offset != U32_MAX ? 0 : -ENODEV;
1752}
1753
1754static int xe_oa_stream_init(struct xe_oa_stream *stream,
1755 struct xe_oa_open_param *param)
1756{
1757 struct xe_oa_unit *u = param->hwe->oa_unit;
1758 struct xe_gt *gt = param->hwe->gt;
1759 unsigned int fw_ref;
1760 int ret;
1761
1762 stream->exec_q = param->exec_q;
1763 stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS;
1764 stream->hwe = param->hwe;
1765 stream->gt = stream->hwe->gt;
1766 stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format];
1767
1768 stream->sample = param->sample;
1769 stream->periodic = param->period_exponent >= 0;
1770 stream->period_exponent = param->period_exponent;
1771 stream->no_preempt = param->no_preempt;
1772 stream->wait_num_reports = param->wait_num_reports;
1773
1774 stream->xef = xe_file_get(param->xef);
1775 stream->num_syncs = param->num_syncs;
1776 stream->syncs = param->syncs;
1777
1778 /*
1779 * For Xe2+, when overrun mode is enabled, there are no partial reports at the end
1780 * of buffer, making the OA buffer effectively a non-power-of-2 size circular
1781 * buffer whose size, circ_size, is a multiple of the report size
1782 */
1783 if (GRAPHICS_VER(stream->oa->xe) >= 20 &&
1784 stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample)
1785 stream->oa_buffer.circ_size =
1786 param->oa_buffer_size -
1787 param->oa_buffer_size % stream->oa_buffer.format->size;
1788 else
1789 stream->oa_buffer.circ_size = param->oa_buffer_size;
1790
1791 if (stream->exec_q && engine_supports_mi_query(stream->hwe)) {
1792 /* If we don't find the context offset, just return error */
1793 ret = xe_oa_set_ctx_ctrl_offset(stream);
1794 if (ret) {
1795 drm_err(&stream->oa->xe->drm,
1796 "xe_oa_set_ctx_ctrl_offset failed for %s\n",
1797 stream->hwe->name);
1798 goto exit;
1799 }
1800 }
1801
1802 stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set);
1803 if (!stream->oa_config) {
1804 drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set);
1805 ret = -EINVAL;
1806 goto exit;
1807 }
1808
1809 /*
1810 * Wa_1509372804:pvc
1811 *
1812 * GuC reset of engines causes OA to lose configuration
1813 * state. Prevent this by overriding GUCRC mode.
1814 */
1815 if (stream->oa->xe->info.platform == XE_PVC) {
1816 ret = xe_guc_pc_override_gucrc_mode(>->uc.guc.pc,
1817 SLPC_GUCRC_MODE_GUCRC_NO_RC6);
1818 if (ret)
1819 goto err_free_configs;
1820
1821 stream->override_gucrc = true;
1822 }
1823
1824 /* Take runtime pm ref and forcewake to disable RC6 */
1825 xe_pm_runtime_get(stream->oa->xe);
1826 fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
1827 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
1828 ret = -ETIMEDOUT;
1829 goto err_fw_put;
1830 }
1831
1832 ret = xe_oa_alloc_oa_buffer(stream, param->oa_buffer_size);
1833 if (ret)
1834 goto err_fw_put;
1835
1836 stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL,
1837 BIT(stream->hwe->logical_instance), 1,
1838 stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
1839 if (IS_ERR(stream->k_exec_q)) {
1840 ret = PTR_ERR(stream->k_exec_q);
1841 drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d",
1842 stream->gt->info.id, stream->hwe->name, ret);
1843 goto err_free_oa_buf;
1844 }
1845
1846 ret = xe_oa_enable_metric_set(stream);
1847 if (ret) {
1848 drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n");
1849 goto err_put_k_exec_q;
1850 }
1851
1852 drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n",
1853 stream->oa_config->uuid);
1854
1855 WRITE_ONCE(u->exclusive_stream, stream);
1856
1857 hrtimer_init(&stream->poll_check_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1858 stream->poll_check_timer.function = xe_oa_poll_check_timer_cb;
1859 init_waitqueue_head(&stream->poll_wq);
1860
1861 spin_lock_init(&stream->oa_buffer.ptr_lock);
1862 mutex_init(&stream->stream_lock);
1863
1864 return 0;
1865
1866err_put_k_exec_q:
1867 xe_oa_disable_metric_set(stream);
1868 xe_exec_queue_put(stream->k_exec_q);
1869err_free_oa_buf:
1870 xe_oa_free_oa_buffer(stream);
1871err_fw_put:
1872 xe_force_wake_put(gt_to_fw(gt), fw_ref);
1873 xe_pm_runtime_put(stream->oa->xe);
1874 if (stream->override_gucrc)
1875 xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(>->uc.guc.pc));
1876err_free_configs:
1877 xe_oa_free_configs(stream);
1878exit:
1879 xe_file_put(stream->xef);
1880 return ret;
1881}
1882
1883static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa,
1884 struct xe_oa_open_param *param)
1885{
1886 struct xe_oa_stream *stream;
1887 int stream_fd;
1888 int ret;
1889
1890 /* We currently only allow exclusive access */
1891 if (param->hwe->oa_unit->exclusive_stream) {
1892 drm_dbg(&oa->xe->drm, "OA unit already in use\n");
1893 ret = -EBUSY;
1894 goto exit;
1895 }
1896
1897 stream = kzalloc(sizeof(*stream), GFP_KERNEL);
1898 if (!stream) {
1899 ret = -ENOMEM;
1900 goto exit;
1901 }
1902
1903 stream->oa = oa;
1904 ret = xe_oa_stream_init(stream, param);
1905 if (ret)
1906 goto err_free;
1907
1908 if (!param->disabled) {
1909 ret = xe_oa_enable_locked(stream);
1910 if (ret)
1911 goto err_destroy;
1912 }
1913
1914 stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0);
1915 if (stream_fd < 0) {
1916 ret = stream_fd;
1917 goto err_disable;
1918 }
1919
1920 /* Hold a reference on the drm device till stream_fd is released */
1921 drm_dev_get(&stream->oa->xe->drm);
1922
1923 return stream_fd;
1924err_disable:
1925 if (!param->disabled)
1926 xe_oa_disable_locked(stream);
1927err_destroy:
1928 xe_oa_stream_destroy(stream);
1929err_free:
1930 kfree(stream);
1931exit:
1932 return ret;
1933}
1934
1935/**
1936 * xe_oa_timestamp_frequency - Return OA timestamp frequency
1937 * @gt: @xe_gt
1938 *
1939 * OA timestamp frequency = CS timestamp frequency in most platforms. On some
1940 * platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such
1941 * cases, return the adjusted CS timestamp frequency to the user.
1942 */
1943u32 xe_oa_timestamp_frequency(struct xe_gt *gt)
1944{
1945 u32 reg, shift;
1946
1947 /*
1948 * Wa_18013179988:dg2
1949 * Wa_14015568240:pvc
1950 * Wa_14015846243:mtl
1951 */
1952 switch (gt_to_xe(gt)->info.platform) {
1953 case XE_DG2:
1954 case XE_PVC:
1955 case XE_METEORLAKE:
1956 xe_pm_runtime_get(gt_to_xe(gt));
1957 reg = xe_mmio_read32(>->mmio, RPM_CONFIG0);
1958 xe_pm_runtime_put(gt_to_xe(gt));
1959
1960 shift = REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg);
1961 return gt->info.reference_clock << (3 - shift);
1962
1963 default:
1964 return gt->info.reference_clock;
1965 }
1966}
1967
1968static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent)
1969{
1970 u64 nom = (2ULL << exponent) * NSEC_PER_SEC;
1971 u32 den = xe_oa_timestamp_frequency(gt);
1972
1973 return div_u64(nom + den - 1, den);
1974}
1975
1976static bool engine_supports_oa_format(const struct xe_hw_engine *hwe, int type)
1977{
1978 switch (hwe->oa_unit->type) {
1979 case DRM_XE_OA_UNIT_TYPE_OAG:
1980 return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR ||
1981 type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC;
1982 case DRM_XE_OA_UNIT_TYPE_OAM:
1983 return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC;
1984 default:
1985 return false;
1986 }
1987}
1988
1989/**
1990 * xe_oa_unit_id - Return OA unit ID for a hardware engine
1991 * @hwe: @xe_hw_engine
1992 *
1993 * Return OA unit ID for a hardware engine when available
1994 */
1995u16 xe_oa_unit_id(struct xe_hw_engine *hwe)
1996{
1997 return hwe->oa_unit && hwe->oa_unit->num_engines ?
1998 hwe->oa_unit->oa_unit_id : U16_MAX;
1999}
2000
2001static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param)
2002{
2003 struct xe_gt *gt;
2004 int i, ret = 0;
2005
2006 if (param->exec_q) {
2007 /* When we have an exec_q, get hwe from the exec_q */
2008 param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class,
2009 param->engine_instance, true);
2010 } else {
2011 struct xe_hw_engine *hwe;
2012 enum xe_hw_engine_id id;
2013
2014 /* Else just get the first hwe attached to the oa unit */
2015 for_each_gt(gt, oa->xe, i) {
2016 for_each_hw_engine(hwe, gt, id) {
2017 if (xe_oa_unit_id(hwe) == param->oa_unit_id) {
2018 param->hwe = hwe;
2019 goto out;
2020 }
2021 }
2022 }
2023 }
2024out:
2025 if (!param->hwe || xe_oa_unit_id(param->hwe) != param->oa_unit_id) {
2026 drm_dbg(&oa->xe->drm, "Unable to find hwe (%d, %d) for OA unit ID %d\n",
2027 param->exec_q ? param->exec_q->class : -1,
2028 param->engine_instance, param->oa_unit_id);
2029 ret = -EINVAL;
2030 }
2031
2032 return ret;
2033}
2034
2035/**
2036 * xe_oa_stream_open_ioctl - Opens an OA stream
2037 * @dev: @drm_device
2038 * @data: pointer to struct @drm_xe_oa_config
2039 * @file: @drm_file
2040 *
2041 * The functions opens an OA stream. An OA stream, opened with specified
2042 * properties, enables OA counter samples to be collected, either
2043 * periodically (time based sampling), or on request (using OA queries)
2044 */
2045int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
2046{
2047 struct xe_device *xe = to_xe_device(dev);
2048 struct xe_oa *oa = &xe->oa;
2049 struct xe_file *xef = to_xe_file(file);
2050 struct xe_oa_open_param param = {};
2051 const struct xe_oa_format *f;
2052 bool privileged_op = true;
2053 int ret;
2054
2055 if (!oa->xe) {
2056 drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
2057 return -ENODEV;
2058 }
2059
2060 param.xef = xef;
2061 param.period_exponent = -1;
2062 ret = xe_oa_user_extensions(oa, XE_OA_USER_EXTN_FROM_OPEN, data, 0, ¶m);
2063 if (ret)
2064 return ret;
2065
2066 if (param.exec_queue_id > 0) {
2067 param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id);
2068 if (XE_IOCTL_DBG(oa->xe, !param.exec_q))
2069 return -ENOENT;
2070
2071 if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1))
2072 return -EOPNOTSUPP;
2073 }
2074
2075 /*
2076 * Query based sampling (using MI_REPORT_PERF_COUNT) with OAR/OAC,
2077 * without global stream access, can be an unprivileged operation
2078 */
2079 if (param.exec_q && !param.sample)
2080 privileged_op = false;
2081
2082 if (param.no_preempt) {
2083 if (!param.exec_q) {
2084 drm_dbg(&oa->xe->drm, "Preemption disable without exec_q!\n");
2085 ret = -EINVAL;
2086 goto err_exec_q;
2087 }
2088 privileged_op = true;
2089 }
2090
2091 if (privileged_op && xe_observation_paranoid && !perfmon_capable()) {
2092 drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n");
2093 ret = -EACCES;
2094 goto err_exec_q;
2095 }
2096
2097 if (!param.exec_q && !param.sample) {
2098 drm_dbg(&oa->xe->drm, "Only OA report sampling supported\n");
2099 ret = -EINVAL;
2100 goto err_exec_q;
2101 }
2102
2103 ret = xe_oa_assign_hwe(oa, ¶m);
2104 if (ret)
2105 goto err_exec_q;
2106
2107 f = &oa->oa_formats[param.oa_format];
2108 if (!param.oa_format || !f->size ||
2109 !engine_supports_oa_format(param.hwe, f->type)) {
2110 drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n",
2111 param.oa_format, f->type, f->size, param.hwe->class);
2112 ret = -EINVAL;
2113 goto err_exec_q;
2114 }
2115
2116 if (param.period_exponent >= 0) {
2117 u64 oa_period, oa_freq_hz;
2118
2119 /* Requesting samples from OAG buffer is a privileged operation */
2120 if (!param.sample) {
2121 drm_dbg(&oa->xe->drm, "OA_EXPONENT specified without SAMPLE_OA\n");
2122 ret = -EINVAL;
2123 goto err_exec_q;
2124 }
2125 oa_period = oa_exponent_to_ns(param.hwe->gt, param.period_exponent);
2126 oa_freq_hz = div64_u64(NSEC_PER_SEC, oa_period);
2127 drm_dbg(&oa->xe->drm, "Using periodic sampling freq %lld Hz\n", oa_freq_hz);
2128 }
2129
2130 if (!param.oa_buffer_size)
2131 param.oa_buffer_size = DEFAULT_XE_OA_BUFFER_SIZE;
2132
2133 if (!param.wait_num_reports)
2134 param.wait_num_reports = 1;
2135 if (param.wait_num_reports > param.oa_buffer_size / f->size) {
2136 drm_dbg(&oa->xe->drm, "wait_num_reports %d\n", param.wait_num_reports);
2137 ret = -EINVAL;
2138 goto err_exec_q;
2139 }
2140
2141 ret = xe_oa_parse_syncs(oa, ¶m);
2142 if (ret)
2143 goto err_exec_q;
2144
2145 mutex_lock(¶m.hwe->gt->oa.gt_lock);
2146 ret = xe_oa_stream_open_ioctl_locked(oa, ¶m);
2147 mutex_unlock(¶m.hwe->gt->oa.gt_lock);
2148 if (ret < 0)
2149 goto err_sync_cleanup;
2150
2151 return ret;
2152
2153err_sync_cleanup:
2154 while (param.num_syncs--)
2155 xe_sync_entry_cleanup(¶m.syncs[param.num_syncs]);
2156 kfree(param.syncs);
2157err_exec_q:
2158 if (param.exec_q)
2159 xe_exec_queue_put(param.exec_q);
2160 return ret;
2161}
2162
2163static bool xe_oa_is_valid_flex_addr(struct xe_oa *oa, u32 addr)
2164{
2165 static const struct xe_reg flex_eu_regs[] = {
2166 EU_PERF_CNTL0,
2167 EU_PERF_CNTL1,
2168 EU_PERF_CNTL2,
2169 EU_PERF_CNTL3,
2170 EU_PERF_CNTL4,
2171 EU_PERF_CNTL5,
2172 EU_PERF_CNTL6,
2173 };
2174 int i;
2175
2176 for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) {
2177 if (flex_eu_regs[i].addr == addr)
2178 return true;
2179 }
2180 return false;
2181}
2182
2183static bool xe_oa_reg_in_range_table(u32 addr, const struct xe_mmio_range *table)
2184{
2185 while (table->start && table->end) {
2186 if (addr >= table->start && addr <= table->end)
2187 return true;
2188
2189 table++;
2190 }
2191
2192 return false;
2193}
2194
2195static const struct xe_mmio_range xehp_oa_b_counters[] = {
2196 { .start = 0xdc48, .end = 0xdc48 }, /* OAA_ENABLE_REG */
2197 { .start = 0xdd00, .end = 0xdd48 }, /* OAG_LCE0_0 - OAA_LENABLE_REG */
2198 {}
2199};
2200
2201static const struct xe_mmio_range gen12_oa_b_counters[] = {
2202 { .start = 0x2b2c, .end = 0x2b2c }, /* OAG_OA_PESS */
2203 { .start = 0xd900, .end = 0xd91c }, /* OAG_OASTARTTRIG[1-8] */
2204 { .start = 0xd920, .end = 0xd93c }, /* OAG_OAREPORTTRIG1[1-8] */
2205 { .start = 0xd940, .end = 0xd97c }, /* OAG_CEC[0-7][0-1] */
2206 { .start = 0xdc00, .end = 0xdc3c }, /* OAG_SCEC[0-7][0-1] */
2207 { .start = 0xdc40, .end = 0xdc40 }, /* OAG_SPCTR_CNF */
2208 { .start = 0xdc44, .end = 0xdc44 }, /* OAA_DBG_REG */
2209 {}
2210};
2211
2212static const struct xe_mmio_range mtl_oam_b_counters[] = {
2213 { .start = 0x393000, .end = 0x39301c }, /* OAM_STARTTRIG1[1-8] */
2214 { .start = 0x393020, .end = 0x39303c }, /* OAM_REPORTTRIG1[1-8] */
2215 { .start = 0x393040, .end = 0x39307c }, /* OAM_CEC[0-7][0-1] */
2216 { .start = 0x393200, .end = 0x39323C }, /* MPES[0-7] */
2217 {}
2218};
2219
2220static const struct xe_mmio_range xe2_oa_b_counters[] = {
2221 { .start = 0x393200, .end = 0x39323C }, /* MPES_0_MPES_SAG - MPES_7_UPPER_MPES_SAG */
2222 { .start = 0x394200, .end = 0x39423C }, /* MPES_0_MPES_SCMI0 - MPES_7_UPPER_MPES_SCMI0 */
2223 { .start = 0x394A00, .end = 0x394A3C }, /* MPES_0_MPES_SCMI1 - MPES_7_UPPER_MPES_SCMI1 */
2224 {},
2225};
2226
2227static bool xe_oa_is_valid_b_counter_addr(struct xe_oa *oa, u32 addr)
2228{
2229 return xe_oa_reg_in_range_table(addr, xehp_oa_b_counters) ||
2230 xe_oa_reg_in_range_table(addr, gen12_oa_b_counters) ||
2231 xe_oa_reg_in_range_table(addr, mtl_oam_b_counters) ||
2232 (GRAPHICS_VER(oa->xe) >= 20 &&
2233 xe_oa_reg_in_range_table(addr, xe2_oa_b_counters));
2234}
2235
2236static const struct xe_mmio_range mtl_oa_mux_regs[] = {
2237 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
2238 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
2239 { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */
2240 { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */
2241 { .start = 0x38d100, .end = 0x38d114}, /* VISACTL */
2242 {}
2243};
2244
2245static const struct xe_mmio_range gen12_oa_mux_regs[] = {
2246 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
2247 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
2248 { .start = 0x9840, .end = 0x9840 }, /* GDT_CHICKEN_BITS */
2249 { .start = 0x9884, .end = 0x9888 }, /* NOA_WRITE */
2250 { .start = 0x20cc, .end = 0x20cc }, /* WAIT_FOR_RC6_EXIT */
2251 {}
2252};
2253
2254static const struct xe_mmio_range xe2_oa_mux_regs[] = {
2255 { .start = 0x5194, .end = 0x5194 }, /* SYS_MEM_LAT_MEASURE_MERTF_GRP_3D */
2256 { .start = 0x8704, .end = 0x8704 }, /* LMEM_LAT_MEASURE_MCFG_GRP */
2257 { .start = 0xB1BC, .end = 0xB1BC }, /* L3_BANK_LAT_MEASURE_LBCF_GFX */
2258 { .start = 0xD0E0, .end = 0xD0F4 }, /* VISACTL */
2259 { .start = 0xE18C, .end = 0xE18C }, /* SAMPLER_MODE */
2260 { .start = 0xE590, .end = 0xE590 }, /* TDL_LSC_LAT_MEASURE_TDL_GFX */
2261 { .start = 0x13000, .end = 0x137FC }, /* PES_0_PESL0 - PES_63_UPPER_PESL3 */
2262 {},
2263};
2264
2265static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr)
2266{
2267 if (GRAPHICS_VER(oa->xe) >= 20)
2268 return xe_oa_reg_in_range_table(addr, xe2_oa_mux_regs);
2269 else if (GRAPHICS_VERx100(oa->xe) >= 1270)
2270 return xe_oa_reg_in_range_table(addr, mtl_oa_mux_regs);
2271 else
2272 return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs);
2273}
2274
2275static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr)
2276{
2277 return xe_oa_is_valid_flex_addr(oa, addr) ||
2278 xe_oa_is_valid_b_counter_addr(oa, addr) ||
2279 xe_oa_is_valid_mux_addr(oa, addr);
2280}
2281
2282static struct xe_oa_reg *
2283xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr),
2284 u32 __user *regs, u32 n_regs)
2285{
2286 struct xe_oa_reg *oa_regs;
2287 int err;
2288 u32 i;
2289
2290 oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL);
2291 if (!oa_regs)
2292 return ERR_PTR(-ENOMEM);
2293
2294 for (i = 0; i < n_regs; i++) {
2295 u32 addr, value;
2296
2297 err = get_user(addr, regs);
2298 if (err)
2299 goto addr_err;
2300
2301 if (!is_valid(oa, addr)) {
2302 drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr);
2303 err = -EINVAL;
2304 goto addr_err;
2305 }
2306
2307 err = get_user(value, regs + 1);
2308 if (err)
2309 goto addr_err;
2310
2311 oa_regs[i].addr = XE_REG(addr);
2312 oa_regs[i].value = value;
2313
2314 regs += 2;
2315 }
2316
2317 return oa_regs;
2318
2319addr_err:
2320 kfree(oa_regs);
2321 return ERR_PTR(err);
2322}
2323
2324static ssize_t show_dynamic_id(struct kobject *kobj,
2325 struct kobj_attribute *attr,
2326 char *buf)
2327{
2328 struct xe_oa_config *oa_config =
2329 container_of(attr, typeof(*oa_config), sysfs_metric_id);
2330
2331 return sysfs_emit(buf, "%d\n", oa_config->id);
2332}
2333
2334static int create_dynamic_oa_sysfs_entry(struct xe_oa *oa,
2335 struct xe_oa_config *oa_config)
2336{
2337 sysfs_attr_init(&oa_config->sysfs_metric_id.attr);
2338 oa_config->sysfs_metric_id.attr.name = "id";
2339 oa_config->sysfs_metric_id.attr.mode = 0444;
2340 oa_config->sysfs_metric_id.show = show_dynamic_id;
2341 oa_config->sysfs_metric_id.store = NULL;
2342
2343 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr;
2344 oa_config->attrs[1] = NULL;
2345
2346 oa_config->sysfs_metric.name = oa_config->uuid;
2347 oa_config->sysfs_metric.attrs = oa_config->attrs;
2348
2349 return sysfs_create_group(oa->metrics_kobj, &oa_config->sysfs_metric);
2350}
2351
2352/**
2353 * xe_oa_add_config_ioctl - Adds one OA config
2354 * @dev: @drm_device
2355 * @data: pointer to struct @drm_xe_oa_config
2356 * @file: @drm_file
2357 *
2358 * The functions adds an OA config to the set of OA configs maintained in
2359 * the kernel. The config determines which OA metrics are collected for an
2360 * OA stream.
2361 */
2362int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
2363{
2364 struct xe_device *xe = to_xe_device(dev);
2365 struct xe_oa *oa = &xe->oa;
2366 struct drm_xe_oa_config param;
2367 struct drm_xe_oa_config *arg = ¶m;
2368 struct xe_oa_config *oa_config, *tmp;
2369 struct xe_oa_reg *regs;
2370 int err, id;
2371
2372 if (!oa->xe) {
2373 drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
2374 return -ENODEV;
2375 }
2376
2377 if (xe_observation_paranoid && !perfmon_capable()) {
2378 drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n");
2379 return -EACCES;
2380 }
2381
2382 err = __copy_from_user(¶m, u64_to_user_ptr(data), sizeof(param));
2383 if (XE_IOCTL_DBG(oa->xe, err))
2384 return -EFAULT;
2385
2386 if (XE_IOCTL_DBG(oa->xe, arg->extensions) ||
2387 XE_IOCTL_DBG(oa->xe, !arg->regs_ptr) ||
2388 XE_IOCTL_DBG(oa->xe, !arg->n_regs))
2389 return -EINVAL;
2390
2391 oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL);
2392 if (!oa_config)
2393 return -ENOMEM;
2394
2395 oa_config->oa = oa;
2396 kref_init(&oa_config->ref);
2397
2398 if (!uuid_is_valid(arg->uuid)) {
2399 drm_dbg(&oa->xe->drm, "Invalid uuid format for OA config\n");
2400 err = -EINVAL;
2401 goto reg_err;
2402 }
2403
2404 /* Last character in oa_config->uuid will be 0 because oa_config is kzalloc */
2405 memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid));
2406
2407 oa_config->regs_len = arg->n_regs;
2408 regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr,
2409 u64_to_user_ptr(arg->regs_ptr),
2410 arg->n_regs);
2411 if (IS_ERR(regs)) {
2412 drm_dbg(&oa->xe->drm, "Failed to create OA config for mux_regs\n");
2413 err = PTR_ERR(regs);
2414 goto reg_err;
2415 }
2416 oa_config->regs = regs;
2417
2418 err = mutex_lock_interruptible(&oa->metrics_lock);
2419 if (err)
2420 goto reg_err;
2421
2422 /* We shouldn't have too many configs, so this iteration shouldn't be too costly */
2423 idr_for_each_entry(&oa->metrics_idr, tmp, id) {
2424 if (!strcmp(tmp->uuid, oa_config->uuid)) {
2425 drm_dbg(&oa->xe->drm, "OA config already exists with this uuid\n");
2426 err = -EADDRINUSE;
2427 goto sysfs_err;
2428 }
2429 }
2430
2431 err = create_dynamic_oa_sysfs_entry(oa, oa_config);
2432 if (err) {
2433 drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n");
2434 goto sysfs_err;
2435 }
2436
2437 oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL);
2438 if (oa_config->id < 0) {
2439 drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n");
2440 err = oa_config->id;
2441 goto sysfs_err;
2442 }
2443
2444 mutex_unlock(&oa->metrics_lock);
2445
2446 drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, oa_config->id);
2447
2448 return oa_config->id;
2449
2450sysfs_err:
2451 mutex_unlock(&oa->metrics_lock);
2452reg_err:
2453 xe_oa_config_put(oa_config);
2454 drm_dbg(&oa->xe->drm, "Failed to add new OA config\n");
2455 return err;
2456}
2457
2458/**
2459 * xe_oa_remove_config_ioctl - Removes one OA config
2460 * @dev: @drm_device
2461 * @data: pointer to struct @drm_xe_observation_param
2462 * @file: @drm_file
2463 */
2464int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
2465{
2466 struct xe_device *xe = to_xe_device(dev);
2467 struct xe_oa *oa = &xe->oa;
2468 struct xe_oa_config *oa_config;
2469 u64 arg, *ptr = u64_to_user_ptr(data);
2470 int ret;
2471
2472 if (!oa->xe) {
2473 drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
2474 return -ENODEV;
2475 }
2476
2477 if (xe_observation_paranoid && !perfmon_capable()) {
2478 drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n");
2479 return -EACCES;
2480 }
2481
2482 ret = get_user(arg, ptr);
2483 if (XE_IOCTL_DBG(oa->xe, ret))
2484 return ret;
2485
2486 ret = mutex_lock_interruptible(&oa->metrics_lock);
2487 if (ret)
2488 return ret;
2489
2490 oa_config = idr_find(&oa->metrics_idr, arg);
2491 if (!oa_config) {
2492 drm_dbg(&oa->xe->drm, "Failed to remove unknown OA config\n");
2493 ret = -ENOENT;
2494 goto err_unlock;
2495 }
2496
2497 WARN_ON(arg != oa_config->id);
2498
2499 sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric);
2500 idr_remove(&oa->metrics_idr, arg);
2501
2502 mutex_unlock(&oa->metrics_lock);
2503
2504 drm_dbg(&oa->xe->drm, "Removed config %s id=%i\n", oa_config->uuid, oa_config->id);
2505
2506 xe_oa_config_put(oa_config);
2507
2508 return 0;
2509
2510err_unlock:
2511 mutex_unlock(&oa->metrics_lock);
2512 return ret;
2513}
2514
2515/**
2516 * xe_oa_register - Xe OA registration
2517 * @xe: @xe_device
2518 *
2519 * Exposes the metrics sysfs directory upon completion of module initialization
2520 */
2521void xe_oa_register(struct xe_device *xe)
2522{
2523 struct xe_oa *oa = &xe->oa;
2524
2525 if (!oa->xe)
2526 return;
2527
2528 oa->metrics_kobj = kobject_create_and_add("metrics",
2529 &xe->drm.primary->kdev->kobj);
2530}
2531
2532/**
2533 * xe_oa_unregister - Xe OA de-registration
2534 * @xe: @xe_device
2535 */
2536void xe_oa_unregister(struct xe_device *xe)
2537{
2538 struct xe_oa *oa = &xe->oa;
2539
2540 if (!oa->metrics_kobj)
2541 return;
2542
2543 kobject_put(oa->metrics_kobj);
2544 oa->metrics_kobj = NULL;
2545}
2546
2547static u32 num_oa_units_per_gt(struct xe_gt *gt)
2548{
2549 return 1;
2550}
2551
2552static u32 __hwe_oam_unit(struct xe_hw_engine *hwe)
2553{
2554 if (GRAPHICS_VERx100(gt_to_xe(hwe->gt)) >= 1270) {
2555 /*
2556 * There's 1 SAMEDIA gt and 1 OAM per SAMEDIA gt. All media slices
2557 * within the gt use the same OAM. All MTL/LNL SKUs list 1 SA MEDIA
2558 */
2559 xe_gt_WARN_ON(hwe->gt, hwe->gt->info.type != XE_GT_TYPE_MEDIA);
2560
2561 return 0;
2562 }
2563
2564 return XE_OA_UNIT_INVALID;
2565}
2566
2567static u32 __hwe_oa_unit(struct xe_hw_engine *hwe)
2568{
2569 switch (hwe->class) {
2570 case XE_ENGINE_CLASS_RENDER:
2571 case XE_ENGINE_CLASS_COMPUTE:
2572 return 0;
2573
2574 case XE_ENGINE_CLASS_VIDEO_DECODE:
2575 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
2576 return __hwe_oam_unit(hwe);
2577
2578 default:
2579 return XE_OA_UNIT_INVALID;
2580 }
2581}
2582
2583static struct xe_oa_regs __oam_regs(u32 base)
2584{
2585 return (struct xe_oa_regs) {
2586 base,
2587 OAM_HEAD_POINTER(base),
2588 OAM_TAIL_POINTER(base),
2589 OAM_BUFFER(base),
2590 OAM_CONTEXT_CONTROL(base),
2591 OAM_CONTROL(base),
2592 OAM_DEBUG(base),
2593 OAM_STATUS(base),
2594 OAM_CONTROL_COUNTER_SEL_MASK,
2595 };
2596}
2597
2598static struct xe_oa_regs __oag_regs(void)
2599{
2600 return (struct xe_oa_regs) {
2601 0,
2602 OAG_OAHEADPTR,
2603 OAG_OATAILPTR,
2604 OAG_OABUFFER,
2605 OAG_OAGLBCTXCTRL,
2606 OAG_OACONTROL,
2607 OAG_OA_DEBUG,
2608 OAG_OASTATUS,
2609 OAG_OACONTROL_OA_COUNTER_SEL_MASK,
2610 };
2611}
2612
2613static void __xe_oa_init_oa_units(struct xe_gt *gt)
2614{
2615 const u32 mtl_oa_base[] = { 0x13000 };
2616 int i, num_units = gt->oa.num_oa_units;
2617
2618 for (i = 0; i < num_units; i++) {
2619 struct xe_oa_unit *u = >->oa.oa_unit[i];
2620
2621 if (gt->info.type != XE_GT_TYPE_MEDIA) {
2622 u->regs = __oag_regs();
2623 u->type = DRM_XE_OA_UNIT_TYPE_OAG;
2624 } else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
2625 u->regs = __oam_regs(mtl_oa_base[i]);
2626 u->type = DRM_XE_OA_UNIT_TYPE_OAM;
2627 }
2628
2629 xe_mmio_write32(>->mmio, u->regs.oa_ctrl, 0);
2630
2631 /* Ensure MMIO trigger remains disabled till there is a stream */
2632 xe_mmio_write32(>->mmio, u->regs.oa_debug,
2633 oag_configure_mmio_trigger(NULL, false));
2634
2635 /* Set oa_unit_ids now to ensure ids remain contiguous */
2636 u->oa_unit_id = gt_to_xe(gt)->oa.oa_unit_ids++;
2637 }
2638}
2639
2640static int xe_oa_init_gt(struct xe_gt *gt)
2641{
2642 u32 num_oa_units = num_oa_units_per_gt(gt);
2643 struct xe_hw_engine *hwe;
2644 enum xe_hw_engine_id id;
2645 struct xe_oa_unit *u;
2646
2647 u = drmm_kcalloc(>_to_xe(gt)->drm, num_oa_units, sizeof(*u), GFP_KERNEL);
2648 if (!u)
2649 return -ENOMEM;
2650
2651 for_each_hw_engine(hwe, gt, id) {
2652 u32 index = __hwe_oa_unit(hwe);
2653
2654 hwe->oa_unit = NULL;
2655 if (index < num_oa_units) {
2656 u[index].num_engines++;
2657 hwe->oa_unit = &u[index];
2658 }
2659 }
2660
2661 /*
2662 * Fused off engines can result in oa_unit's with num_engines == 0. These units
2663 * will appear in OA unit query, but no OA streams can be opened on them.
2664 */
2665 gt->oa.num_oa_units = num_oa_units;
2666 gt->oa.oa_unit = u;
2667
2668 __xe_oa_init_oa_units(gt);
2669
2670 drmm_mutex_init(>_to_xe(gt)->drm, >->oa.gt_lock);
2671
2672 return 0;
2673}
2674
2675static int xe_oa_init_oa_units(struct xe_oa *oa)
2676{
2677 struct xe_gt *gt;
2678 int i, ret;
2679
2680 for_each_gt(gt, oa->xe, i) {
2681 ret = xe_oa_init_gt(gt);
2682 if (ret)
2683 return ret;
2684 }
2685
2686 return 0;
2687}
2688
2689static void oa_format_add(struct xe_oa *oa, enum xe_oa_format_name format)
2690{
2691 __set_bit(format, oa->format_mask);
2692}
2693
2694static void xe_oa_init_supported_formats(struct xe_oa *oa)
2695{
2696 if (GRAPHICS_VER(oa->xe) >= 20) {
2697 /* Xe2+ */
2698 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8);
2699 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8);
2700 oa_format_add(oa, XE_OA_FORMAT_PEC64u64);
2701 oa_format_add(oa, XE_OA_FORMAT_PEC64u64_B8_C8);
2702 oa_format_add(oa, XE_OA_FORMAT_PEC64u32);
2703 oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G1);
2704 oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G1);
2705 oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G2);
2706 oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G2);
2707 oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_32_G2_4);
2708 oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_4_G2_32);
2709 } else if (GRAPHICS_VERx100(oa->xe) >= 1270) {
2710 /* XE_METEORLAKE */
2711 oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8);
2712 oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8);
2713 oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8);
2714 oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8);
2715 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8);
2716 oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8);
2717 } else if (GRAPHICS_VERx100(oa->xe) >= 1255) {
2718 /* XE_DG2, XE_PVC */
2719 oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8);
2720 oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8);
2721 oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8);
2722 oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8);
2723 } else {
2724 /* Gen12+ */
2725 xe_assert(oa->xe, GRAPHICS_VER(oa->xe) >= 12);
2726 oa_format_add(oa, XE_OA_FORMAT_A12);
2727 oa_format_add(oa, XE_OA_FORMAT_A12_B8_C8);
2728 oa_format_add(oa, XE_OA_FORMAT_A32u40_A4u32_B8_C8);
2729 oa_format_add(oa, XE_OA_FORMAT_C4_B8);
2730 }
2731}
2732
2733/**
2734 * xe_oa_init - OA initialization during device probe
2735 * @xe: @xe_device
2736 *
2737 * Return: 0 on success or a negative error code on failure
2738 */
2739int xe_oa_init(struct xe_device *xe)
2740{
2741 struct xe_oa *oa = &xe->oa;
2742 int ret;
2743
2744 /* Support OA only with GuC submission and Gen12+ */
2745 if (!xe_device_uc_enabled(xe) || GRAPHICS_VER(xe) < 12)
2746 return 0;
2747
2748 if (IS_SRIOV_VF(xe))
2749 return 0;
2750
2751 oa->xe = xe;
2752 oa->oa_formats = oa_formats;
2753
2754 drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock);
2755 idr_init_base(&oa->metrics_idr, 1);
2756
2757 ret = xe_oa_init_oa_units(oa);
2758 if (ret) {
2759 drm_err(&xe->drm, "OA initialization failed (%pe)\n", ERR_PTR(ret));
2760 goto exit;
2761 }
2762
2763 xe_oa_init_supported_formats(oa);
2764 return 0;
2765exit:
2766 oa->xe = NULL;
2767 return ret;
2768}
2769
2770static int destroy_config(int id, void *p, void *data)
2771{
2772 xe_oa_config_put(p);
2773 return 0;
2774}
2775
2776/**
2777 * xe_oa_fini - OA de-initialization during device remove
2778 * @xe: @xe_device
2779 */
2780void xe_oa_fini(struct xe_device *xe)
2781{
2782 struct xe_oa *oa = &xe->oa;
2783
2784 if (!oa->xe)
2785 return;
2786
2787 idr_for_each(&oa->metrics_idr, destroy_config, oa);
2788 idr_destroy(&oa->metrics_idr);
2789
2790 oa->xe = NULL;
2791}