Loading...
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26#include <linux/kthread.h>
27#include <linux/pci.h>
28#include <linux/uaccess.h>
29#include <linux/pm_runtime.h>
30
31#include "amdgpu.h"
32#include "amdgpu_pm.h"
33#include "amdgpu_dm_debugfs.h"
34#include "amdgpu_ras.h"
35#include "amdgpu_rap.h"
36#include "amdgpu_securedisplay.h"
37#include "amdgpu_fw_attestation.h"
38#include "amdgpu_umr.h"
39
40#include "amdgpu_reset.h"
41#include "amdgpu_psp_ta.h"
42
43#if defined(CONFIG_DEBUG_FS)
44
45/**
46 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
47 *
48 * @read: True if reading
49 * @f: open file handle
50 * @buf: User buffer to write/read to
51 * @size: Number of bytes to write/read
52 * @pos: Offset to seek to
53 *
54 * This debugfs entry has special meaning on the offset being sought.
55 * Various bits have different meanings:
56 *
57 * Bit 62: Indicates a GRBM bank switch is needed
58 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
59 * zero)
60 * Bits 24..33: The SE or ME selector if needed
61 * Bits 34..43: The SH (or SA) or PIPE selector if needed
62 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
63 *
64 * Bit 23: Indicates that the PM power gating lock should be held
65 * This is necessary to read registers that might be
66 * unreliable during a power gating transistion.
67 *
68 * The lower bits are the BYTE offset of the register to read. This
69 * allows reading multiple registers in a single call and having
70 * the returned size reflect that.
71 */
72static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
73 char __user *buf, size_t size, loff_t *pos)
74{
75 struct amdgpu_device *adev = file_inode(f)->i_private;
76 ssize_t result = 0;
77 int r;
78 bool pm_pg_lock, use_bank, use_ring;
79 unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
80
81 pm_pg_lock = use_bank = use_ring = false;
82 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
83
84 if (size & 0x3 || *pos & 0x3 ||
85 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
86 return -EINVAL;
87
88 /* are we reading registers for which a PG lock is necessary? */
89 pm_pg_lock = (*pos >> 23) & 1;
90
91 if (*pos & (1ULL << 62)) {
92 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
93 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
94 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
95
96 if (se_bank == 0x3FF)
97 se_bank = 0xFFFFFFFF;
98 if (sh_bank == 0x3FF)
99 sh_bank = 0xFFFFFFFF;
100 if (instance_bank == 0x3FF)
101 instance_bank = 0xFFFFFFFF;
102 use_bank = true;
103 } else if (*pos & (1ULL << 61)) {
104
105 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
106 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
107 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
108 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
109
110 use_ring = true;
111 } else {
112 use_bank = use_ring = false;
113 }
114
115 *pos &= (1UL << 22) - 1;
116
117 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
118 if (r < 0) {
119 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
120 return r;
121 }
122
123 r = amdgpu_virt_enable_access_debugfs(adev);
124 if (r < 0) {
125 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
126 return r;
127 }
128
129 if (use_bank) {
130 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
131 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
132 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
133 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
134 amdgpu_virt_disable_access_debugfs(adev);
135 return -EINVAL;
136 }
137 mutex_lock(&adev->grbm_idx_mutex);
138 amdgpu_gfx_select_se_sh(adev, se_bank,
139 sh_bank, instance_bank);
140 } else if (use_ring) {
141 mutex_lock(&adev->srbm_mutex);
142 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
143 }
144
145 if (pm_pg_lock)
146 mutex_lock(&adev->pm.mutex);
147
148 while (size) {
149 uint32_t value;
150
151 if (read) {
152 value = RREG32(*pos >> 2);
153 r = put_user(value, (uint32_t *)buf);
154 } else {
155 r = get_user(value, (uint32_t *)buf);
156 if (!r)
157 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
158 }
159 if (r) {
160 result = r;
161 goto end;
162 }
163
164 result += 4;
165 buf += 4;
166 *pos += 4;
167 size -= 4;
168 }
169
170end:
171 if (use_bank) {
172 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
173 mutex_unlock(&adev->grbm_idx_mutex);
174 } else if (use_ring) {
175 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
176 mutex_unlock(&adev->srbm_mutex);
177 }
178
179 if (pm_pg_lock)
180 mutex_unlock(&adev->pm.mutex);
181
182 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
183 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
184
185 amdgpu_virt_disable_access_debugfs(adev);
186 return result;
187}
188
189/*
190 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
191 */
192static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
193 size_t size, loff_t *pos)
194{
195 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
196}
197
198/*
199 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
200 */
201static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
202 size_t size, loff_t *pos)
203{
204 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
205}
206
207static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
208{
209 struct amdgpu_debugfs_regs2_data *rd;
210
211 rd = kzalloc(sizeof *rd, GFP_KERNEL);
212 if (!rd)
213 return -ENOMEM;
214 rd->adev = file_inode(file)->i_private;
215 file->private_data = rd;
216 mutex_init(&rd->lock);
217
218 return 0;
219}
220
221static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
222{
223 struct amdgpu_debugfs_regs2_data *rd = file->private_data;
224 mutex_destroy(&rd->lock);
225 kfree(file->private_data);
226 return 0;
227}
228
229static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
230{
231 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
232 struct amdgpu_device *adev = rd->adev;
233 ssize_t result = 0;
234 int r;
235 uint32_t value;
236
237 if (size & 0x3 || offset & 0x3)
238 return -EINVAL;
239
240 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
241 if (r < 0) {
242 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
243 return r;
244 }
245
246 r = amdgpu_virt_enable_access_debugfs(adev);
247 if (r < 0) {
248 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
249 return r;
250 }
251
252 mutex_lock(&rd->lock);
253
254 if (rd->id.use_grbm) {
255 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
256 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
257 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
258 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
259 amdgpu_virt_disable_access_debugfs(adev);
260 mutex_unlock(&rd->lock);
261 return -EINVAL;
262 }
263 mutex_lock(&adev->grbm_idx_mutex);
264 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
265 rd->id.grbm.sh,
266 rd->id.grbm.instance);
267 }
268
269 if (rd->id.use_srbm) {
270 mutex_lock(&adev->srbm_mutex);
271 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
272 rd->id.srbm.queue, rd->id.srbm.vmid);
273 }
274
275 if (rd->id.pg_lock)
276 mutex_lock(&adev->pm.mutex);
277
278 while (size) {
279 if (!write_en) {
280 value = RREG32(offset >> 2);
281 r = put_user(value, (uint32_t *)buf);
282 } else {
283 r = get_user(value, (uint32_t *)buf);
284 if (!r)
285 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value);
286 }
287 if (r) {
288 result = r;
289 goto end;
290 }
291 offset += 4;
292 size -= 4;
293 result += 4;
294 buf += 4;
295 }
296end:
297 if (rd->id.use_grbm) {
298 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
299 mutex_unlock(&adev->grbm_idx_mutex);
300 }
301
302 if (rd->id.use_srbm) {
303 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
304 mutex_unlock(&adev->srbm_mutex);
305 }
306
307 if (rd->id.pg_lock)
308 mutex_unlock(&adev->pm.mutex);
309
310 mutex_unlock(&rd->lock);
311
312 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
313 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
314
315 amdgpu_virt_disable_access_debugfs(adev);
316 return result;
317}
318
319static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
320{
321 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
322 int r;
323
324 switch (cmd) {
325 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
326 mutex_lock(&rd->lock);
327 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata *)data, sizeof rd->id);
328 mutex_unlock(&rd->lock);
329 return r ? -EINVAL : 0;
330 default:
331 return -EINVAL;
332 }
333 return 0;
334}
335
336static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
337{
338 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
339}
340
341static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
342{
343 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
344}
345
346
347/**
348 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
349 *
350 * @f: open file handle
351 * @buf: User buffer to store read data in
352 * @size: Number of bytes to read
353 * @pos: Offset to seek to
354 *
355 * The lower bits are the BYTE offset of the register to read. This
356 * allows reading multiple registers in a single call and having
357 * the returned size reflect that.
358 */
359static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
360 size_t size, loff_t *pos)
361{
362 struct amdgpu_device *adev = file_inode(f)->i_private;
363 ssize_t result = 0;
364 int r;
365
366 if (size & 0x3 || *pos & 0x3)
367 return -EINVAL;
368
369 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
370 if (r < 0) {
371 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
372 return r;
373 }
374
375 r = amdgpu_virt_enable_access_debugfs(adev);
376 if (r < 0) {
377 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
378 return r;
379 }
380
381 while (size) {
382 uint32_t value;
383
384 value = RREG32_PCIE(*pos);
385 r = put_user(value, (uint32_t *)buf);
386 if (r)
387 goto out;
388
389 result += 4;
390 buf += 4;
391 *pos += 4;
392 size -= 4;
393 }
394
395 r = result;
396out:
397 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
398 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
399 amdgpu_virt_disable_access_debugfs(adev);
400 return r;
401}
402
403/**
404 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
405 *
406 * @f: open file handle
407 * @buf: User buffer to write data from
408 * @size: Number of bytes to write
409 * @pos: Offset to seek to
410 *
411 * The lower bits are the BYTE offset of the register to write. This
412 * allows writing multiple registers in a single call and having
413 * the returned size reflect that.
414 */
415static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
416 size_t size, loff_t *pos)
417{
418 struct amdgpu_device *adev = file_inode(f)->i_private;
419 ssize_t result = 0;
420 int r;
421
422 if (size & 0x3 || *pos & 0x3)
423 return -EINVAL;
424
425 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
426 if (r < 0) {
427 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
428 return r;
429 }
430
431 r = amdgpu_virt_enable_access_debugfs(adev);
432 if (r < 0) {
433 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
434 return r;
435 }
436
437 while (size) {
438 uint32_t value;
439
440 r = get_user(value, (uint32_t *)buf);
441 if (r)
442 goto out;
443
444 WREG32_PCIE(*pos, value);
445
446 result += 4;
447 buf += 4;
448 *pos += 4;
449 size -= 4;
450 }
451
452 r = result;
453out:
454 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
455 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
456 amdgpu_virt_disable_access_debugfs(adev);
457 return r;
458}
459
460/**
461 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
462 *
463 * @f: open file handle
464 * @buf: User buffer to store read data in
465 * @size: Number of bytes to read
466 * @pos: Offset to seek to
467 *
468 * The lower bits are the BYTE offset of the register to read. This
469 * allows reading multiple registers in a single call and having
470 * the returned size reflect that.
471 */
472static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
473 size_t size, loff_t *pos)
474{
475 struct amdgpu_device *adev = file_inode(f)->i_private;
476 ssize_t result = 0;
477 int r;
478
479 if (size & 0x3 || *pos & 0x3)
480 return -EINVAL;
481
482 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
483 if (r < 0) {
484 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
485 return r;
486 }
487
488 r = amdgpu_virt_enable_access_debugfs(adev);
489 if (r < 0) {
490 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
491 return r;
492 }
493
494 while (size) {
495 uint32_t value;
496
497 value = RREG32_DIDT(*pos >> 2);
498 r = put_user(value, (uint32_t *)buf);
499 if (r)
500 goto out;
501
502 result += 4;
503 buf += 4;
504 *pos += 4;
505 size -= 4;
506 }
507
508 r = result;
509out:
510 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
511 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
512 amdgpu_virt_disable_access_debugfs(adev);
513 return r;
514}
515
516/**
517 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
518 *
519 * @f: open file handle
520 * @buf: User buffer to write data from
521 * @size: Number of bytes to write
522 * @pos: Offset to seek to
523 *
524 * The lower bits are the BYTE offset of the register to write. This
525 * allows writing multiple registers in a single call and having
526 * the returned size reflect that.
527 */
528static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
529 size_t size, loff_t *pos)
530{
531 struct amdgpu_device *adev = file_inode(f)->i_private;
532 ssize_t result = 0;
533 int r;
534
535 if (size & 0x3 || *pos & 0x3)
536 return -EINVAL;
537
538 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
539 if (r < 0) {
540 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
541 return r;
542 }
543
544 r = amdgpu_virt_enable_access_debugfs(adev);
545 if (r < 0) {
546 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
547 return r;
548 }
549
550 while (size) {
551 uint32_t value;
552
553 r = get_user(value, (uint32_t *)buf);
554 if (r)
555 goto out;
556
557 WREG32_DIDT(*pos >> 2, value);
558
559 result += 4;
560 buf += 4;
561 *pos += 4;
562 size -= 4;
563 }
564
565 r = result;
566out:
567 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
568 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
569 amdgpu_virt_disable_access_debugfs(adev);
570 return r;
571}
572
573/**
574 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
575 *
576 * @f: open file handle
577 * @buf: User buffer to store read data in
578 * @size: Number of bytes to read
579 * @pos: Offset to seek to
580 *
581 * The lower bits are the BYTE offset of the register to read. This
582 * allows reading multiple registers in a single call and having
583 * the returned size reflect that.
584 */
585static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
586 size_t size, loff_t *pos)
587{
588 struct amdgpu_device *adev = file_inode(f)->i_private;
589 ssize_t result = 0;
590 int r;
591
592 if (size & 0x3 || *pos & 0x3)
593 return -EINVAL;
594
595 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
596 if (r < 0) {
597 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
598 return r;
599 }
600
601 r = amdgpu_virt_enable_access_debugfs(adev);
602 if (r < 0) {
603 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
604 return r;
605 }
606
607 while (size) {
608 uint32_t value;
609
610 value = RREG32_SMC(*pos);
611 r = put_user(value, (uint32_t *)buf);
612 if (r)
613 goto out;
614
615 result += 4;
616 buf += 4;
617 *pos += 4;
618 size -= 4;
619 }
620
621 r = result;
622out:
623 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
624 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
625 amdgpu_virt_disable_access_debugfs(adev);
626 return r;
627}
628
629/**
630 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
631 *
632 * @f: open file handle
633 * @buf: User buffer to write data from
634 * @size: Number of bytes to write
635 * @pos: Offset to seek to
636 *
637 * The lower bits are the BYTE offset of the register to write. This
638 * allows writing multiple registers in a single call and having
639 * the returned size reflect that.
640 */
641static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
642 size_t size, loff_t *pos)
643{
644 struct amdgpu_device *adev = file_inode(f)->i_private;
645 ssize_t result = 0;
646 int r;
647
648 if (size & 0x3 || *pos & 0x3)
649 return -EINVAL;
650
651 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
652 if (r < 0) {
653 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
654 return r;
655 }
656
657 r = amdgpu_virt_enable_access_debugfs(adev);
658 if (r < 0) {
659 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
660 return r;
661 }
662
663 while (size) {
664 uint32_t value;
665
666 r = get_user(value, (uint32_t *)buf);
667 if (r)
668 goto out;
669
670 WREG32_SMC(*pos, value);
671
672 result += 4;
673 buf += 4;
674 *pos += 4;
675 size -= 4;
676 }
677
678 r = result;
679out:
680 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
681 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
682 amdgpu_virt_disable_access_debugfs(adev);
683 return r;
684}
685
686/**
687 * amdgpu_debugfs_gca_config_read - Read from gfx config data
688 *
689 * @f: open file handle
690 * @buf: User buffer to store read data in
691 * @size: Number of bytes to read
692 * @pos: Offset to seek to
693 *
694 * This file is used to access configuration data in a somewhat
695 * stable fashion. The format is a series of DWORDs with the first
696 * indicating which revision it is. New content is appended to the
697 * end so that older software can still read the data.
698 */
699
700static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
701 size_t size, loff_t *pos)
702{
703 struct amdgpu_device *adev = file_inode(f)->i_private;
704 ssize_t result = 0;
705 int r;
706 uint32_t *config, no_regs = 0;
707
708 if (size & 0x3 || *pos & 0x3)
709 return -EINVAL;
710
711 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
712 if (!config)
713 return -ENOMEM;
714
715 /* version, increment each time something is added */
716 config[no_regs++] = 5;
717 config[no_regs++] = adev->gfx.config.max_shader_engines;
718 config[no_regs++] = adev->gfx.config.max_tile_pipes;
719 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
720 config[no_regs++] = adev->gfx.config.max_sh_per_se;
721 config[no_regs++] = adev->gfx.config.max_backends_per_se;
722 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
723 config[no_regs++] = adev->gfx.config.max_gprs;
724 config[no_regs++] = adev->gfx.config.max_gs_threads;
725 config[no_regs++] = adev->gfx.config.max_hw_contexts;
726 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
727 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
728 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
729 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
730 config[no_regs++] = adev->gfx.config.num_tile_pipes;
731 config[no_regs++] = adev->gfx.config.backend_enable_mask;
732 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
733 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
734 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
735 config[no_regs++] = adev->gfx.config.num_gpus;
736 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
737 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
738 config[no_regs++] = adev->gfx.config.gb_addr_config;
739 config[no_regs++] = adev->gfx.config.num_rbs;
740
741 /* rev==1 */
742 config[no_regs++] = adev->rev_id;
743 config[no_regs++] = lower_32_bits(adev->pg_flags);
744 config[no_regs++] = lower_32_bits(adev->cg_flags);
745
746 /* rev==2 */
747 config[no_regs++] = adev->family;
748 config[no_regs++] = adev->external_rev_id;
749
750 /* rev==3 */
751 config[no_regs++] = adev->pdev->device;
752 config[no_regs++] = adev->pdev->revision;
753 config[no_regs++] = adev->pdev->subsystem_device;
754 config[no_regs++] = adev->pdev->subsystem_vendor;
755
756 /* rev==4 APU flag */
757 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
758
759 /* rev==5 PG/CG flag upper 32bit */
760 config[no_regs++] = upper_32_bits(adev->pg_flags);
761 config[no_regs++] = upper_32_bits(adev->cg_flags);
762
763 while (size && (*pos < no_regs * 4)) {
764 uint32_t value;
765
766 value = config[*pos >> 2];
767 r = put_user(value, (uint32_t *)buf);
768 if (r) {
769 kfree(config);
770 return r;
771 }
772
773 result += 4;
774 buf += 4;
775 *pos += 4;
776 size -= 4;
777 }
778
779 kfree(config);
780 return result;
781}
782
783/**
784 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
785 *
786 * @f: open file handle
787 * @buf: User buffer to store read data in
788 * @size: Number of bytes to read
789 * @pos: Offset to seek to
790 *
791 * The offset is treated as the BYTE address of one of the sensors
792 * enumerated in amd/include/kgd_pp_interface.h under the
793 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
794 * you would use the offset 3 * 4 = 12.
795 */
796static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
797 size_t size, loff_t *pos)
798{
799 struct amdgpu_device *adev = file_inode(f)->i_private;
800 int idx, x, outsize, r, valuesize;
801 uint32_t values[16];
802
803 if (size & 3 || *pos & 0x3)
804 return -EINVAL;
805
806 if (!adev->pm.dpm_enabled)
807 return -EINVAL;
808
809 /* convert offset to sensor number */
810 idx = *pos >> 2;
811
812 valuesize = sizeof(values);
813
814 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
815 if (r < 0) {
816 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
817 return r;
818 }
819
820 r = amdgpu_virt_enable_access_debugfs(adev);
821 if (r < 0) {
822 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
823 return r;
824 }
825
826 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
827
828 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
829 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
830
831 if (r) {
832 amdgpu_virt_disable_access_debugfs(adev);
833 return r;
834 }
835
836 if (size > valuesize) {
837 amdgpu_virt_disable_access_debugfs(adev);
838 return -EINVAL;
839 }
840
841 outsize = 0;
842 x = 0;
843 if (!r) {
844 while (size) {
845 r = put_user(values[x++], (int32_t *)buf);
846 buf += 4;
847 size -= 4;
848 outsize += 4;
849 }
850 }
851
852 amdgpu_virt_disable_access_debugfs(adev);
853 return !r ? outsize : r;
854}
855
856/** amdgpu_debugfs_wave_read - Read WAVE STATUS data
857 *
858 * @f: open file handle
859 * @buf: User buffer to store read data in
860 * @size: Number of bytes to read
861 * @pos: Offset to seek to
862 *
863 * The offset being sought changes which wave that the status data
864 * will be returned for. The bits are used as follows:
865 *
866 * Bits 0..6: Byte offset into data
867 * Bits 7..14: SE selector
868 * Bits 15..22: SH/SA selector
869 * Bits 23..30: CU/{WGP+SIMD} selector
870 * Bits 31..36: WAVE ID selector
871 * Bits 37..44: SIMD ID selector
872 *
873 * The returned data begins with one DWORD of version information
874 * Followed by WAVE STATUS registers relevant to the GFX IP version
875 * being used. See gfx_v8_0_read_wave_data() for an example output.
876 */
877static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
878 size_t size, loff_t *pos)
879{
880 struct amdgpu_device *adev = f->f_inode->i_private;
881 int r, x;
882 ssize_t result = 0;
883 uint32_t offset, se, sh, cu, wave, simd, data[32];
884
885 if (size & 3 || *pos & 3)
886 return -EINVAL;
887
888 /* decode offset */
889 offset = (*pos & GENMASK_ULL(6, 0));
890 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
891 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
892 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
893 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
894 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
895
896 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
897 if (r < 0) {
898 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
899 return r;
900 }
901
902 r = amdgpu_virt_enable_access_debugfs(adev);
903 if (r < 0) {
904 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
905 return r;
906 }
907
908 /* switch to the specific se/sh/cu */
909 mutex_lock(&adev->grbm_idx_mutex);
910 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
911
912 x = 0;
913 if (adev->gfx.funcs->read_wave_data)
914 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
915
916 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
917 mutex_unlock(&adev->grbm_idx_mutex);
918
919 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
920 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
921
922 if (!x) {
923 amdgpu_virt_disable_access_debugfs(adev);
924 return -EINVAL;
925 }
926
927 while (size && (offset < x * 4)) {
928 uint32_t value;
929
930 value = data[offset >> 2];
931 r = put_user(value, (uint32_t *)buf);
932 if (r) {
933 amdgpu_virt_disable_access_debugfs(adev);
934 return r;
935 }
936
937 result += 4;
938 buf += 4;
939 offset += 4;
940 size -= 4;
941 }
942
943 amdgpu_virt_disable_access_debugfs(adev);
944 return result;
945}
946
947/** amdgpu_debugfs_gpr_read - Read wave gprs
948 *
949 * @f: open file handle
950 * @buf: User buffer to store read data in
951 * @size: Number of bytes to read
952 * @pos: Offset to seek to
953 *
954 * The offset being sought changes which wave that the status data
955 * will be returned for. The bits are used as follows:
956 *
957 * Bits 0..11: Byte offset into data
958 * Bits 12..19: SE selector
959 * Bits 20..27: SH/SA selector
960 * Bits 28..35: CU/{WGP+SIMD} selector
961 * Bits 36..43: WAVE ID selector
962 * Bits 37..44: SIMD ID selector
963 * Bits 52..59: Thread selector
964 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
965 *
966 * The return data comes from the SGPR or VGPR register bank for
967 * the selected operational unit.
968 */
969static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
970 size_t size, loff_t *pos)
971{
972 struct amdgpu_device *adev = f->f_inode->i_private;
973 int r;
974 ssize_t result = 0;
975 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
976
977 if (size > 4096 || size & 3 || *pos & 3)
978 return -EINVAL;
979
980 /* decode offset */
981 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
982 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
983 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
984 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
985 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
986 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
987 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
988 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
989
990 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
991 if (!data)
992 return -ENOMEM;
993
994 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
995 if (r < 0)
996 goto err;
997
998 r = amdgpu_virt_enable_access_debugfs(adev);
999 if (r < 0)
1000 goto err;
1001
1002 /* switch to the specific se/sh/cu */
1003 mutex_lock(&adev->grbm_idx_mutex);
1004 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
1005
1006 if (bank == 0) {
1007 if (adev->gfx.funcs->read_wave_vgprs)
1008 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
1009 } else {
1010 if (adev->gfx.funcs->read_wave_sgprs)
1011 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
1012 }
1013
1014 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
1015 mutex_unlock(&adev->grbm_idx_mutex);
1016
1017 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1018 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1019
1020 while (size) {
1021 uint32_t value;
1022
1023 value = data[result >> 2];
1024 r = put_user(value, (uint32_t *)buf);
1025 if (r) {
1026 amdgpu_virt_disable_access_debugfs(adev);
1027 goto err;
1028 }
1029
1030 result += 4;
1031 buf += 4;
1032 size -= 4;
1033 }
1034
1035 kfree(data);
1036 amdgpu_virt_disable_access_debugfs(adev);
1037 return result;
1038
1039err:
1040 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1041 kfree(data);
1042 return r;
1043}
1044
1045/**
1046 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1047 *
1048 * @f: open file handle
1049 * @buf: User buffer to store read data in
1050 * @size: Number of bytes to read
1051 * @pos: Offset to seek to
1052 *
1053 * Read the last residency value logged. It doesn't auto update, one needs to
1054 * stop logging before getting the current value.
1055 */
1056static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1057 size_t size, loff_t *pos)
1058{
1059 struct amdgpu_device *adev = file_inode(f)->i_private;
1060 ssize_t result = 0;
1061 int r;
1062
1063 if (size & 0x3 || *pos & 0x3)
1064 return -EINVAL;
1065
1066 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1067 if (r < 0) {
1068 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1069 return r;
1070 }
1071
1072 while (size) {
1073 uint32_t value;
1074
1075 r = amdgpu_get_gfx_off_residency(adev, &value);
1076 if (r)
1077 goto out;
1078
1079 r = put_user(value, (uint32_t *)buf);
1080 if (r)
1081 goto out;
1082
1083 result += 4;
1084 buf += 4;
1085 *pos += 4;
1086 size -= 4;
1087 }
1088
1089 r = result;
1090out:
1091 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1092 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1093
1094 return r;
1095}
1096
1097/**
1098 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1099 *
1100 * @f: open file handle
1101 * @buf: User buffer to write data from
1102 * @size: Number of bytes to write
1103 * @pos: Offset to seek to
1104 *
1105 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1106 */
1107static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1108 size_t size, loff_t *pos)
1109{
1110 struct amdgpu_device *adev = file_inode(f)->i_private;
1111 ssize_t result = 0;
1112 int r;
1113
1114 if (size & 0x3 || *pos & 0x3)
1115 return -EINVAL;
1116
1117 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1118 if (r < 0) {
1119 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1120 return r;
1121 }
1122
1123 while (size) {
1124 u32 value;
1125
1126 r = get_user(value, (uint32_t *)buf);
1127 if (r)
1128 goto out;
1129
1130 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1131
1132 result += 4;
1133 buf += 4;
1134 *pos += 4;
1135 size -= 4;
1136 }
1137
1138 r = result;
1139out:
1140 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1141 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1142
1143 return r;
1144}
1145
1146
1147/**
1148 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1149 *
1150 * @f: open file handle
1151 * @buf: User buffer to store read data in
1152 * @size: Number of bytes to read
1153 * @pos: Offset to seek to
1154 */
1155static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1156 size_t size, loff_t *pos)
1157{
1158 struct amdgpu_device *adev = file_inode(f)->i_private;
1159 ssize_t result = 0;
1160 int r;
1161
1162 if (size & 0x3 || *pos & 0x3)
1163 return -EINVAL;
1164
1165 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1166 if (r < 0) {
1167 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1168 return r;
1169 }
1170
1171 while (size) {
1172 u64 value = 0;
1173
1174 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1175 if (r)
1176 goto out;
1177
1178 r = put_user(value, (u64 *)buf);
1179 if (r)
1180 goto out;
1181
1182 result += 4;
1183 buf += 4;
1184 *pos += 4;
1185 size -= 4;
1186 }
1187
1188 r = result;
1189out:
1190 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1191 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1192
1193 return r;
1194}
1195
1196/**
1197 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1198 *
1199 * @f: open file handle
1200 * @buf: User buffer to write data from
1201 * @size: Number of bytes to write
1202 * @pos: Offset to seek to
1203 *
1204 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1205 */
1206static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1207 size_t size, loff_t *pos)
1208{
1209 struct amdgpu_device *adev = file_inode(f)->i_private;
1210 ssize_t result = 0;
1211 int r;
1212
1213 if (size & 0x3 || *pos & 0x3)
1214 return -EINVAL;
1215
1216 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1217 if (r < 0) {
1218 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1219 return r;
1220 }
1221
1222 while (size) {
1223 uint32_t value;
1224
1225 r = get_user(value, (uint32_t *)buf);
1226 if (r)
1227 goto out;
1228
1229 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1230
1231 result += 4;
1232 buf += 4;
1233 *pos += 4;
1234 size -= 4;
1235 }
1236
1237 r = result;
1238out:
1239 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1240 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1241
1242 return r;
1243}
1244
1245
1246/**
1247 * amdgpu_debugfs_gfxoff_read - read gfxoff status
1248 *
1249 * @f: open file handle
1250 * @buf: User buffer to store read data in
1251 * @size: Number of bytes to read
1252 * @pos: Offset to seek to
1253 */
1254static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1255 size_t size, loff_t *pos)
1256{
1257 struct amdgpu_device *adev = file_inode(f)->i_private;
1258 ssize_t result = 0;
1259 int r;
1260
1261 if (size & 0x3 || *pos & 0x3)
1262 return -EINVAL;
1263
1264 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1265 if (r < 0) {
1266 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1267 return r;
1268 }
1269
1270 while (size) {
1271 u32 value = adev->gfx.gfx_off_state;
1272
1273 r = put_user(value, (u32 *)buf);
1274 if (r)
1275 goto out;
1276
1277 result += 4;
1278 buf += 4;
1279 *pos += 4;
1280 size -= 4;
1281 }
1282
1283 r = result;
1284out:
1285 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1286 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1287
1288 return r;
1289}
1290
1291static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1292 size_t size, loff_t *pos)
1293{
1294 struct amdgpu_device *adev = file_inode(f)->i_private;
1295 ssize_t result = 0;
1296 int r;
1297
1298 if (size & 0x3 || *pos & 0x3)
1299 return -EINVAL;
1300
1301 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1302 if (r < 0) {
1303 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1304 return r;
1305 }
1306
1307 while (size) {
1308 u32 value;
1309
1310 r = amdgpu_get_gfx_off_status(adev, &value);
1311 if (r)
1312 goto out;
1313
1314 r = put_user(value, (u32 *)buf);
1315 if (r)
1316 goto out;
1317
1318 result += 4;
1319 buf += 4;
1320 *pos += 4;
1321 size -= 4;
1322 }
1323
1324 r = result;
1325out:
1326 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1327 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1328
1329 return r;
1330}
1331
1332static const struct file_operations amdgpu_debugfs_regs2_fops = {
1333 .owner = THIS_MODULE,
1334 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1335 .read = amdgpu_debugfs_regs2_read,
1336 .write = amdgpu_debugfs_regs2_write,
1337 .open = amdgpu_debugfs_regs2_open,
1338 .release = amdgpu_debugfs_regs2_release,
1339 .llseek = default_llseek
1340};
1341
1342static const struct file_operations amdgpu_debugfs_regs_fops = {
1343 .owner = THIS_MODULE,
1344 .read = amdgpu_debugfs_regs_read,
1345 .write = amdgpu_debugfs_regs_write,
1346 .llseek = default_llseek
1347};
1348static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1349 .owner = THIS_MODULE,
1350 .read = amdgpu_debugfs_regs_didt_read,
1351 .write = amdgpu_debugfs_regs_didt_write,
1352 .llseek = default_llseek
1353};
1354static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1355 .owner = THIS_MODULE,
1356 .read = amdgpu_debugfs_regs_pcie_read,
1357 .write = amdgpu_debugfs_regs_pcie_write,
1358 .llseek = default_llseek
1359};
1360static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1361 .owner = THIS_MODULE,
1362 .read = amdgpu_debugfs_regs_smc_read,
1363 .write = amdgpu_debugfs_regs_smc_write,
1364 .llseek = default_llseek
1365};
1366
1367static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1368 .owner = THIS_MODULE,
1369 .read = amdgpu_debugfs_gca_config_read,
1370 .llseek = default_llseek
1371};
1372
1373static const struct file_operations amdgpu_debugfs_sensors_fops = {
1374 .owner = THIS_MODULE,
1375 .read = amdgpu_debugfs_sensor_read,
1376 .llseek = default_llseek
1377};
1378
1379static const struct file_operations amdgpu_debugfs_wave_fops = {
1380 .owner = THIS_MODULE,
1381 .read = amdgpu_debugfs_wave_read,
1382 .llseek = default_llseek
1383};
1384static const struct file_operations amdgpu_debugfs_gpr_fops = {
1385 .owner = THIS_MODULE,
1386 .read = amdgpu_debugfs_gpr_read,
1387 .llseek = default_llseek
1388};
1389
1390static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1391 .owner = THIS_MODULE,
1392 .read = amdgpu_debugfs_gfxoff_read,
1393 .write = amdgpu_debugfs_gfxoff_write,
1394 .llseek = default_llseek
1395};
1396
1397static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1398 .owner = THIS_MODULE,
1399 .read = amdgpu_debugfs_gfxoff_status_read,
1400 .llseek = default_llseek
1401};
1402
1403static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1404 .owner = THIS_MODULE,
1405 .read = amdgpu_debugfs_gfxoff_count_read,
1406 .llseek = default_llseek
1407};
1408
1409static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1410 .owner = THIS_MODULE,
1411 .read = amdgpu_debugfs_gfxoff_residency_read,
1412 .write = amdgpu_debugfs_gfxoff_residency_write,
1413 .llseek = default_llseek
1414};
1415
1416static const struct file_operations *debugfs_regs[] = {
1417 &amdgpu_debugfs_regs_fops,
1418 &amdgpu_debugfs_regs2_fops,
1419 &amdgpu_debugfs_regs_didt_fops,
1420 &amdgpu_debugfs_regs_pcie_fops,
1421 &amdgpu_debugfs_regs_smc_fops,
1422 &amdgpu_debugfs_gca_config_fops,
1423 &amdgpu_debugfs_sensors_fops,
1424 &amdgpu_debugfs_wave_fops,
1425 &amdgpu_debugfs_gpr_fops,
1426 &amdgpu_debugfs_gfxoff_fops,
1427 &amdgpu_debugfs_gfxoff_status_fops,
1428 &amdgpu_debugfs_gfxoff_count_fops,
1429 &amdgpu_debugfs_gfxoff_residency_fops,
1430};
1431
1432static const char *debugfs_regs_names[] = {
1433 "amdgpu_regs",
1434 "amdgpu_regs2",
1435 "amdgpu_regs_didt",
1436 "amdgpu_regs_pcie",
1437 "amdgpu_regs_smc",
1438 "amdgpu_gca_config",
1439 "amdgpu_sensors",
1440 "amdgpu_wave",
1441 "amdgpu_gpr",
1442 "amdgpu_gfxoff",
1443 "amdgpu_gfxoff_status",
1444 "amdgpu_gfxoff_count",
1445 "amdgpu_gfxoff_residency",
1446};
1447
1448/**
1449 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1450 * register access.
1451 *
1452 * @adev: The device to attach the debugfs entries to
1453 */
1454int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1455{
1456 struct drm_minor *minor = adev_to_drm(adev)->primary;
1457 struct dentry *ent, *root = minor->debugfs_root;
1458 unsigned int i;
1459
1460 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1461 ent = debugfs_create_file(debugfs_regs_names[i],
1462 S_IFREG | S_IRUGO, root,
1463 adev, debugfs_regs[i]);
1464 if (!i && !IS_ERR_OR_NULL(ent))
1465 i_size_write(ent->d_inode, adev->rmmio_size);
1466 }
1467
1468 return 0;
1469}
1470
1471static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1472{
1473 struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1474 struct drm_device *dev = adev_to_drm(adev);
1475 int r = 0, i;
1476
1477 r = pm_runtime_get_sync(dev->dev);
1478 if (r < 0) {
1479 pm_runtime_put_autosuspend(dev->dev);
1480 return r;
1481 }
1482
1483 /* Avoid accidently unparking the sched thread during GPU reset */
1484 r = down_write_killable(&adev->reset_domain->sem);
1485 if (r)
1486 return r;
1487
1488 /* hold on the scheduler */
1489 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1490 struct amdgpu_ring *ring = adev->rings[i];
1491
1492 if (!ring || !ring->sched.thread)
1493 continue;
1494 kthread_park(ring->sched.thread);
1495 }
1496
1497 seq_printf(m, "run ib test:\n");
1498 r = amdgpu_ib_ring_tests(adev);
1499 if (r)
1500 seq_printf(m, "ib ring tests failed (%d).\n", r);
1501 else
1502 seq_printf(m, "ib ring tests passed.\n");
1503
1504 /* go on the scheduler */
1505 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1506 struct amdgpu_ring *ring = adev->rings[i];
1507
1508 if (!ring || !ring->sched.thread)
1509 continue;
1510 kthread_unpark(ring->sched.thread);
1511 }
1512
1513 up_write(&adev->reset_domain->sem);
1514
1515 pm_runtime_mark_last_busy(dev->dev);
1516 pm_runtime_put_autosuspend(dev->dev);
1517
1518 return 0;
1519}
1520
1521static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1522{
1523 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1524 struct drm_device *dev = adev_to_drm(adev);
1525 int r;
1526
1527 r = pm_runtime_get_sync(dev->dev);
1528 if (r < 0) {
1529 pm_runtime_put_autosuspend(dev->dev);
1530 return r;
1531 }
1532
1533 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1534
1535 pm_runtime_mark_last_busy(dev->dev);
1536 pm_runtime_put_autosuspend(dev->dev);
1537
1538 return 0;
1539}
1540
1541
1542static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1543{
1544 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1545 struct drm_device *dev = adev_to_drm(adev);
1546 int r;
1547
1548 r = pm_runtime_get_sync(dev->dev);
1549 if (r < 0) {
1550 pm_runtime_put_autosuspend(dev->dev);
1551 return r;
1552 }
1553
1554 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1555
1556 pm_runtime_mark_last_busy(dev->dev);
1557 pm_runtime_put_autosuspend(dev->dev);
1558
1559 return 0;
1560}
1561
1562static int amdgpu_debugfs_benchmark(void *data, u64 val)
1563{
1564 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1565 struct drm_device *dev = adev_to_drm(adev);
1566 int r;
1567
1568 r = pm_runtime_get_sync(dev->dev);
1569 if (r < 0) {
1570 pm_runtime_put_autosuspend(dev->dev);
1571 return r;
1572 }
1573
1574 r = amdgpu_benchmark(adev, val);
1575
1576 pm_runtime_mark_last_busy(dev->dev);
1577 pm_runtime_put_autosuspend(dev->dev);
1578
1579 return r;
1580}
1581
1582static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1583{
1584 struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1585 struct drm_device *dev = adev_to_drm(adev);
1586 struct drm_file *file;
1587 int r;
1588
1589 r = mutex_lock_interruptible(&dev->filelist_mutex);
1590 if (r)
1591 return r;
1592
1593 list_for_each_entry(file, &dev->filelist, lhead) {
1594 struct amdgpu_fpriv *fpriv = file->driver_priv;
1595 struct amdgpu_vm *vm = &fpriv->vm;
1596
1597 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1598 vm->task_info.pid, vm->task_info.process_name);
1599 r = amdgpu_bo_reserve(vm->root.bo, true);
1600 if (r)
1601 break;
1602 amdgpu_debugfs_vm_bo_info(vm, m);
1603 amdgpu_bo_unreserve(vm->root.bo);
1604 }
1605
1606 mutex_unlock(&dev->filelist_mutex);
1607
1608 return r;
1609}
1610
1611DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1612DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1613DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1614 NULL, "%lld\n");
1615DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1616 NULL, "%lld\n");
1617DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1618 "%lld\n");
1619
1620static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1621 struct dma_fence **fences)
1622{
1623 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1624 uint32_t sync_seq, last_seq;
1625
1626 last_seq = atomic_read(&ring->fence_drv.last_seq);
1627 sync_seq = ring->fence_drv.sync_seq;
1628
1629 last_seq &= drv->num_fences_mask;
1630 sync_seq &= drv->num_fences_mask;
1631
1632 do {
1633 struct dma_fence *fence, **ptr;
1634
1635 ++last_seq;
1636 last_seq &= drv->num_fences_mask;
1637 ptr = &drv->fences[last_seq];
1638
1639 fence = rcu_dereference_protected(*ptr, 1);
1640 RCU_INIT_POINTER(*ptr, NULL);
1641
1642 if (!fence)
1643 continue;
1644
1645 fences[last_seq] = fence;
1646
1647 } while (last_seq != sync_seq);
1648}
1649
1650static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1651 int length)
1652{
1653 int i;
1654 struct dma_fence *fence;
1655
1656 for (i = 0; i < length; i++) {
1657 fence = fences[i];
1658 if (!fence)
1659 continue;
1660 dma_fence_signal(fence);
1661 dma_fence_put(fence);
1662 }
1663}
1664
1665static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1666{
1667 struct drm_sched_job *s_job;
1668 struct dma_fence *fence;
1669
1670 spin_lock(&sched->job_list_lock);
1671 list_for_each_entry(s_job, &sched->pending_list, list) {
1672 fence = sched->ops->run_job(s_job);
1673 dma_fence_put(fence);
1674 }
1675 spin_unlock(&sched->job_list_lock);
1676}
1677
1678static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1679{
1680 struct amdgpu_job *job;
1681 struct drm_sched_job *s_job, *tmp;
1682 uint32_t preempt_seq;
1683 struct dma_fence *fence, **ptr;
1684 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1685 struct drm_gpu_scheduler *sched = &ring->sched;
1686 bool preempted = true;
1687
1688 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1689 return;
1690
1691 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1692 if (preempt_seq <= atomic_read(&drv->last_seq)) {
1693 preempted = false;
1694 goto no_preempt;
1695 }
1696
1697 preempt_seq &= drv->num_fences_mask;
1698 ptr = &drv->fences[preempt_seq];
1699 fence = rcu_dereference_protected(*ptr, 1);
1700
1701no_preempt:
1702 spin_lock(&sched->job_list_lock);
1703 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1704 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1705 /* remove job from ring_mirror_list */
1706 list_del_init(&s_job->list);
1707 sched->ops->free_job(s_job);
1708 continue;
1709 }
1710 job = to_amdgpu_job(s_job);
1711 if (preempted && (&job->hw_fence) == fence)
1712 /* mark the job as preempted */
1713 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1714 }
1715 spin_unlock(&sched->job_list_lock);
1716}
1717
1718static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1719{
1720 int r, resched, length;
1721 struct amdgpu_ring *ring;
1722 struct dma_fence **fences = NULL;
1723 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1724
1725 if (val >= AMDGPU_MAX_RINGS)
1726 return -EINVAL;
1727
1728 ring = adev->rings[val];
1729
1730 if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1731 return -EINVAL;
1732
1733 /* the last preemption failed */
1734 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1735 return -EBUSY;
1736
1737 length = ring->fence_drv.num_fences_mask + 1;
1738 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1739 if (!fences)
1740 return -ENOMEM;
1741
1742 /* Avoid accidently unparking the sched thread during GPU reset */
1743 r = down_read_killable(&adev->reset_domain->sem);
1744 if (r)
1745 goto pro_end;
1746
1747 /* stop the scheduler */
1748 kthread_park(ring->sched.thread);
1749
1750 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1751
1752 /* preempt the IB */
1753 r = amdgpu_ring_preempt_ib(ring);
1754 if (r) {
1755 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1756 goto failure;
1757 }
1758
1759 amdgpu_fence_process(ring);
1760
1761 if (atomic_read(&ring->fence_drv.last_seq) !=
1762 ring->fence_drv.sync_seq) {
1763 DRM_INFO("ring %d was preempted\n", ring->idx);
1764
1765 amdgpu_ib_preempt_mark_partial_job(ring);
1766
1767 /* swap out the old fences */
1768 amdgpu_ib_preempt_fences_swap(ring, fences);
1769
1770 amdgpu_fence_driver_force_completion(ring);
1771
1772 /* resubmit unfinished jobs */
1773 amdgpu_ib_preempt_job_recovery(&ring->sched);
1774
1775 /* wait for jobs finished */
1776 amdgpu_fence_wait_empty(ring);
1777
1778 /* signal the old fences */
1779 amdgpu_ib_preempt_signal_fences(fences, length);
1780 }
1781
1782failure:
1783 /* restart the scheduler */
1784 kthread_unpark(ring->sched.thread);
1785
1786 up_read(&adev->reset_domain->sem);
1787
1788 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1789
1790pro_end:
1791 kfree(fences);
1792
1793 return r;
1794}
1795
1796static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1797{
1798 int ret = 0;
1799 uint32_t max_freq, min_freq;
1800 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1801
1802 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1803 return -EINVAL;
1804
1805 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1806 if (ret < 0) {
1807 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1808 return ret;
1809 }
1810
1811 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
1812 if (ret == -EOPNOTSUPP) {
1813 ret = 0;
1814 goto out;
1815 }
1816 if (ret || val > max_freq || val < min_freq) {
1817 ret = -EINVAL;
1818 goto out;
1819 }
1820
1821 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
1822 if (ret)
1823 ret = -EINVAL;
1824
1825out:
1826 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1827 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1828
1829 return ret;
1830}
1831
1832DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1833 amdgpu_debugfs_ib_preempt, "%llu\n");
1834
1835DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
1836 amdgpu_debugfs_sclk_set, "%llu\n");
1837
1838static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
1839 char __user *buf, size_t size, loff_t *pos)
1840{
1841 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
1842 char reg_offset[12];
1843 int i, ret, len = 0;
1844
1845 if (*pos)
1846 return 0;
1847
1848 memset(reg_offset, 0, 12);
1849 ret = down_read_killable(&adev->reset_domain->sem);
1850 if (ret)
1851 return ret;
1852
1853 for (i = 0; i < adev->num_regs; i++) {
1854 sprintf(reg_offset, "0x%x\n", adev->reset_dump_reg_list[i]);
1855 up_read(&adev->reset_domain->sem);
1856 if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
1857 return -EFAULT;
1858
1859 len += strlen(reg_offset);
1860 ret = down_read_killable(&adev->reset_domain->sem);
1861 if (ret)
1862 return ret;
1863 }
1864
1865 up_read(&adev->reset_domain->sem);
1866 *pos += len;
1867
1868 return len;
1869}
1870
1871static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
1872 const char __user *buf, size_t size, loff_t *pos)
1873{
1874 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
1875 char reg_offset[11];
1876 uint32_t *new = NULL, *tmp = NULL;
1877 int ret, i = 0, len = 0;
1878
1879 do {
1880 memset(reg_offset, 0, 11);
1881 if (copy_from_user(reg_offset, buf + len,
1882 min(10, ((int)size-len)))) {
1883 ret = -EFAULT;
1884 goto error_free;
1885 }
1886
1887 new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
1888 if (!new) {
1889 ret = -ENOMEM;
1890 goto error_free;
1891 }
1892 tmp = new;
1893 if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
1894 ret = -EINVAL;
1895 goto error_free;
1896 }
1897
1898 len += ret;
1899 i++;
1900 } while (len < size);
1901
1902 new = kmalloc_array(i, sizeof(uint32_t), GFP_KERNEL);
1903 if (!new) {
1904 ret = -ENOMEM;
1905 goto error_free;
1906 }
1907 ret = down_write_killable(&adev->reset_domain->sem);
1908 if (ret)
1909 goto error_free;
1910
1911 swap(adev->reset_dump_reg_list, tmp);
1912 swap(adev->reset_dump_reg_value, new);
1913 adev->num_regs = i;
1914 up_write(&adev->reset_domain->sem);
1915 ret = size;
1916
1917error_free:
1918 if (tmp != new)
1919 kfree(tmp);
1920 kfree(new);
1921 return ret;
1922}
1923
1924static const struct file_operations amdgpu_reset_dump_register_list = {
1925 .owner = THIS_MODULE,
1926 .read = amdgpu_reset_dump_register_list_read,
1927 .write = amdgpu_reset_dump_register_list_write,
1928 .llseek = default_llseek
1929};
1930
1931int amdgpu_debugfs_init(struct amdgpu_device *adev)
1932{
1933 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
1934 struct dentry *ent;
1935 int r, i;
1936
1937 if (!debugfs_initialized())
1938 return 0;
1939
1940 debugfs_create_x32("amdgpu_smu_debug", 0600, root,
1941 &adev->pm.smu_debug_mask);
1942
1943 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
1944 &fops_ib_preempt);
1945 if (IS_ERR(ent)) {
1946 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1947 return PTR_ERR(ent);
1948 }
1949
1950 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
1951 &fops_sclk_set);
1952 if (IS_ERR(ent)) {
1953 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1954 return PTR_ERR(ent);
1955 }
1956
1957 /* Register debugfs entries for amdgpu_ttm */
1958 amdgpu_ttm_debugfs_init(adev);
1959 amdgpu_debugfs_pm_init(adev);
1960 amdgpu_debugfs_sa_init(adev);
1961 amdgpu_debugfs_fence_init(adev);
1962 amdgpu_debugfs_gem_init(adev);
1963
1964 r = amdgpu_debugfs_regs_init(adev);
1965 if (r)
1966 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1967
1968 amdgpu_debugfs_firmware_init(adev);
1969 amdgpu_ta_if_debugfs_init(adev);
1970
1971#if defined(CONFIG_DRM_AMD_DC)
1972 if (adev->dc_enabled)
1973 dtn_debugfs_init(adev);
1974#endif
1975
1976 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1977 struct amdgpu_ring *ring = adev->rings[i];
1978
1979 if (!ring)
1980 continue;
1981
1982 amdgpu_debugfs_ring_init(adev, ring);
1983 }
1984
1985 for ( i = 0; i < adev->vcn.num_vcn_inst; i++) {
1986 if (!amdgpu_vcnfw_log)
1987 break;
1988
1989 if (adev->vcn.harvest_config & (1 << i))
1990 continue;
1991
1992 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
1993 }
1994
1995 amdgpu_ras_debugfs_create_all(adev);
1996 amdgpu_rap_debugfs_init(adev);
1997 amdgpu_securedisplay_debugfs_init(adev);
1998 amdgpu_fw_attestation_debugfs_init(adev);
1999
2000 debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
2001 &amdgpu_evict_vram_fops);
2002 debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
2003 &amdgpu_evict_gtt_fops);
2004 debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2005 &amdgpu_debugfs_test_ib_fops);
2006 debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2007 &amdgpu_debugfs_vm_info_fops);
2008 debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2009 &amdgpu_benchmark_fops);
2010 debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
2011 &amdgpu_reset_dump_register_list);
2012
2013 adev->debugfs_vbios_blob.data = adev->bios;
2014 adev->debugfs_vbios_blob.size = adev->bios_size;
2015 debugfs_create_blob("amdgpu_vbios", 0444, root,
2016 &adev->debugfs_vbios_blob);
2017
2018 adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2019 adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2020 debugfs_create_blob("amdgpu_discovery", 0444, root,
2021 &adev->debugfs_discovery_blob);
2022
2023 return 0;
2024}
2025
2026#else
2027int amdgpu_debugfs_init(struct amdgpu_device *adev)
2028{
2029 return 0;
2030}
2031int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2032{
2033 return 0;
2034}
2035#endif
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26#include <linux/kthread.h>
27#include <linux/pci.h>
28#include <linux/uaccess.h>
29
30#include <drm/drm_debugfs.h>
31
32#include "amdgpu.h"
33
34/**
35 * amdgpu_debugfs_add_files - Add simple debugfs entries
36 *
37 * @adev: Device to attach debugfs entries to
38 * @files: Array of function callbacks that respond to reads
39 * @nfiles: Number of callbacks to register
40 *
41 */
42int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
43 const struct drm_info_list *files,
44 unsigned nfiles)
45{
46 unsigned i;
47
48 for (i = 0; i < adev->debugfs_count; i++) {
49 if (adev->debugfs[i].files == files) {
50 /* Already registered */
51 return 0;
52 }
53 }
54
55 i = adev->debugfs_count + 1;
56 if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
57 DRM_ERROR("Reached maximum number of debugfs components.\n");
58 DRM_ERROR("Report so we increase "
59 "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
60 return -EINVAL;
61 }
62 adev->debugfs[adev->debugfs_count].files = files;
63 adev->debugfs[adev->debugfs_count].num_files = nfiles;
64 adev->debugfs_count = i;
65#if defined(CONFIG_DEBUG_FS)
66 drm_debugfs_create_files(files, nfiles,
67 adev->ddev->primary->debugfs_root,
68 adev->ddev->primary);
69#endif
70 return 0;
71}
72
73#if defined(CONFIG_DEBUG_FS)
74
75/**
76 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
77 *
78 * @read: True if reading
79 * @f: open file handle
80 * @buf: User buffer to write/read to
81 * @size: Number of bytes to write/read
82 * @pos: Offset to seek to
83 *
84 * This debugfs entry has special meaning on the offset being sought.
85 * Various bits have different meanings:
86 *
87 * Bit 62: Indicates a GRBM bank switch is needed
88 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
89 * zero)
90 * Bits 24..33: The SE or ME selector if needed
91 * Bits 34..43: The SH (or SA) or PIPE selector if needed
92 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
93 *
94 * Bit 23: Indicates that the PM power gating lock should be held
95 * This is necessary to read registers that might be
96 * unreliable during a power gating transistion.
97 *
98 * The lower bits are the BYTE offset of the register to read. This
99 * allows reading multiple registers in a single call and having
100 * the returned size reflect that.
101 */
102static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
103 char __user *buf, size_t size, loff_t *pos)
104{
105 struct amdgpu_device *adev = file_inode(f)->i_private;
106 ssize_t result = 0;
107 int r;
108 bool pm_pg_lock, use_bank, use_ring;
109 unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
110
111 pm_pg_lock = use_bank = use_ring = false;
112 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
113
114 if (size & 0x3 || *pos & 0x3 ||
115 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
116 return -EINVAL;
117
118 /* are we reading registers for which a PG lock is necessary? */
119 pm_pg_lock = (*pos >> 23) & 1;
120
121 if (*pos & (1ULL << 62)) {
122 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
123 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
124 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
125
126 if (se_bank == 0x3FF)
127 se_bank = 0xFFFFFFFF;
128 if (sh_bank == 0x3FF)
129 sh_bank = 0xFFFFFFFF;
130 if (instance_bank == 0x3FF)
131 instance_bank = 0xFFFFFFFF;
132 use_bank = 1;
133 } else if (*pos & (1ULL << 61)) {
134
135 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
136 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
137 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
138 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
139
140 use_ring = 1;
141 } else {
142 use_bank = use_ring = 0;
143 }
144
145 *pos &= (1UL << 22) - 1;
146
147 if (use_bank) {
148 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
149 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
150 return -EINVAL;
151 mutex_lock(&adev->grbm_idx_mutex);
152 amdgpu_gfx_select_se_sh(adev, se_bank,
153 sh_bank, instance_bank);
154 } else if (use_ring) {
155 mutex_lock(&adev->srbm_mutex);
156 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
157 }
158
159 if (pm_pg_lock)
160 mutex_lock(&adev->pm.mutex);
161
162 while (size) {
163 uint32_t value;
164
165 if (read) {
166 value = RREG32(*pos >> 2);
167 r = put_user(value, (uint32_t *)buf);
168 } else {
169 r = get_user(value, (uint32_t *)buf);
170 if (!r)
171 WREG32(*pos >> 2, value);
172 }
173 if (r) {
174 result = r;
175 goto end;
176 }
177
178 result += 4;
179 buf += 4;
180 *pos += 4;
181 size -= 4;
182 }
183
184end:
185 if (use_bank) {
186 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
187 mutex_unlock(&adev->grbm_idx_mutex);
188 } else if (use_ring) {
189 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
190 mutex_unlock(&adev->srbm_mutex);
191 }
192
193 if (pm_pg_lock)
194 mutex_unlock(&adev->pm.mutex);
195
196 return result;
197}
198
199/**
200 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
201 */
202static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
203 size_t size, loff_t *pos)
204{
205 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
206}
207
208/**
209 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
210 */
211static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
212 size_t size, loff_t *pos)
213{
214 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
215}
216
217
218/**
219 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
220 *
221 * @f: open file handle
222 * @buf: User buffer to store read data in
223 * @size: Number of bytes to read
224 * @pos: Offset to seek to
225 *
226 * The lower bits are the BYTE offset of the register to read. This
227 * allows reading multiple registers in a single call and having
228 * the returned size reflect that.
229 */
230static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
231 size_t size, loff_t *pos)
232{
233 struct amdgpu_device *adev = file_inode(f)->i_private;
234 ssize_t result = 0;
235 int r;
236
237 if (size & 0x3 || *pos & 0x3)
238 return -EINVAL;
239
240 while (size) {
241 uint32_t value;
242
243 value = RREG32_PCIE(*pos >> 2);
244 r = put_user(value, (uint32_t *)buf);
245 if (r)
246 return r;
247
248 result += 4;
249 buf += 4;
250 *pos += 4;
251 size -= 4;
252 }
253
254 return result;
255}
256
257/**
258 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
259 *
260 * @f: open file handle
261 * @buf: User buffer to write data from
262 * @size: Number of bytes to write
263 * @pos: Offset to seek to
264 *
265 * The lower bits are the BYTE offset of the register to write. This
266 * allows writing multiple registers in a single call and having
267 * the returned size reflect that.
268 */
269static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
270 size_t size, loff_t *pos)
271{
272 struct amdgpu_device *adev = file_inode(f)->i_private;
273 ssize_t result = 0;
274 int r;
275
276 if (size & 0x3 || *pos & 0x3)
277 return -EINVAL;
278
279 while (size) {
280 uint32_t value;
281
282 r = get_user(value, (uint32_t *)buf);
283 if (r)
284 return r;
285
286 WREG32_PCIE(*pos >> 2, value);
287
288 result += 4;
289 buf += 4;
290 *pos += 4;
291 size -= 4;
292 }
293
294 return result;
295}
296
297/**
298 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
299 *
300 * @f: open file handle
301 * @buf: User buffer to store read data in
302 * @size: Number of bytes to read
303 * @pos: Offset to seek to
304 *
305 * The lower bits are the BYTE offset of the register to read. This
306 * allows reading multiple registers in a single call and having
307 * the returned size reflect that.
308 */
309static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
310 size_t size, loff_t *pos)
311{
312 struct amdgpu_device *adev = file_inode(f)->i_private;
313 ssize_t result = 0;
314 int r;
315
316 if (size & 0x3 || *pos & 0x3)
317 return -EINVAL;
318
319 while (size) {
320 uint32_t value;
321
322 value = RREG32_DIDT(*pos >> 2);
323 r = put_user(value, (uint32_t *)buf);
324 if (r)
325 return r;
326
327 result += 4;
328 buf += 4;
329 *pos += 4;
330 size -= 4;
331 }
332
333 return result;
334}
335
336/**
337 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
338 *
339 * @f: open file handle
340 * @buf: User buffer to write data from
341 * @size: Number of bytes to write
342 * @pos: Offset to seek to
343 *
344 * The lower bits are the BYTE offset of the register to write. This
345 * allows writing multiple registers in a single call and having
346 * the returned size reflect that.
347 */
348static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
349 size_t size, loff_t *pos)
350{
351 struct amdgpu_device *adev = file_inode(f)->i_private;
352 ssize_t result = 0;
353 int r;
354
355 if (size & 0x3 || *pos & 0x3)
356 return -EINVAL;
357
358 while (size) {
359 uint32_t value;
360
361 r = get_user(value, (uint32_t *)buf);
362 if (r)
363 return r;
364
365 WREG32_DIDT(*pos >> 2, value);
366
367 result += 4;
368 buf += 4;
369 *pos += 4;
370 size -= 4;
371 }
372
373 return result;
374}
375
376/**
377 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
378 *
379 * @f: open file handle
380 * @buf: User buffer to store read data in
381 * @size: Number of bytes to read
382 * @pos: Offset to seek to
383 *
384 * The lower bits are the BYTE offset of the register to read. This
385 * allows reading multiple registers in a single call and having
386 * the returned size reflect that.
387 */
388static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
389 size_t size, loff_t *pos)
390{
391 struct amdgpu_device *adev = file_inode(f)->i_private;
392 ssize_t result = 0;
393 int r;
394
395 if (size & 0x3 || *pos & 0x3)
396 return -EINVAL;
397
398 while (size) {
399 uint32_t value;
400
401 value = RREG32_SMC(*pos);
402 r = put_user(value, (uint32_t *)buf);
403 if (r)
404 return r;
405
406 result += 4;
407 buf += 4;
408 *pos += 4;
409 size -= 4;
410 }
411
412 return result;
413}
414
415/**
416 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
417 *
418 * @f: open file handle
419 * @buf: User buffer to write data from
420 * @size: Number of bytes to write
421 * @pos: Offset to seek to
422 *
423 * The lower bits are the BYTE offset of the register to write. This
424 * allows writing multiple registers in a single call and having
425 * the returned size reflect that.
426 */
427static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
428 size_t size, loff_t *pos)
429{
430 struct amdgpu_device *adev = file_inode(f)->i_private;
431 ssize_t result = 0;
432 int r;
433
434 if (size & 0x3 || *pos & 0x3)
435 return -EINVAL;
436
437 while (size) {
438 uint32_t value;
439
440 r = get_user(value, (uint32_t *)buf);
441 if (r)
442 return r;
443
444 WREG32_SMC(*pos, value);
445
446 result += 4;
447 buf += 4;
448 *pos += 4;
449 size -= 4;
450 }
451
452 return result;
453}
454
455/**
456 * amdgpu_debugfs_gca_config_read - Read from gfx config data
457 *
458 * @f: open file handle
459 * @buf: User buffer to store read data in
460 * @size: Number of bytes to read
461 * @pos: Offset to seek to
462 *
463 * This file is used to access configuration data in a somewhat
464 * stable fashion. The format is a series of DWORDs with the first
465 * indicating which revision it is. New content is appended to the
466 * end so that older software can still read the data.
467 */
468
469static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
470 size_t size, loff_t *pos)
471{
472 struct amdgpu_device *adev = file_inode(f)->i_private;
473 ssize_t result = 0;
474 int r;
475 uint32_t *config, no_regs = 0;
476
477 if (size & 0x3 || *pos & 0x3)
478 return -EINVAL;
479
480 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
481 if (!config)
482 return -ENOMEM;
483
484 /* version, increment each time something is added */
485 config[no_regs++] = 3;
486 config[no_regs++] = adev->gfx.config.max_shader_engines;
487 config[no_regs++] = adev->gfx.config.max_tile_pipes;
488 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
489 config[no_regs++] = adev->gfx.config.max_sh_per_se;
490 config[no_regs++] = adev->gfx.config.max_backends_per_se;
491 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
492 config[no_regs++] = adev->gfx.config.max_gprs;
493 config[no_regs++] = adev->gfx.config.max_gs_threads;
494 config[no_regs++] = adev->gfx.config.max_hw_contexts;
495 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
496 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
497 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
498 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
499 config[no_regs++] = adev->gfx.config.num_tile_pipes;
500 config[no_regs++] = adev->gfx.config.backend_enable_mask;
501 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
502 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
503 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
504 config[no_regs++] = adev->gfx.config.num_gpus;
505 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
506 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
507 config[no_regs++] = adev->gfx.config.gb_addr_config;
508 config[no_regs++] = adev->gfx.config.num_rbs;
509
510 /* rev==1 */
511 config[no_regs++] = adev->rev_id;
512 config[no_regs++] = adev->pg_flags;
513 config[no_regs++] = adev->cg_flags;
514
515 /* rev==2 */
516 config[no_regs++] = adev->family;
517 config[no_regs++] = adev->external_rev_id;
518
519 /* rev==3 */
520 config[no_regs++] = adev->pdev->device;
521 config[no_regs++] = adev->pdev->revision;
522 config[no_regs++] = adev->pdev->subsystem_device;
523 config[no_regs++] = adev->pdev->subsystem_vendor;
524
525 while (size && (*pos < no_regs * 4)) {
526 uint32_t value;
527
528 value = config[*pos >> 2];
529 r = put_user(value, (uint32_t *)buf);
530 if (r) {
531 kfree(config);
532 return r;
533 }
534
535 result += 4;
536 buf += 4;
537 *pos += 4;
538 size -= 4;
539 }
540
541 kfree(config);
542 return result;
543}
544
545/**
546 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
547 *
548 * @f: open file handle
549 * @buf: User buffer to store read data in
550 * @size: Number of bytes to read
551 * @pos: Offset to seek to
552 *
553 * The offset is treated as the BYTE address of one of the sensors
554 * enumerated in amd/include/kgd_pp_interface.h under the
555 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
556 * you would use the offset 3 * 4 = 12.
557 */
558static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
559 size_t size, loff_t *pos)
560{
561 struct amdgpu_device *adev = file_inode(f)->i_private;
562 int idx, x, outsize, r, valuesize;
563 uint32_t values[16];
564
565 if (size & 3 || *pos & 0x3)
566 return -EINVAL;
567
568 if (!adev->pm.dpm_enabled)
569 return -EINVAL;
570
571 /* convert offset to sensor number */
572 idx = *pos >> 2;
573
574 valuesize = sizeof(values);
575 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
576 if (r)
577 return r;
578
579 if (size > valuesize)
580 return -EINVAL;
581
582 outsize = 0;
583 x = 0;
584 if (!r) {
585 while (size) {
586 r = put_user(values[x++], (int32_t *)buf);
587 buf += 4;
588 size -= 4;
589 outsize += 4;
590 }
591 }
592
593 return !r ? outsize : r;
594}
595
596/** amdgpu_debugfs_wave_read - Read WAVE STATUS data
597 *
598 * @f: open file handle
599 * @buf: User buffer to store read data in
600 * @size: Number of bytes to read
601 * @pos: Offset to seek to
602 *
603 * The offset being sought changes which wave that the status data
604 * will be returned for. The bits are used as follows:
605 *
606 * Bits 0..6: Byte offset into data
607 * Bits 7..14: SE selector
608 * Bits 15..22: SH/SA selector
609 * Bits 23..30: CU/{WGP+SIMD} selector
610 * Bits 31..36: WAVE ID selector
611 * Bits 37..44: SIMD ID selector
612 *
613 * The returned data begins with one DWORD of version information
614 * Followed by WAVE STATUS registers relevant to the GFX IP version
615 * being used. See gfx_v8_0_read_wave_data() for an example output.
616 */
617static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
618 size_t size, loff_t *pos)
619{
620 struct amdgpu_device *adev = f->f_inode->i_private;
621 int r, x;
622 ssize_t result=0;
623 uint32_t offset, se, sh, cu, wave, simd, data[32];
624
625 if (size & 3 || *pos & 3)
626 return -EINVAL;
627
628 /* decode offset */
629 offset = (*pos & GENMASK_ULL(6, 0));
630 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
631 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
632 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
633 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
634 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
635
636 /* switch to the specific se/sh/cu */
637 mutex_lock(&adev->grbm_idx_mutex);
638 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
639
640 x = 0;
641 if (adev->gfx.funcs->read_wave_data)
642 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
643
644 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
645 mutex_unlock(&adev->grbm_idx_mutex);
646
647 if (!x)
648 return -EINVAL;
649
650 while (size && (offset < x * 4)) {
651 uint32_t value;
652
653 value = data[offset >> 2];
654 r = put_user(value, (uint32_t *)buf);
655 if (r)
656 return r;
657
658 result += 4;
659 buf += 4;
660 offset += 4;
661 size -= 4;
662 }
663
664 return result;
665}
666
667/** amdgpu_debugfs_gpr_read - Read wave gprs
668 *
669 * @f: open file handle
670 * @buf: User buffer to store read data in
671 * @size: Number of bytes to read
672 * @pos: Offset to seek to
673 *
674 * The offset being sought changes which wave that the status data
675 * will be returned for. The bits are used as follows:
676 *
677 * Bits 0..11: Byte offset into data
678 * Bits 12..19: SE selector
679 * Bits 20..27: SH/SA selector
680 * Bits 28..35: CU/{WGP+SIMD} selector
681 * Bits 36..43: WAVE ID selector
682 * Bits 37..44: SIMD ID selector
683 * Bits 52..59: Thread selector
684 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
685 *
686 * The return data comes from the SGPR or VGPR register bank for
687 * the selected operational unit.
688 */
689static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
690 size_t size, loff_t *pos)
691{
692 struct amdgpu_device *adev = f->f_inode->i_private;
693 int r;
694 ssize_t result = 0;
695 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
696
697 if (size & 3 || *pos & 3)
698 return -EINVAL;
699
700 /* decode offset */
701 offset = *pos & GENMASK_ULL(11, 0);
702 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
703 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
704 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
705 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
706 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
707 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
708 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
709
710 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
711 if (!data)
712 return -ENOMEM;
713
714 /* switch to the specific se/sh/cu */
715 mutex_lock(&adev->grbm_idx_mutex);
716 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
717
718 if (bank == 0) {
719 if (adev->gfx.funcs->read_wave_vgprs)
720 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
721 } else {
722 if (adev->gfx.funcs->read_wave_sgprs)
723 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
724 }
725
726 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
727 mutex_unlock(&adev->grbm_idx_mutex);
728
729 while (size) {
730 uint32_t value;
731
732 value = data[offset++];
733 r = put_user(value, (uint32_t *)buf);
734 if (r) {
735 result = r;
736 goto err;
737 }
738
739 result += 4;
740 buf += 4;
741 size -= 4;
742 }
743
744err:
745 kfree(data);
746 return result;
747}
748
749static const struct file_operations amdgpu_debugfs_regs_fops = {
750 .owner = THIS_MODULE,
751 .read = amdgpu_debugfs_regs_read,
752 .write = amdgpu_debugfs_regs_write,
753 .llseek = default_llseek
754};
755static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
756 .owner = THIS_MODULE,
757 .read = amdgpu_debugfs_regs_didt_read,
758 .write = amdgpu_debugfs_regs_didt_write,
759 .llseek = default_llseek
760};
761static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
762 .owner = THIS_MODULE,
763 .read = amdgpu_debugfs_regs_pcie_read,
764 .write = amdgpu_debugfs_regs_pcie_write,
765 .llseek = default_llseek
766};
767static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
768 .owner = THIS_MODULE,
769 .read = amdgpu_debugfs_regs_smc_read,
770 .write = amdgpu_debugfs_regs_smc_write,
771 .llseek = default_llseek
772};
773
774static const struct file_operations amdgpu_debugfs_gca_config_fops = {
775 .owner = THIS_MODULE,
776 .read = amdgpu_debugfs_gca_config_read,
777 .llseek = default_llseek
778};
779
780static const struct file_operations amdgpu_debugfs_sensors_fops = {
781 .owner = THIS_MODULE,
782 .read = amdgpu_debugfs_sensor_read,
783 .llseek = default_llseek
784};
785
786static const struct file_operations amdgpu_debugfs_wave_fops = {
787 .owner = THIS_MODULE,
788 .read = amdgpu_debugfs_wave_read,
789 .llseek = default_llseek
790};
791static const struct file_operations amdgpu_debugfs_gpr_fops = {
792 .owner = THIS_MODULE,
793 .read = amdgpu_debugfs_gpr_read,
794 .llseek = default_llseek
795};
796
797static const struct file_operations *debugfs_regs[] = {
798 &amdgpu_debugfs_regs_fops,
799 &amdgpu_debugfs_regs_didt_fops,
800 &amdgpu_debugfs_regs_pcie_fops,
801 &amdgpu_debugfs_regs_smc_fops,
802 &amdgpu_debugfs_gca_config_fops,
803 &amdgpu_debugfs_sensors_fops,
804 &amdgpu_debugfs_wave_fops,
805 &amdgpu_debugfs_gpr_fops,
806};
807
808static const char *debugfs_regs_names[] = {
809 "amdgpu_regs",
810 "amdgpu_regs_didt",
811 "amdgpu_regs_pcie",
812 "amdgpu_regs_smc",
813 "amdgpu_gca_config",
814 "amdgpu_sensors",
815 "amdgpu_wave",
816 "amdgpu_gpr",
817};
818
819/**
820 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
821 * register access.
822 *
823 * @adev: The device to attach the debugfs entries to
824 */
825int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
826{
827 struct drm_minor *minor = adev->ddev->primary;
828 struct dentry *ent, *root = minor->debugfs_root;
829 unsigned int i;
830
831 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
832 ent = debugfs_create_file(debugfs_regs_names[i],
833 S_IFREG | S_IRUGO, root,
834 adev, debugfs_regs[i]);
835 if (!i && !IS_ERR_OR_NULL(ent))
836 i_size_write(ent->d_inode, adev->rmmio_size);
837 adev->debugfs_regs[i] = ent;
838 }
839
840 return 0;
841}
842
843void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
844{
845 unsigned i;
846
847 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
848 if (adev->debugfs_regs[i]) {
849 debugfs_remove(adev->debugfs_regs[i]);
850 adev->debugfs_regs[i] = NULL;
851 }
852 }
853}
854
855static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
856{
857 struct drm_info_node *node = (struct drm_info_node *) m->private;
858 struct drm_device *dev = node->minor->dev;
859 struct amdgpu_device *adev = dev->dev_private;
860 int r = 0, i;
861
862 /* hold on the scheduler */
863 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
864 struct amdgpu_ring *ring = adev->rings[i];
865
866 if (!ring || !ring->sched.thread)
867 continue;
868 kthread_park(ring->sched.thread);
869 }
870
871 seq_printf(m, "run ib test:\n");
872 r = amdgpu_ib_ring_tests(adev);
873 if (r)
874 seq_printf(m, "ib ring tests failed (%d).\n", r);
875 else
876 seq_printf(m, "ib ring tests passed.\n");
877
878 /* go on the scheduler */
879 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
880 struct amdgpu_ring *ring = adev->rings[i];
881
882 if (!ring || !ring->sched.thread)
883 continue;
884 kthread_unpark(ring->sched.thread);
885 }
886
887 return 0;
888}
889
890static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
891{
892 struct drm_info_node *node = (struct drm_info_node *) m->private;
893 struct drm_device *dev = node->minor->dev;
894 struct amdgpu_device *adev = dev->dev_private;
895
896 seq_write(m, adev->bios, adev->bios_size);
897 return 0;
898}
899
900static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data)
901{
902 struct drm_info_node *node = (struct drm_info_node *)m->private;
903 struct drm_device *dev = node->minor->dev;
904 struct amdgpu_device *adev = dev->dev_private;
905
906 seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
907 return 0;
908}
909
910static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data)
911{
912 struct drm_info_node *node = (struct drm_info_node *)m->private;
913 struct drm_device *dev = node->minor->dev;
914 struct amdgpu_device *adev = dev->dev_private;
915
916 seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT));
917 return 0;
918}
919
920static const struct drm_info_list amdgpu_debugfs_list[] = {
921 {"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump},
922 {"amdgpu_test_ib", &amdgpu_debugfs_test_ib},
923 {"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram},
924 {"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
925};
926
927static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
928 struct dma_fence **fences)
929{
930 struct amdgpu_fence_driver *drv = &ring->fence_drv;
931 uint32_t sync_seq, last_seq;
932
933 last_seq = atomic_read(&ring->fence_drv.last_seq);
934 sync_seq = ring->fence_drv.sync_seq;
935
936 last_seq &= drv->num_fences_mask;
937 sync_seq &= drv->num_fences_mask;
938
939 do {
940 struct dma_fence *fence, **ptr;
941
942 ++last_seq;
943 last_seq &= drv->num_fences_mask;
944 ptr = &drv->fences[last_seq];
945
946 fence = rcu_dereference_protected(*ptr, 1);
947 RCU_INIT_POINTER(*ptr, NULL);
948
949 if (!fence)
950 continue;
951
952 fences[last_seq] = fence;
953
954 } while (last_seq != sync_seq);
955}
956
957static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
958 int length)
959{
960 int i;
961 struct dma_fence *fence;
962
963 for (i = 0; i < length; i++) {
964 fence = fences[i];
965 if (!fence)
966 continue;
967 dma_fence_signal(fence);
968 dma_fence_put(fence);
969 }
970}
971
972static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
973{
974 struct drm_sched_job *s_job;
975 struct dma_fence *fence;
976
977 spin_lock(&sched->job_list_lock);
978 list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
979 fence = sched->ops->run_job(s_job);
980 dma_fence_put(fence);
981 }
982 spin_unlock(&sched->job_list_lock);
983}
984
985static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
986{
987 struct amdgpu_job *job;
988 struct drm_sched_job *s_job;
989 uint32_t preempt_seq;
990 struct dma_fence *fence, **ptr;
991 struct amdgpu_fence_driver *drv = &ring->fence_drv;
992 struct drm_gpu_scheduler *sched = &ring->sched;
993
994 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
995 return;
996
997 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
998 if (preempt_seq <= atomic_read(&drv->last_seq))
999 return;
1000
1001 preempt_seq &= drv->num_fences_mask;
1002 ptr = &drv->fences[preempt_seq];
1003 fence = rcu_dereference_protected(*ptr, 1);
1004
1005 spin_lock(&sched->job_list_lock);
1006 list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1007 job = to_amdgpu_job(s_job);
1008 if (job->fence == fence)
1009 /* mark the job as preempted */
1010 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1011 }
1012 spin_unlock(&sched->job_list_lock);
1013}
1014
1015static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1016{
1017 int r, resched, length;
1018 struct amdgpu_ring *ring;
1019 struct dma_fence **fences = NULL;
1020 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1021
1022 if (val >= AMDGPU_MAX_RINGS)
1023 return -EINVAL;
1024
1025 ring = adev->rings[val];
1026
1027 if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1028 return -EINVAL;
1029
1030 /* the last preemption failed */
1031 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1032 return -EBUSY;
1033
1034 length = ring->fence_drv.num_fences_mask + 1;
1035 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1036 if (!fences)
1037 return -ENOMEM;
1038
1039 /* stop the scheduler */
1040 kthread_park(ring->sched.thread);
1041
1042 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1043
1044 /* preempt the IB */
1045 r = amdgpu_ring_preempt_ib(ring);
1046 if (r) {
1047 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1048 goto failure;
1049 }
1050
1051 amdgpu_fence_process(ring);
1052
1053 if (atomic_read(&ring->fence_drv.last_seq) !=
1054 ring->fence_drv.sync_seq) {
1055 DRM_INFO("ring %d was preempted\n", ring->idx);
1056
1057 amdgpu_ib_preempt_mark_partial_job(ring);
1058
1059 /* swap out the old fences */
1060 amdgpu_ib_preempt_fences_swap(ring, fences);
1061
1062 amdgpu_fence_driver_force_completion(ring);
1063
1064 /* resubmit unfinished jobs */
1065 amdgpu_ib_preempt_job_recovery(&ring->sched);
1066
1067 /* wait for jobs finished */
1068 amdgpu_fence_wait_empty(ring);
1069
1070 /* signal the old fences */
1071 amdgpu_ib_preempt_signal_fences(fences, length);
1072 }
1073
1074failure:
1075 /* restart the scheduler */
1076 kthread_unpark(ring->sched.thread);
1077
1078 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1079
1080 if (fences)
1081 kfree(fences);
1082
1083 return 0;
1084}
1085
1086DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
1087 amdgpu_debugfs_ib_preempt, "%llu\n");
1088
1089int amdgpu_debugfs_init(struct amdgpu_device *adev)
1090{
1091 adev->debugfs_preempt =
1092 debugfs_create_file("amdgpu_preempt_ib", 0600,
1093 adev->ddev->primary->debugfs_root,
1094 (void *)adev, &fops_ib_preempt);
1095 if (!(adev->debugfs_preempt)) {
1096 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1097 return -EIO;
1098 }
1099
1100 return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
1101 ARRAY_SIZE(amdgpu_debugfs_list));
1102}
1103
1104void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev)
1105{
1106 if (adev->debugfs_preempt)
1107 debugfs_remove(adev->debugfs_preempt);
1108}
1109
1110#else
1111int amdgpu_debugfs_init(struct amdgpu_device *adev)
1112{
1113 return 0;
1114}
1115void amdgpu_debugfs_preempt_cleanup(struct amdgpu_device *adev) { }
1116int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1117{
1118 return 0;
1119}
1120void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
1121#endif