Loading...
Note: File does not exist in v3.1.
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26#include <linux/kthread.h>
27#include <linux/pci.h>
28#include <linux/uaccess.h>
29#include <linux/pm_runtime.h>
30#include <linux/poll.h>
31
32#include "amdgpu.h"
33#include "amdgpu_pm.h"
34#include "amdgpu_dm_debugfs.h"
35#include "amdgpu_ras.h"
36#include "amdgpu_rap.h"
37#include "amdgpu_securedisplay.h"
38#include "amdgpu_fw_attestation.h"
39
40int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
41{
42#if defined(CONFIG_DEBUG_FS)
43 unsigned long timeout = 600 * HZ;
44 int ret;
45
46 wake_up_interruptible(&adev->autodump.gpu_hang);
47
48 ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
49 if (ret == 0) {
50 pr_err("autodump: timeout, move on to gpu recovery\n");
51 return -ETIMEDOUT;
52 }
53#endif
54 return 0;
55}
56
57#if defined(CONFIG_DEBUG_FS)
58
59static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
60{
61 struct amdgpu_device *adev = inode->i_private;
62 int ret;
63
64 file->private_data = adev;
65
66 ret = down_read_killable(&adev->reset_sem);
67 if (ret)
68 return ret;
69
70 if (adev->autodump.dumping.done) {
71 reinit_completion(&adev->autodump.dumping);
72 ret = 0;
73 } else {
74 ret = -EBUSY;
75 }
76
77 up_read(&adev->reset_sem);
78
79 return ret;
80}
81
82static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
83{
84 struct amdgpu_device *adev = file->private_data;
85
86 complete_all(&adev->autodump.dumping);
87 return 0;
88}
89
90static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
91{
92 struct amdgpu_device *adev = file->private_data;
93
94 poll_wait(file, &adev->autodump.gpu_hang, poll_table);
95
96 if (amdgpu_in_reset(adev))
97 return POLLIN | POLLRDNORM | POLLWRNORM;
98
99 return 0;
100}
101
102static const struct file_operations autodump_debug_fops = {
103 .owner = THIS_MODULE,
104 .open = amdgpu_debugfs_autodump_open,
105 .poll = amdgpu_debugfs_autodump_poll,
106 .release = amdgpu_debugfs_autodump_release,
107};
108
109static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
110{
111 init_completion(&adev->autodump.dumping);
112 complete_all(&adev->autodump.dumping);
113 init_waitqueue_head(&adev->autodump.gpu_hang);
114
115 debugfs_create_file("amdgpu_autodump", 0600,
116 adev_to_drm(adev)->primary->debugfs_root,
117 adev, &autodump_debug_fops);
118}
119
120/**
121 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
122 *
123 * @read: True if reading
124 * @f: open file handle
125 * @buf: User buffer to write/read to
126 * @size: Number of bytes to write/read
127 * @pos: Offset to seek to
128 *
129 * This debugfs entry has special meaning on the offset being sought.
130 * Various bits have different meanings:
131 *
132 * Bit 62: Indicates a GRBM bank switch is needed
133 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
134 * zero)
135 * Bits 24..33: The SE or ME selector if needed
136 * Bits 34..43: The SH (or SA) or PIPE selector if needed
137 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
138 *
139 * Bit 23: Indicates that the PM power gating lock should be held
140 * This is necessary to read registers that might be
141 * unreliable during a power gating transistion.
142 *
143 * The lower bits are the BYTE offset of the register to read. This
144 * allows reading multiple registers in a single call and having
145 * the returned size reflect that.
146 */
147static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
148 char __user *buf, size_t size, loff_t *pos)
149{
150 struct amdgpu_device *adev = file_inode(f)->i_private;
151 ssize_t result = 0;
152 int r;
153 bool pm_pg_lock, use_bank, use_ring;
154 unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
155
156 pm_pg_lock = use_bank = use_ring = false;
157 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
158
159 if (size & 0x3 || *pos & 0x3 ||
160 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
161 return -EINVAL;
162
163 /* are we reading registers for which a PG lock is necessary? */
164 pm_pg_lock = (*pos >> 23) & 1;
165
166 if (*pos & (1ULL << 62)) {
167 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
168 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
169 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
170
171 if (se_bank == 0x3FF)
172 se_bank = 0xFFFFFFFF;
173 if (sh_bank == 0x3FF)
174 sh_bank = 0xFFFFFFFF;
175 if (instance_bank == 0x3FF)
176 instance_bank = 0xFFFFFFFF;
177 use_bank = true;
178 } else if (*pos & (1ULL << 61)) {
179
180 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
181 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
182 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
183 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
184
185 use_ring = true;
186 } else {
187 use_bank = use_ring = false;
188 }
189
190 *pos &= (1UL << 22) - 1;
191
192 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
193 if (r < 0) {
194 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
195 return r;
196 }
197
198 r = amdgpu_virt_enable_access_debugfs(adev);
199 if (r < 0) {
200 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
201 return r;
202 }
203
204 if (use_bank) {
205 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
206 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
207 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
208 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
209 amdgpu_virt_disable_access_debugfs(adev);
210 return -EINVAL;
211 }
212 mutex_lock(&adev->grbm_idx_mutex);
213 amdgpu_gfx_select_se_sh(adev, se_bank,
214 sh_bank, instance_bank);
215 } else if (use_ring) {
216 mutex_lock(&adev->srbm_mutex);
217 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
218 }
219
220 if (pm_pg_lock)
221 mutex_lock(&adev->pm.mutex);
222
223 while (size) {
224 uint32_t value;
225
226 if (read) {
227 value = RREG32(*pos >> 2);
228 r = put_user(value, (uint32_t *)buf);
229 } else {
230 r = get_user(value, (uint32_t *)buf);
231 if (!r)
232 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
233 }
234 if (r) {
235 result = r;
236 goto end;
237 }
238
239 result += 4;
240 buf += 4;
241 *pos += 4;
242 size -= 4;
243 }
244
245end:
246 if (use_bank) {
247 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
248 mutex_unlock(&adev->grbm_idx_mutex);
249 } else if (use_ring) {
250 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
251 mutex_unlock(&adev->srbm_mutex);
252 }
253
254 if (pm_pg_lock)
255 mutex_unlock(&adev->pm.mutex);
256
257 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
258 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
259
260 amdgpu_virt_disable_access_debugfs(adev);
261 return result;
262}
263
264/*
265 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
266 */
267static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
268 size_t size, loff_t *pos)
269{
270 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
271}
272
273/*
274 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
275 */
276static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
277 size_t size, loff_t *pos)
278{
279 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
280}
281
282
283/**
284 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
285 *
286 * @f: open file handle
287 * @buf: User buffer to store read data in
288 * @size: Number of bytes to read
289 * @pos: Offset to seek to
290 *
291 * The lower bits are the BYTE offset of the register to read. This
292 * allows reading multiple registers in a single call and having
293 * the returned size reflect that.
294 */
295static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
296 size_t size, loff_t *pos)
297{
298 struct amdgpu_device *adev = file_inode(f)->i_private;
299 ssize_t result = 0;
300 int r;
301
302 if (size & 0x3 || *pos & 0x3)
303 return -EINVAL;
304
305 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
306 if (r < 0) {
307 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
308 return r;
309 }
310
311 r = amdgpu_virt_enable_access_debugfs(adev);
312 if (r < 0) {
313 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
314 return r;
315 }
316
317 while (size) {
318 uint32_t value;
319
320 value = RREG32_PCIE(*pos);
321 r = put_user(value, (uint32_t *)buf);
322 if (r) {
323 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
324 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
325 amdgpu_virt_disable_access_debugfs(adev);
326 return r;
327 }
328
329 result += 4;
330 buf += 4;
331 *pos += 4;
332 size -= 4;
333 }
334
335 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
336 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
337
338 amdgpu_virt_disable_access_debugfs(adev);
339 return result;
340}
341
342/**
343 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
344 *
345 * @f: open file handle
346 * @buf: User buffer to write data from
347 * @size: Number of bytes to write
348 * @pos: Offset to seek to
349 *
350 * The lower bits are the BYTE offset of the register to write. This
351 * allows writing multiple registers in a single call and having
352 * the returned size reflect that.
353 */
354static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
355 size_t size, loff_t *pos)
356{
357 struct amdgpu_device *adev = file_inode(f)->i_private;
358 ssize_t result = 0;
359 int r;
360
361 if (size & 0x3 || *pos & 0x3)
362 return -EINVAL;
363
364 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
365 if (r < 0) {
366 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
367 return r;
368 }
369
370 r = amdgpu_virt_enable_access_debugfs(adev);
371 if (r < 0) {
372 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
373 return r;
374 }
375
376 while (size) {
377 uint32_t value;
378
379 r = get_user(value, (uint32_t *)buf);
380 if (r) {
381 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
382 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
383 amdgpu_virt_disable_access_debugfs(adev);
384 return r;
385 }
386
387 WREG32_PCIE(*pos, value);
388
389 result += 4;
390 buf += 4;
391 *pos += 4;
392 size -= 4;
393 }
394
395 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
396 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
397
398 amdgpu_virt_disable_access_debugfs(adev);
399 return result;
400}
401
402/**
403 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
404 *
405 * @f: open file handle
406 * @buf: User buffer to store read data in
407 * @size: Number of bytes to read
408 * @pos: Offset to seek to
409 *
410 * The lower bits are the BYTE offset of the register to read. This
411 * allows reading multiple registers in a single call and having
412 * the returned size reflect that.
413 */
414static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
415 size_t size, loff_t *pos)
416{
417 struct amdgpu_device *adev = file_inode(f)->i_private;
418 ssize_t result = 0;
419 int r;
420
421 if (size & 0x3 || *pos & 0x3)
422 return -EINVAL;
423
424 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
425 if (r < 0) {
426 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
427 return r;
428 }
429
430 r = amdgpu_virt_enable_access_debugfs(adev);
431 if (r < 0) {
432 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
433 return r;
434 }
435
436 while (size) {
437 uint32_t value;
438
439 value = RREG32_DIDT(*pos >> 2);
440 r = put_user(value, (uint32_t *)buf);
441 if (r) {
442 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
443 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
444 amdgpu_virt_disable_access_debugfs(adev);
445 return r;
446 }
447
448 result += 4;
449 buf += 4;
450 *pos += 4;
451 size -= 4;
452 }
453
454 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
455 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
456
457 amdgpu_virt_disable_access_debugfs(adev);
458 return result;
459}
460
461/**
462 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
463 *
464 * @f: open file handle
465 * @buf: User buffer to write data from
466 * @size: Number of bytes to write
467 * @pos: Offset to seek to
468 *
469 * The lower bits are the BYTE offset of the register to write. This
470 * allows writing multiple registers in a single call and having
471 * the returned size reflect that.
472 */
473static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
474 size_t size, loff_t *pos)
475{
476 struct amdgpu_device *adev = file_inode(f)->i_private;
477 ssize_t result = 0;
478 int r;
479
480 if (size & 0x3 || *pos & 0x3)
481 return -EINVAL;
482
483 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
484 if (r < 0) {
485 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
486 return r;
487 }
488
489 r = amdgpu_virt_enable_access_debugfs(adev);
490 if (r < 0) {
491 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
492 return r;
493 }
494
495 while (size) {
496 uint32_t value;
497
498 r = get_user(value, (uint32_t *)buf);
499 if (r) {
500 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
501 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
502 amdgpu_virt_disable_access_debugfs(adev);
503 return r;
504 }
505
506 WREG32_DIDT(*pos >> 2, value);
507
508 result += 4;
509 buf += 4;
510 *pos += 4;
511 size -= 4;
512 }
513
514 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
515 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
516
517 amdgpu_virt_disable_access_debugfs(adev);
518 return result;
519}
520
521/**
522 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
523 *
524 * @f: open file handle
525 * @buf: User buffer to store read data in
526 * @size: Number of bytes to read
527 * @pos: Offset to seek to
528 *
529 * The lower bits are the BYTE offset of the register to read. This
530 * allows reading multiple registers in a single call and having
531 * the returned size reflect that.
532 */
533static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
534 size_t size, loff_t *pos)
535{
536 struct amdgpu_device *adev = file_inode(f)->i_private;
537 ssize_t result = 0;
538 int r;
539
540 if (size & 0x3 || *pos & 0x3)
541 return -EINVAL;
542
543 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
544 if (r < 0) {
545 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
546 return r;
547 }
548
549 r = amdgpu_virt_enable_access_debugfs(adev);
550 if (r < 0) {
551 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
552 return r;
553 }
554
555 while (size) {
556 uint32_t value;
557
558 value = RREG32_SMC(*pos);
559 r = put_user(value, (uint32_t *)buf);
560 if (r) {
561 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
562 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
563 amdgpu_virt_disable_access_debugfs(adev);
564 return r;
565 }
566
567 result += 4;
568 buf += 4;
569 *pos += 4;
570 size -= 4;
571 }
572
573 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
574 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
575
576 amdgpu_virt_disable_access_debugfs(adev);
577 return result;
578}
579
580/**
581 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
582 *
583 * @f: open file handle
584 * @buf: User buffer to write data from
585 * @size: Number of bytes to write
586 * @pos: Offset to seek to
587 *
588 * The lower bits are the BYTE offset of the register to write. This
589 * allows writing multiple registers in a single call and having
590 * the returned size reflect that.
591 */
592static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
593 size_t size, loff_t *pos)
594{
595 struct amdgpu_device *adev = file_inode(f)->i_private;
596 ssize_t result = 0;
597 int r;
598
599 if (size & 0x3 || *pos & 0x3)
600 return -EINVAL;
601
602 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
603 if (r < 0) {
604 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
605 return r;
606 }
607
608 r = amdgpu_virt_enable_access_debugfs(adev);
609 if (r < 0) {
610 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
611 return r;
612 }
613
614 while (size) {
615 uint32_t value;
616
617 r = get_user(value, (uint32_t *)buf);
618 if (r) {
619 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
620 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
621 amdgpu_virt_disable_access_debugfs(adev);
622 return r;
623 }
624
625 WREG32_SMC(*pos, value);
626
627 result += 4;
628 buf += 4;
629 *pos += 4;
630 size -= 4;
631 }
632
633 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
634 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
635
636 amdgpu_virt_disable_access_debugfs(adev);
637 return result;
638}
639
640/**
641 * amdgpu_debugfs_gca_config_read - Read from gfx config data
642 *
643 * @f: open file handle
644 * @buf: User buffer to store read data in
645 * @size: Number of bytes to read
646 * @pos: Offset to seek to
647 *
648 * This file is used to access configuration data in a somewhat
649 * stable fashion. The format is a series of DWORDs with the first
650 * indicating which revision it is. New content is appended to the
651 * end so that older software can still read the data.
652 */
653
654static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
655 size_t size, loff_t *pos)
656{
657 struct amdgpu_device *adev = file_inode(f)->i_private;
658 ssize_t result = 0;
659 int r;
660 uint32_t *config, no_regs = 0;
661
662 if (size & 0x3 || *pos & 0x3)
663 return -EINVAL;
664
665 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
666 if (!config)
667 return -ENOMEM;
668
669 /* version, increment each time something is added */
670 config[no_regs++] = 3;
671 config[no_regs++] = adev->gfx.config.max_shader_engines;
672 config[no_regs++] = adev->gfx.config.max_tile_pipes;
673 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
674 config[no_regs++] = adev->gfx.config.max_sh_per_se;
675 config[no_regs++] = adev->gfx.config.max_backends_per_se;
676 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
677 config[no_regs++] = adev->gfx.config.max_gprs;
678 config[no_regs++] = adev->gfx.config.max_gs_threads;
679 config[no_regs++] = adev->gfx.config.max_hw_contexts;
680 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
681 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
682 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
683 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
684 config[no_regs++] = adev->gfx.config.num_tile_pipes;
685 config[no_regs++] = adev->gfx.config.backend_enable_mask;
686 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
687 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
688 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
689 config[no_regs++] = adev->gfx.config.num_gpus;
690 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
691 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
692 config[no_regs++] = adev->gfx.config.gb_addr_config;
693 config[no_regs++] = adev->gfx.config.num_rbs;
694
695 /* rev==1 */
696 config[no_regs++] = adev->rev_id;
697 config[no_regs++] = adev->pg_flags;
698 config[no_regs++] = adev->cg_flags;
699
700 /* rev==2 */
701 config[no_regs++] = adev->family;
702 config[no_regs++] = adev->external_rev_id;
703
704 /* rev==3 */
705 config[no_regs++] = adev->pdev->device;
706 config[no_regs++] = adev->pdev->revision;
707 config[no_regs++] = adev->pdev->subsystem_device;
708 config[no_regs++] = adev->pdev->subsystem_vendor;
709
710 while (size && (*pos < no_regs * 4)) {
711 uint32_t value;
712
713 value = config[*pos >> 2];
714 r = put_user(value, (uint32_t *)buf);
715 if (r) {
716 kfree(config);
717 return r;
718 }
719
720 result += 4;
721 buf += 4;
722 *pos += 4;
723 size -= 4;
724 }
725
726 kfree(config);
727 return result;
728}
729
730/**
731 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
732 *
733 * @f: open file handle
734 * @buf: User buffer to store read data in
735 * @size: Number of bytes to read
736 * @pos: Offset to seek to
737 *
738 * The offset is treated as the BYTE address of one of the sensors
739 * enumerated in amd/include/kgd_pp_interface.h under the
740 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
741 * you would use the offset 3 * 4 = 12.
742 */
743static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
744 size_t size, loff_t *pos)
745{
746 struct amdgpu_device *adev = file_inode(f)->i_private;
747 int idx, x, outsize, r, valuesize;
748 uint32_t values[16];
749
750 if (size & 3 || *pos & 0x3)
751 return -EINVAL;
752
753 if (!adev->pm.dpm_enabled)
754 return -EINVAL;
755
756 /* convert offset to sensor number */
757 idx = *pos >> 2;
758
759 valuesize = sizeof(values);
760
761 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
762 if (r < 0) {
763 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
764 return r;
765 }
766
767 r = amdgpu_virt_enable_access_debugfs(adev);
768 if (r < 0) {
769 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
770 return r;
771 }
772
773 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
774
775 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
776 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
777
778 if (r) {
779 amdgpu_virt_disable_access_debugfs(adev);
780 return r;
781 }
782
783 if (size > valuesize) {
784 amdgpu_virt_disable_access_debugfs(adev);
785 return -EINVAL;
786 }
787
788 outsize = 0;
789 x = 0;
790 if (!r) {
791 while (size) {
792 r = put_user(values[x++], (int32_t *)buf);
793 buf += 4;
794 size -= 4;
795 outsize += 4;
796 }
797 }
798
799 amdgpu_virt_disable_access_debugfs(adev);
800 return !r ? outsize : r;
801}
802
803/** amdgpu_debugfs_wave_read - Read WAVE STATUS data
804 *
805 * @f: open file handle
806 * @buf: User buffer to store read data in
807 * @size: Number of bytes to read
808 * @pos: Offset to seek to
809 *
810 * The offset being sought changes which wave that the status data
811 * will be returned for. The bits are used as follows:
812 *
813 * Bits 0..6: Byte offset into data
814 * Bits 7..14: SE selector
815 * Bits 15..22: SH/SA selector
816 * Bits 23..30: CU/{WGP+SIMD} selector
817 * Bits 31..36: WAVE ID selector
818 * Bits 37..44: SIMD ID selector
819 *
820 * The returned data begins with one DWORD of version information
821 * Followed by WAVE STATUS registers relevant to the GFX IP version
822 * being used. See gfx_v8_0_read_wave_data() for an example output.
823 */
824static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
825 size_t size, loff_t *pos)
826{
827 struct amdgpu_device *adev = f->f_inode->i_private;
828 int r, x;
829 ssize_t result = 0;
830 uint32_t offset, se, sh, cu, wave, simd, data[32];
831
832 if (size & 3 || *pos & 3)
833 return -EINVAL;
834
835 /* decode offset */
836 offset = (*pos & GENMASK_ULL(6, 0));
837 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
838 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
839 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
840 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
841 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
842
843 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
844 if (r < 0) {
845 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
846 return r;
847 }
848
849 r = amdgpu_virt_enable_access_debugfs(adev);
850 if (r < 0) {
851 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
852 return r;
853 }
854
855 /* switch to the specific se/sh/cu */
856 mutex_lock(&adev->grbm_idx_mutex);
857 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
858
859 x = 0;
860 if (adev->gfx.funcs->read_wave_data)
861 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
862
863 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
864 mutex_unlock(&adev->grbm_idx_mutex);
865
866 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
867 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
868
869 if (!x) {
870 amdgpu_virt_disable_access_debugfs(adev);
871 return -EINVAL;
872 }
873
874 while (size && (offset < x * 4)) {
875 uint32_t value;
876
877 value = data[offset >> 2];
878 r = put_user(value, (uint32_t *)buf);
879 if (r) {
880 amdgpu_virt_disable_access_debugfs(adev);
881 return r;
882 }
883
884 result += 4;
885 buf += 4;
886 offset += 4;
887 size -= 4;
888 }
889
890 amdgpu_virt_disable_access_debugfs(adev);
891 return result;
892}
893
894/** amdgpu_debugfs_gpr_read - Read wave gprs
895 *
896 * @f: open file handle
897 * @buf: User buffer to store read data in
898 * @size: Number of bytes to read
899 * @pos: Offset to seek to
900 *
901 * The offset being sought changes which wave that the status data
902 * will be returned for. The bits are used as follows:
903 *
904 * Bits 0..11: Byte offset into data
905 * Bits 12..19: SE selector
906 * Bits 20..27: SH/SA selector
907 * Bits 28..35: CU/{WGP+SIMD} selector
908 * Bits 36..43: WAVE ID selector
909 * Bits 37..44: SIMD ID selector
910 * Bits 52..59: Thread selector
911 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
912 *
913 * The return data comes from the SGPR or VGPR register bank for
914 * the selected operational unit.
915 */
916static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
917 size_t size, loff_t *pos)
918{
919 struct amdgpu_device *adev = f->f_inode->i_private;
920 int r;
921 ssize_t result = 0;
922 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
923
924 if (size > 4096 || size & 3 || *pos & 3)
925 return -EINVAL;
926
927 /* decode offset */
928 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
929 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
930 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
931 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
932 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
933 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
934 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
935 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
936
937 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
938 if (!data)
939 return -ENOMEM;
940
941 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
942 if (r < 0)
943 goto err;
944
945 r = amdgpu_virt_enable_access_debugfs(adev);
946 if (r < 0)
947 goto err;
948
949 /* switch to the specific se/sh/cu */
950 mutex_lock(&adev->grbm_idx_mutex);
951 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
952
953 if (bank == 0) {
954 if (adev->gfx.funcs->read_wave_vgprs)
955 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
956 } else {
957 if (adev->gfx.funcs->read_wave_sgprs)
958 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
959 }
960
961 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
962 mutex_unlock(&adev->grbm_idx_mutex);
963
964 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
965 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
966
967 while (size) {
968 uint32_t value;
969
970 value = data[result >> 2];
971 r = put_user(value, (uint32_t *)buf);
972 if (r) {
973 amdgpu_virt_disable_access_debugfs(adev);
974 goto err;
975 }
976
977 result += 4;
978 buf += 4;
979 size -= 4;
980 }
981
982 kfree(data);
983 amdgpu_virt_disable_access_debugfs(adev);
984 return result;
985
986err:
987 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
988 kfree(data);
989 return r;
990}
991
992/**
993 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
994 *
995 * @f: open file handle
996 * @buf: User buffer to write data from
997 * @size: Number of bytes to write
998 * @pos: Offset to seek to
999 *
1000 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1001 */
1002static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1003 size_t size, loff_t *pos)
1004{
1005 struct amdgpu_device *adev = file_inode(f)->i_private;
1006 ssize_t result = 0;
1007 int r;
1008
1009 if (size & 0x3 || *pos & 0x3)
1010 return -EINVAL;
1011
1012 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1013 if (r < 0) {
1014 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1015 return r;
1016 }
1017
1018 while (size) {
1019 uint32_t value;
1020
1021 r = get_user(value, (uint32_t *)buf);
1022 if (r) {
1023 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1024 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1025 return r;
1026 }
1027
1028 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1029
1030 result += 4;
1031 buf += 4;
1032 *pos += 4;
1033 size -= 4;
1034 }
1035
1036 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1037 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1038
1039 return result;
1040}
1041
1042
1043/**
1044 * amdgpu_debugfs_gfxoff_read - read gfxoff status
1045 *
1046 * @f: open file handle
1047 * @buf: User buffer to store read data in
1048 * @size: Number of bytes to read
1049 * @pos: Offset to seek to
1050 */
1051static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1052 size_t size, loff_t *pos)
1053{
1054 struct amdgpu_device *adev = file_inode(f)->i_private;
1055 ssize_t result = 0;
1056 int r;
1057
1058 if (size & 0x3 || *pos & 0x3)
1059 return -EINVAL;
1060
1061 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1062 if (r < 0)
1063 return r;
1064
1065 while (size) {
1066 uint32_t value;
1067
1068 r = amdgpu_get_gfx_off_status(adev, &value);
1069 if (r) {
1070 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1071 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1072 return r;
1073 }
1074
1075 r = put_user(value, (uint32_t *)buf);
1076 if (r) {
1077 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1078 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1079 return r;
1080 }
1081
1082 result += 4;
1083 buf += 4;
1084 *pos += 4;
1085 size -= 4;
1086 }
1087
1088 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1089 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1090
1091 return result;
1092}
1093
1094static const struct file_operations amdgpu_debugfs_regs_fops = {
1095 .owner = THIS_MODULE,
1096 .read = amdgpu_debugfs_regs_read,
1097 .write = amdgpu_debugfs_regs_write,
1098 .llseek = default_llseek
1099};
1100static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1101 .owner = THIS_MODULE,
1102 .read = amdgpu_debugfs_regs_didt_read,
1103 .write = amdgpu_debugfs_regs_didt_write,
1104 .llseek = default_llseek
1105};
1106static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1107 .owner = THIS_MODULE,
1108 .read = amdgpu_debugfs_regs_pcie_read,
1109 .write = amdgpu_debugfs_regs_pcie_write,
1110 .llseek = default_llseek
1111};
1112static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1113 .owner = THIS_MODULE,
1114 .read = amdgpu_debugfs_regs_smc_read,
1115 .write = amdgpu_debugfs_regs_smc_write,
1116 .llseek = default_llseek
1117};
1118
1119static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1120 .owner = THIS_MODULE,
1121 .read = amdgpu_debugfs_gca_config_read,
1122 .llseek = default_llseek
1123};
1124
1125static const struct file_operations amdgpu_debugfs_sensors_fops = {
1126 .owner = THIS_MODULE,
1127 .read = amdgpu_debugfs_sensor_read,
1128 .llseek = default_llseek
1129};
1130
1131static const struct file_operations amdgpu_debugfs_wave_fops = {
1132 .owner = THIS_MODULE,
1133 .read = amdgpu_debugfs_wave_read,
1134 .llseek = default_llseek
1135};
1136static const struct file_operations amdgpu_debugfs_gpr_fops = {
1137 .owner = THIS_MODULE,
1138 .read = amdgpu_debugfs_gpr_read,
1139 .llseek = default_llseek
1140};
1141
1142static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1143 .owner = THIS_MODULE,
1144 .read = amdgpu_debugfs_gfxoff_read,
1145 .write = amdgpu_debugfs_gfxoff_write,
1146 .llseek = default_llseek
1147};
1148
1149static const struct file_operations *debugfs_regs[] = {
1150 &amdgpu_debugfs_regs_fops,
1151 &amdgpu_debugfs_regs_didt_fops,
1152 &amdgpu_debugfs_regs_pcie_fops,
1153 &amdgpu_debugfs_regs_smc_fops,
1154 &amdgpu_debugfs_gca_config_fops,
1155 &amdgpu_debugfs_sensors_fops,
1156 &amdgpu_debugfs_wave_fops,
1157 &amdgpu_debugfs_gpr_fops,
1158 &amdgpu_debugfs_gfxoff_fops,
1159};
1160
1161static const char *debugfs_regs_names[] = {
1162 "amdgpu_regs",
1163 "amdgpu_regs_didt",
1164 "amdgpu_regs_pcie",
1165 "amdgpu_regs_smc",
1166 "amdgpu_gca_config",
1167 "amdgpu_sensors",
1168 "amdgpu_wave",
1169 "amdgpu_gpr",
1170 "amdgpu_gfxoff",
1171};
1172
1173/**
1174 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1175 * register access.
1176 *
1177 * @adev: The device to attach the debugfs entries to
1178 */
1179int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1180{
1181 struct drm_minor *minor = adev_to_drm(adev)->primary;
1182 struct dentry *ent, *root = minor->debugfs_root;
1183 unsigned int i;
1184
1185 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1186 ent = debugfs_create_file(debugfs_regs_names[i],
1187 S_IFREG | S_IRUGO, root,
1188 adev, debugfs_regs[i]);
1189 if (!i && !IS_ERR_OR_NULL(ent))
1190 i_size_write(ent->d_inode, adev->rmmio_size);
1191 }
1192
1193 return 0;
1194}
1195
1196static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1197{
1198 struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1199 struct drm_device *dev = adev_to_drm(adev);
1200 int r = 0, i;
1201
1202 r = pm_runtime_get_sync(dev->dev);
1203 if (r < 0) {
1204 pm_runtime_put_autosuspend(dev->dev);
1205 return r;
1206 }
1207
1208 /* Avoid accidently unparking the sched thread during GPU reset */
1209 r = down_read_killable(&adev->reset_sem);
1210 if (r)
1211 return r;
1212
1213 /* hold on the scheduler */
1214 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1215 struct amdgpu_ring *ring = adev->rings[i];
1216
1217 if (!ring || !ring->sched.thread)
1218 continue;
1219 kthread_park(ring->sched.thread);
1220 }
1221
1222 seq_printf(m, "run ib test:\n");
1223 r = amdgpu_ib_ring_tests(adev);
1224 if (r)
1225 seq_printf(m, "ib ring tests failed (%d).\n", r);
1226 else
1227 seq_printf(m, "ib ring tests passed.\n");
1228
1229 /* go on the scheduler */
1230 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1231 struct amdgpu_ring *ring = adev->rings[i];
1232
1233 if (!ring || !ring->sched.thread)
1234 continue;
1235 kthread_unpark(ring->sched.thread);
1236 }
1237
1238 up_read(&adev->reset_sem);
1239
1240 pm_runtime_mark_last_busy(dev->dev);
1241 pm_runtime_put_autosuspend(dev->dev);
1242
1243 return 0;
1244}
1245
1246static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1247{
1248 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1249 struct drm_device *dev = adev_to_drm(adev);
1250 int r;
1251
1252 r = pm_runtime_get_sync(dev->dev);
1253 if (r < 0) {
1254 pm_runtime_put_autosuspend(dev->dev);
1255 return r;
1256 }
1257
1258 *val = amdgpu_bo_evict_vram(adev);
1259
1260 pm_runtime_mark_last_busy(dev->dev);
1261 pm_runtime_put_autosuspend(dev->dev);
1262
1263 return 0;
1264}
1265
1266
1267static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1268{
1269 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1270 struct drm_device *dev = adev_to_drm(adev);
1271 struct ttm_resource_manager *man;
1272 int r;
1273
1274 r = pm_runtime_get_sync(dev->dev);
1275 if (r < 0) {
1276 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1277 return r;
1278 }
1279
1280 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
1281 *val = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
1282
1283 pm_runtime_mark_last_busy(dev->dev);
1284 pm_runtime_put_autosuspend(dev->dev);
1285
1286 return 0;
1287}
1288
1289
1290static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1291{
1292 struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1293 struct drm_device *dev = adev_to_drm(adev);
1294 struct drm_file *file;
1295 int r;
1296
1297 r = mutex_lock_interruptible(&dev->filelist_mutex);
1298 if (r)
1299 return r;
1300
1301 list_for_each_entry(file, &dev->filelist, lhead) {
1302 struct amdgpu_fpriv *fpriv = file->driver_priv;
1303 struct amdgpu_vm *vm = &fpriv->vm;
1304
1305 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1306 vm->task_info.pid, vm->task_info.process_name);
1307 r = amdgpu_bo_reserve(vm->root.bo, true);
1308 if (r)
1309 break;
1310 amdgpu_debugfs_vm_bo_info(vm, m);
1311 amdgpu_bo_unreserve(vm->root.bo);
1312 }
1313
1314 mutex_unlock(&dev->filelist_mutex);
1315
1316 return r;
1317}
1318
1319DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1320DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1321DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1322 NULL, "%lld\n");
1323DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1324 NULL, "%lld\n");
1325
1326static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1327 struct dma_fence **fences)
1328{
1329 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1330 uint32_t sync_seq, last_seq;
1331
1332 last_seq = atomic_read(&ring->fence_drv.last_seq);
1333 sync_seq = ring->fence_drv.sync_seq;
1334
1335 last_seq &= drv->num_fences_mask;
1336 sync_seq &= drv->num_fences_mask;
1337
1338 do {
1339 struct dma_fence *fence, **ptr;
1340
1341 ++last_seq;
1342 last_seq &= drv->num_fences_mask;
1343 ptr = &drv->fences[last_seq];
1344
1345 fence = rcu_dereference_protected(*ptr, 1);
1346 RCU_INIT_POINTER(*ptr, NULL);
1347
1348 if (!fence)
1349 continue;
1350
1351 fences[last_seq] = fence;
1352
1353 } while (last_seq != sync_seq);
1354}
1355
1356static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1357 int length)
1358{
1359 int i;
1360 struct dma_fence *fence;
1361
1362 for (i = 0; i < length; i++) {
1363 fence = fences[i];
1364 if (!fence)
1365 continue;
1366 dma_fence_signal(fence);
1367 dma_fence_put(fence);
1368 }
1369}
1370
1371static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1372{
1373 struct drm_sched_job *s_job;
1374 struct dma_fence *fence;
1375
1376 spin_lock(&sched->job_list_lock);
1377 list_for_each_entry(s_job, &sched->pending_list, list) {
1378 fence = sched->ops->run_job(s_job);
1379 dma_fence_put(fence);
1380 }
1381 spin_unlock(&sched->job_list_lock);
1382}
1383
1384static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1385{
1386 struct amdgpu_job *job;
1387 struct drm_sched_job *s_job, *tmp;
1388 uint32_t preempt_seq;
1389 struct dma_fence *fence, **ptr;
1390 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1391 struct drm_gpu_scheduler *sched = &ring->sched;
1392 bool preempted = true;
1393
1394 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1395 return;
1396
1397 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1398 if (preempt_seq <= atomic_read(&drv->last_seq)) {
1399 preempted = false;
1400 goto no_preempt;
1401 }
1402
1403 preempt_seq &= drv->num_fences_mask;
1404 ptr = &drv->fences[preempt_seq];
1405 fence = rcu_dereference_protected(*ptr, 1);
1406
1407no_preempt:
1408 spin_lock(&sched->job_list_lock);
1409 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1410 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1411 /* remove job from ring_mirror_list */
1412 list_del_init(&s_job->list);
1413 sched->ops->free_job(s_job);
1414 continue;
1415 }
1416 job = to_amdgpu_job(s_job);
1417 if (preempted && job->fence == fence)
1418 /* mark the job as preempted */
1419 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1420 }
1421 spin_unlock(&sched->job_list_lock);
1422}
1423
1424static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1425{
1426 int r, resched, length;
1427 struct amdgpu_ring *ring;
1428 struct dma_fence **fences = NULL;
1429 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1430
1431 if (val >= AMDGPU_MAX_RINGS)
1432 return -EINVAL;
1433
1434 ring = adev->rings[val];
1435
1436 if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1437 return -EINVAL;
1438
1439 /* the last preemption failed */
1440 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1441 return -EBUSY;
1442
1443 length = ring->fence_drv.num_fences_mask + 1;
1444 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1445 if (!fences)
1446 return -ENOMEM;
1447
1448 /* Avoid accidently unparking the sched thread during GPU reset */
1449 r = down_read_killable(&adev->reset_sem);
1450 if (r)
1451 goto pro_end;
1452
1453 /* stop the scheduler */
1454 kthread_park(ring->sched.thread);
1455
1456 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1457
1458 /* preempt the IB */
1459 r = amdgpu_ring_preempt_ib(ring);
1460 if (r) {
1461 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1462 goto failure;
1463 }
1464
1465 amdgpu_fence_process(ring);
1466
1467 if (atomic_read(&ring->fence_drv.last_seq) !=
1468 ring->fence_drv.sync_seq) {
1469 DRM_INFO("ring %d was preempted\n", ring->idx);
1470
1471 amdgpu_ib_preempt_mark_partial_job(ring);
1472
1473 /* swap out the old fences */
1474 amdgpu_ib_preempt_fences_swap(ring, fences);
1475
1476 amdgpu_fence_driver_force_completion(ring);
1477
1478 /* resubmit unfinished jobs */
1479 amdgpu_ib_preempt_job_recovery(&ring->sched);
1480
1481 /* wait for jobs finished */
1482 amdgpu_fence_wait_empty(ring);
1483
1484 /* signal the old fences */
1485 amdgpu_ib_preempt_signal_fences(fences, length);
1486 }
1487
1488failure:
1489 /* restart the scheduler */
1490 kthread_unpark(ring->sched.thread);
1491
1492 up_read(&adev->reset_sem);
1493
1494 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1495
1496pro_end:
1497 kfree(fences);
1498
1499 return r;
1500}
1501
1502static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1503{
1504 int ret = 0;
1505 uint32_t max_freq, min_freq;
1506 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1507
1508 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1509 return -EINVAL;
1510
1511 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1512 if (ret < 0) {
1513 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1514 return ret;
1515 }
1516
1517 if (is_support_sw_smu(adev)) {
1518 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
1519 if (ret || val > max_freq || val < min_freq)
1520 return -EINVAL;
1521 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
1522 } else {
1523 return 0;
1524 }
1525
1526 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1527 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1528
1529 if (ret)
1530 return -EINVAL;
1531
1532 return 0;
1533}
1534
1535DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1536 amdgpu_debugfs_ib_preempt, "%llu\n");
1537
1538DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
1539 amdgpu_debugfs_sclk_set, "%llu\n");
1540
1541int amdgpu_debugfs_init(struct amdgpu_device *adev)
1542{
1543 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
1544 struct dentry *ent;
1545 int r, i;
1546
1547 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
1548 &fops_ib_preempt);
1549 if (IS_ERR(ent)) {
1550 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1551 return PTR_ERR(ent);
1552 }
1553
1554 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
1555 &fops_sclk_set);
1556 if (IS_ERR(ent)) {
1557 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1558 return PTR_ERR(ent);
1559 }
1560
1561 /* Register debugfs entries for amdgpu_ttm */
1562 amdgpu_ttm_debugfs_init(adev);
1563 amdgpu_debugfs_pm_init(adev);
1564 amdgpu_debugfs_sa_init(adev);
1565 amdgpu_debugfs_fence_init(adev);
1566 amdgpu_debugfs_gem_init(adev);
1567
1568 r = amdgpu_debugfs_regs_init(adev);
1569 if (r)
1570 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1571
1572 amdgpu_debugfs_firmware_init(adev);
1573
1574#if defined(CONFIG_DRM_AMD_DC)
1575 if (amdgpu_device_has_dc_support(adev))
1576 dtn_debugfs_init(adev);
1577#endif
1578
1579 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1580 struct amdgpu_ring *ring = adev->rings[i];
1581
1582 if (!ring)
1583 continue;
1584
1585 if (amdgpu_debugfs_ring_init(adev, ring)) {
1586 DRM_ERROR("Failed to register debugfs file for rings !\n");
1587 }
1588 }
1589
1590 amdgpu_ras_debugfs_create_all(adev);
1591 amdgpu_debugfs_autodump_init(adev);
1592 amdgpu_rap_debugfs_init(adev);
1593 amdgpu_securedisplay_debugfs_init(adev);
1594 amdgpu_fw_attestation_debugfs_init(adev);
1595
1596 debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
1597 &amdgpu_evict_vram_fops);
1598 debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
1599 &amdgpu_evict_gtt_fops);
1600 debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
1601 &amdgpu_debugfs_test_ib_fops);
1602 debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
1603 &amdgpu_debugfs_vm_info_fops);
1604
1605 adev->debugfs_vbios_blob.data = adev->bios;
1606 adev->debugfs_vbios_blob.size = adev->bios_size;
1607 debugfs_create_blob("amdgpu_vbios", 0444, root,
1608 &adev->debugfs_vbios_blob);
1609
1610 return 0;
1611}
1612
1613#else
1614int amdgpu_debugfs_init(struct amdgpu_device *adev)
1615{
1616 return 0;
1617}
1618int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1619{
1620 return 0;
1621}
1622#endif