Loading...
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26#include <linux/kthread.h>
27#include <linux/pci.h>
28#include <linux/uaccess.h>
29#include <linux/pm_runtime.h>
30
31#include "amdgpu.h"
32#include "amdgpu_pm.h"
33#include "amdgpu_dm_debugfs.h"
34#include "amdgpu_ras.h"
35#include "amdgpu_rap.h"
36#include "amdgpu_securedisplay.h"
37#include "amdgpu_fw_attestation.h"
38#include "amdgpu_umr.h"
39
40#include "amdgpu_reset.h"
41#include "amdgpu_psp_ta.h"
42
43#if defined(CONFIG_DEBUG_FS)
44
45/**
46 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
47 *
48 * @read: True if reading
49 * @f: open file handle
50 * @buf: User buffer to write/read to
51 * @size: Number of bytes to write/read
52 * @pos: Offset to seek to
53 *
54 * This debugfs entry has special meaning on the offset being sought.
55 * Various bits have different meanings:
56 *
57 * Bit 62: Indicates a GRBM bank switch is needed
58 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
59 * zero)
60 * Bits 24..33: The SE or ME selector if needed
61 * Bits 34..43: The SH (or SA) or PIPE selector if needed
62 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
63 *
64 * Bit 23: Indicates that the PM power gating lock should be held
65 * This is necessary to read registers that might be
66 * unreliable during a power gating transistion.
67 *
68 * The lower bits are the BYTE offset of the register to read. This
69 * allows reading multiple registers in a single call and having
70 * the returned size reflect that.
71 */
72static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
73 char __user *buf, size_t size, loff_t *pos)
74{
75 struct amdgpu_device *adev = file_inode(f)->i_private;
76 ssize_t result = 0;
77 int r;
78 bool pm_pg_lock, use_bank, use_ring;
79 unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
80
81 pm_pg_lock = use_bank = use_ring = false;
82 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
83
84 if (size & 0x3 || *pos & 0x3 ||
85 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
86 return -EINVAL;
87
88 /* are we reading registers for which a PG lock is necessary? */
89 pm_pg_lock = (*pos >> 23) & 1;
90
91 if (*pos & (1ULL << 62)) {
92 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
93 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
94 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
95
96 if (se_bank == 0x3FF)
97 se_bank = 0xFFFFFFFF;
98 if (sh_bank == 0x3FF)
99 sh_bank = 0xFFFFFFFF;
100 if (instance_bank == 0x3FF)
101 instance_bank = 0xFFFFFFFF;
102 use_bank = true;
103 } else if (*pos & (1ULL << 61)) {
104
105 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
106 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
107 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
108 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
109
110 use_ring = true;
111 } else {
112 use_bank = use_ring = false;
113 }
114
115 *pos &= (1UL << 22) - 1;
116
117 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
118 if (r < 0) {
119 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
120 return r;
121 }
122
123 r = amdgpu_virt_enable_access_debugfs(adev);
124 if (r < 0) {
125 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
126 return r;
127 }
128
129 if (use_bank) {
130 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
131 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
132 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
133 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
134 amdgpu_virt_disable_access_debugfs(adev);
135 return -EINVAL;
136 }
137 mutex_lock(&adev->grbm_idx_mutex);
138 amdgpu_gfx_select_se_sh(adev, se_bank,
139 sh_bank, instance_bank, 0);
140 } else if (use_ring) {
141 mutex_lock(&adev->srbm_mutex);
142 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
143 }
144
145 if (pm_pg_lock)
146 mutex_lock(&adev->pm.mutex);
147
148 while (size) {
149 uint32_t value;
150
151 if (read) {
152 value = RREG32(*pos >> 2);
153 r = put_user(value, (uint32_t *)buf);
154 } else {
155 r = get_user(value, (uint32_t *)buf);
156 if (!r)
157 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
158 }
159 if (r) {
160 result = r;
161 goto end;
162 }
163
164 result += 4;
165 buf += 4;
166 *pos += 4;
167 size -= 4;
168 }
169
170end:
171 if (use_bank) {
172 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
173 mutex_unlock(&adev->grbm_idx_mutex);
174 } else if (use_ring) {
175 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
176 mutex_unlock(&adev->srbm_mutex);
177 }
178
179 if (pm_pg_lock)
180 mutex_unlock(&adev->pm.mutex);
181
182 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
183 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
184
185 amdgpu_virt_disable_access_debugfs(adev);
186 return result;
187}
188
189/*
190 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
191 */
192static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
193 size_t size, loff_t *pos)
194{
195 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
196}
197
198/*
199 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
200 */
201static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
202 size_t size, loff_t *pos)
203{
204 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
205}
206
207static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
208{
209 struct amdgpu_debugfs_regs2_data *rd;
210
211 rd = kzalloc(sizeof(*rd), GFP_KERNEL);
212 if (!rd)
213 return -ENOMEM;
214 rd->adev = file_inode(file)->i_private;
215 file->private_data = rd;
216 mutex_init(&rd->lock);
217
218 return 0;
219}
220
221static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
222{
223 struct amdgpu_debugfs_regs2_data *rd = file->private_data;
224
225 mutex_destroy(&rd->lock);
226 kfree(file->private_data);
227 return 0;
228}
229
230static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
231{
232 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
233 struct amdgpu_device *adev = rd->adev;
234 ssize_t result = 0;
235 int r;
236 uint32_t value;
237
238 if (size & 0x3 || offset & 0x3)
239 return -EINVAL;
240
241 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
242 if (r < 0) {
243 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
244 return r;
245 }
246
247 r = amdgpu_virt_enable_access_debugfs(adev);
248 if (r < 0) {
249 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
250 return r;
251 }
252
253 mutex_lock(&rd->lock);
254
255 if (rd->id.use_grbm) {
256 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
257 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
258 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
259 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
260 amdgpu_virt_disable_access_debugfs(adev);
261 mutex_unlock(&rd->lock);
262 return -EINVAL;
263 }
264 mutex_lock(&adev->grbm_idx_mutex);
265 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
266 rd->id.grbm.sh,
267 rd->id.grbm.instance, rd->id.xcc_id);
268 }
269
270 if (rd->id.use_srbm) {
271 mutex_lock(&adev->srbm_mutex);
272 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
273 rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
274 }
275
276 if (rd->id.pg_lock)
277 mutex_lock(&adev->pm.mutex);
278
279 while (size) {
280 if (!write_en) {
281 value = RREG32(offset >> 2);
282 r = put_user(value, (uint32_t *)buf);
283 } else {
284 r = get_user(value, (uint32_t *)buf);
285 if (!r)
286 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id);
287 }
288 if (r) {
289 result = r;
290 goto end;
291 }
292 offset += 4;
293 size -= 4;
294 result += 4;
295 buf += 4;
296 }
297end:
298 if (rd->id.use_grbm) {
299 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
300 mutex_unlock(&adev->grbm_idx_mutex);
301 }
302
303 if (rd->id.use_srbm) {
304 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
305 mutex_unlock(&adev->srbm_mutex);
306 }
307
308 if (rd->id.pg_lock)
309 mutex_unlock(&adev->pm.mutex);
310
311 mutex_unlock(&rd->lock);
312
313 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
314 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
315
316 amdgpu_virt_disable_access_debugfs(adev);
317 return result;
318}
319
320static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
321{
322 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
323 struct amdgpu_debugfs_regs2_iocdata v1_data;
324 int r;
325
326 mutex_lock(&rd->lock);
327
328 switch (cmd) {
329 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
330 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
331 sizeof(rd->id));
332 if (r)
333 r = -EINVAL;
334 goto done;
335 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
336 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
337 sizeof(v1_data));
338 if (r) {
339 r = -EINVAL;
340 goto done;
341 }
342 goto v1_copy;
343 default:
344 r = -EINVAL;
345 goto done;
346 }
347
348v1_copy:
349 rd->id.use_srbm = v1_data.use_srbm;
350 rd->id.use_grbm = v1_data.use_grbm;
351 rd->id.pg_lock = v1_data.pg_lock;
352 rd->id.grbm.se = v1_data.grbm.se;
353 rd->id.grbm.sh = v1_data.grbm.sh;
354 rd->id.grbm.instance = v1_data.grbm.instance;
355 rd->id.srbm.me = v1_data.srbm.me;
356 rd->id.srbm.pipe = v1_data.srbm.pipe;
357 rd->id.srbm.queue = v1_data.srbm.queue;
358 rd->id.xcc_id = 0;
359done:
360 mutex_unlock(&rd->lock);
361 return r;
362}
363
364static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
365{
366 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
367}
368
369static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
370{
371 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
372}
373
374static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
375{
376 struct amdgpu_debugfs_gprwave_data *rd;
377
378 rd = kzalloc(sizeof(*rd), GFP_KERNEL);
379 if (!rd)
380 return -ENOMEM;
381 rd->adev = file_inode(file)->i_private;
382 file->private_data = rd;
383 mutex_init(&rd->lock);
384
385 return 0;
386}
387
388static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
389{
390 struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
391
392 mutex_destroy(&rd->lock);
393 kfree(file->private_data);
394 return 0;
395}
396
397static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
398{
399 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
400 struct amdgpu_device *adev = rd->adev;
401 ssize_t result = 0;
402 int r;
403 uint32_t *data, x;
404
405 if (size & 0x3 || *pos & 0x3)
406 return -EINVAL;
407
408 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
409 if (r < 0) {
410 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
411 return r;
412 }
413
414 r = amdgpu_virt_enable_access_debugfs(adev);
415 if (r < 0) {
416 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
417 return r;
418 }
419
420 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
421 if (!data) {
422 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
423 amdgpu_virt_disable_access_debugfs(adev);
424 return -ENOMEM;
425 }
426
427 /* switch to the specific se/sh/cu */
428 mutex_lock(&adev->grbm_idx_mutex);
429 amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
430
431 if (!rd->id.gpr_or_wave) {
432 x = 0;
433 if (adev->gfx.funcs->read_wave_data)
434 adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
435 } else {
436 x = size >> 2;
437 if (rd->id.gpr.vpgr_or_sgpr) {
438 if (adev->gfx.funcs->read_wave_vgprs)
439 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
440 } else {
441 if (adev->gfx.funcs->read_wave_sgprs)
442 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
443 }
444 }
445
446 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
447 mutex_unlock(&adev->grbm_idx_mutex);
448
449 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
450 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
451
452 if (!x) {
453 result = -EINVAL;
454 goto done;
455 }
456
457 while (size && (*pos < x * 4)) {
458 uint32_t value;
459
460 value = data[*pos >> 2];
461 r = put_user(value, (uint32_t *)buf);
462 if (r) {
463 result = r;
464 goto done;
465 }
466
467 result += 4;
468 buf += 4;
469 *pos += 4;
470 size -= 4;
471 }
472
473done:
474 amdgpu_virt_disable_access_debugfs(adev);
475 kfree(data);
476 return result;
477}
478
479static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
480{
481 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
482 int r = 0;
483
484 mutex_lock(&rd->lock);
485
486 switch (cmd) {
487 case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
488 if (copy_from_user(&rd->id,
489 (struct amdgpu_debugfs_gprwave_iocdata *)data,
490 sizeof(rd->id)))
491 r = -EFAULT;
492 goto done;
493 default:
494 r = -EINVAL;
495 goto done;
496 }
497
498done:
499 mutex_unlock(&rd->lock);
500 return r;
501}
502
503
504
505
506/**
507 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
508 *
509 * @f: open file handle
510 * @buf: User buffer to store read data in
511 * @size: Number of bytes to read
512 * @pos: Offset to seek to
513 *
514 * The lower bits are the BYTE offset of the register to read. This
515 * allows reading multiple registers in a single call and having
516 * the returned size reflect that.
517 */
518static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
519 size_t size, loff_t *pos)
520{
521 struct amdgpu_device *adev = file_inode(f)->i_private;
522 ssize_t result = 0;
523 int r;
524
525 if (size & 0x3 || *pos & 0x3)
526 return -EINVAL;
527
528 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
529 if (r < 0) {
530 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
531 return r;
532 }
533
534 r = amdgpu_virt_enable_access_debugfs(adev);
535 if (r < 0) {
536 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
537 return r;
538 }
539
540 while (size) {
541 uint32_t value;
542
543 if (upper_32_bits(*pos))
544 value = RREG32_PCIE_EXT(*pos);
545 else
546 value = RREG32_PCIE(*pos);
547
548 r = put_user(value, (uint32_t *)buf);
549 if (r)
550 goto out;
551
552 result += 4;
553 buf += 4;
554 *pos += 4;
555 size -= 4;
556 }
557
558 r = result;
559out:
560 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
561 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
562 amdgpu_virt_disable_access_debugfs(adev);
563 return r;
564}
565
566/**
567 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
568 *
569 * @f: open file handle
570 * @buf: User buffer to write data from
571 * @size: Number of bytes to write
572 * @pos: Offset to seek to
573 *
574 * The lower bits are the BYTE offset of the register to write. This
575 * allows writing multiple registers in a single call and having
576 * the returned size reflect that.
577 */
578static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
579 size_t size, loff_t *pos)
580{
581 struct amdgpu_device *adev = file_inode(f)->i_private;
582 ssize_t result = 0;
583 int r;
584
585 if (size & 0x3 || *pos & 0x3)
586 return -EINVAL;
587
588 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
589 if (r < 0) {
590 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
591 return r;
592 }
593
594 r = amdgpu_virt_enable_access_debugfs(adev);
595 if (r < 0) {
596 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
597 return r;
598 }
599
600 while (size) {
601 uint32_t value;
602
603 r = get_user(value, (uint32_t *)buf);
604 if (r)
605 goto out;
606
607 if (upper_32_bits(*pos))
608 WREG32_PCIE_EXT(*pos, value);
609 else
610 WREG32_PCIE(*pos, value);
611
612 result += 4;
613 buf += 4;
614 *pos += 4;
615 size -= 4;
616 }
617
618 r = result;
619out:
620 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
621 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
622 amdgpu_virt_disable_access_debugfs(adev);
623 return r;
624}
625
626/**
627 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
628 *
629 * @f: open file handle
630 * @buf: User buffer to store read data in
631 * @size: Number of bytes to read
632 * @pos: Offset to seek to
633 *
634 * The lower bits are the BYTE offset of the register to read. This
635 * allows reading multiple registers in a single call and having
636 * the returned size reflect that.
637 */
638static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
639 size_t size, loff_t *pos)
640{
641 struct amdgpu_device *adev = file_inode(f)->i_private;
642 ssize_t result = 0;
643 int r;
644
645 if (size & 0x3 || *pos & 0x3)
646 return -EINVAL;
647
648 if (!adev->didt_rreg)
649 return -EOPNOTSUPP;
650
651 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
652 if (r < 0) {
653 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
654 return r;
655 }
656
657 r = amdgpu_virt_enable_access_debugfs(adev);
658 if (r < 0) {
659 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
660 return r;
661 }
662
663 while (size) {
664 uint32_t value;
665
666 value = RREG32_DIDT(*pos >> 2);
667 r = put_user(value, (uint32_t *)buf);
668 if (r)
669 goto out;
670
671 result += 4;
672 buf += 4;
673 *pos += 4;
674 size -= 4;
675 }
676
677 r = result;
678out:
679 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
680 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
681 amdgpu_virt_disable_access_debugfs(adev);
682 return r;
683}
684
685/**
686 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
687 *
688 * @f: open file handle
689 * @buf: User buffer to write data from
690 * @size: Number of bytes to write
691 * @pos: Offset to seek to
692 *
693 * The lower bits are the BYTE offset of the register to write. This
694 * allows writing multiple registers in a single call and having
695 * the returned size reflect that.
696 */
697static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
698 size_t size, loff_t *pos)
699{
700 struct amdgpu_device *adev = file_inode(f)->i_private;
701 ssize_t result = 0;
702 int r;
703
704 if (size & 0x3 || *pos & 0x3)
705 return -EINVAL;
706
707 if (!adev->didt_wreg)
708 return -EOPNOTSUPP;
709
710 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
711 if (r < 0) {
712 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
713 return r;
714 }
715
716 r = amdgpu_virt_enable_access_debugfs(adev);
717 if (r < 0) {
718 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
719 return r;
720 }
721
722 while (size) {
723 uint32_t value;
724
725 r = get_user(value, (uint32_t *)buf);
726 if (r)
727 goto out;
728
729 WREG32_DIDT(*pos >> 2, value);
730
731 result += 4;
732 buf += 4;
733 *pos += 4;
734 size -= 4;
735 }
736
737 r = result;
738out:
739 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
740 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
741 amdgpu_virt_disable_access_debugfs(adev);
742 return r;
743}
744
745/**
746 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
747 *
748 * @f: open file handle
749 * @buf: User buffer to store read data in
750 * @size: Number of bytes to read
751 * @pos: Offset to seek to
752 *
753 * The lower bits are the BYTE offset of the register to read. This
754 * allows reading multiple registers in a single call and having
755 * the returned size reflect that.
756 */
757static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
758 size_t size, loff_t *pos)
759{
760 struct amdgpu_device *adev = file_inode(f)->i_private;
761 ssize_t result = 0;
762 int r;
763
764 if (!adev->smc_rreg)
765 return -EOPNOTSUPP;
766
767 if (size & 0x3 || *pos & 0x3)
768 return -EINVAL;
769
770 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
771 if (r < 0) {
772 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
773 return r;
774 }
775
776 r = amdgpu_virt_enable_access_debugfs(adev);
777 if (r < 0) {
778 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
779 return r;
780 }
781
782 while (size) {
783 uint32_t value;
784
785 value = RREG32_SMC(*pos);
786 r = put_user(value, (uint32_t *)buf);
787 if (r)
788 goto out;
789
790 result += 4;
791 buf += 4;
792 *pos += 4;
793 size -= 4;
794 }
795
796 r = result;
797out:
798 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
799 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
800 amdgpu_virt_disable_access_debugfs(adev);
801 return r;
802}
803
804/**
805 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
806 *
807 * @f: open file handle
808 * @buf: User buffer to write data from
809 * @size: Number of bytes to write
810 * @pos: Offset to seek to
811 *
812 * The lower bits are the BYTE offset of the register to write. This
813 * allows writing multiple registers in a single call and having
814 * the returned size reflect that.
815 */
816static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
817 size_t size, loff_t *pos)
818{
819 struct amdgpu_device *adev = file_inode(f)->i_private;
820 ssize_t result = 0;
821 int r;
822
823 if (!adev->smc_wreg)
824 return -EOPNOTSUPP;
825
826 if (size & 0x3 || *pos & 0x3)
827 return -EINVAL;
828
829 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
830 if (r < 0) {
831 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
832 return r;
833 }
834
835 r = amdgpu_virt_enable_access_debugfs(adev);
836 if (r < 0) {
837 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
838 return r;
839 }
840
841 while (size) {
842 uint32_t value;
843
844 r = get_user(value, (uint32_t *)buf);
845 if (r)
846 goto out;
847
848 WREG32_SMC(*pos, value);
849
850 result += 4;
851 buf += 4;
852 *pos += 4;
853 size -= 4;
854 }
855
856 r = result;
857out:
858 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
859 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
860 amdgpu_virt_disable_access_debugfs(adev);
861 return r;
862}
863
864/**
865 * amdgpu_debugfs_gca_config_read - Read from gfx config data
866 *
867 * @f: open file handle
868 * @buf: User buffer to store read data in
869 * @size: Number of bytes to read
870 * @pos: Offset to seek to
871 *
872 * This file is used to access configuration data in a somewhat
873 * stable fashion. The format is a series of DWORDs with the first
874 * indicating which revision it is. New content is appended to the
875 * end so that older software can still read the data.
876 */
877
878static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
879 size_t size, loff_t *pos)
880{
881 struct amdgpu_device *adev = file_inode(f)->i_private;
882 ssize_t result = 0;
883 int r;
884 uint32_t *config, no_regs = 0;
885
886 if (size & 0x3 || *pos & 0x3)
887 return -EINVAL;
888
889 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
890 if (!config)
891 return -ENOMEM;
892
893 /* version, increment each time something is added */
894 config[no_regs++] = 5;
895 config[no_regs++] = adev->gfx.config.max_shader_engines;
896 config[no_regs++] = adev->gfx.config.max_tile_pipes;
897 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
898 config[no_regs++] = adev->gfx.config.max_sh_per_se;
899 config[no_regs++] = adev->gfx.config.max_backends_per_se;
900 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
901 config[no_regs++] = adev->gfx.config.max_gprs;
902 config[no_regs++] = adev->gfx.config.max_gs_threads;
903 config[no_regs++] = adev->gfx.config.max_hw_contexts;
904 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
905 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
906 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
907 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
908 config[no_regs++] = adev->gfx.config.num_tile_pipes;
909 config[no_regs++] = adev->gfx.config.backend_enable_mask;
910 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
911 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
912 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
913 config[no_regs++] = adev->gfx.config.num_gpus;
914 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
915 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
916 config[no_regs++] = adev->gfx.config.gb_addr_config;
917 config[no_regs++] = adev->gfx.config.num_rbs;
918
919 /* rev==1 */
920 config[no_regs++] = adev->rev_id;
921 config[no_regs++] = lower_32_bits(adev->pg_flags);
922 config[no_regs++] = lower_32_bits(adev->cg_flags);
923
924 /* rev==2 */
925 config[no_regs++] = adev->family;
926 config[no_regs++] = adev->external_rev_id;
927
928 /* rev==3 */
929 config[no_regs++] = adev->pdev->device;
930 config[no_regs++] = adev->pdev->revision;
931 config[no_regs++] = adev->pdev->subsystem_device;
932 config[no_regs++] = adev->pdev->subsystem_vendor;
933
934 /* rev==4 APU flag */
935 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
936
937 /* rev==5 PG/CG flag upper 32bit */
938 config[no_regs++] = upper_32_bits(adev->pg_flags);
939 config[no_regs++] = upper_32_bits(adev->cg_flags);
940
941 while (size && (*pos < no_regs * 4)) {
942 uint32_t value;
943
944 value = config[*pos >> 2];
945 r = put_user(value, (uint32_t *)buf);
946 if (r) {
947 kfree(config);
948 return r;
949 }
950
951 result += 4;
952 buf += 4;
953 *pos += 4;
954 size -= 4;
955 }
956
957 kfree(config);
958 return result;
959}
960
961/**
962 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
963 *
964 * @f: open file handle
965 * @buf: User buffer to store read data in
966 * @size: Number of bytes to read
967 * @pos: Offset to seek to
968 *
969 * The offset is treated as the BYTE address of one of the sensors
970 * enumerated in amd/include/kgd_pp_interface.h under the
971 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
972 * you would use the offset 3 * 4 = 12.
973 */
974static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
975 size_t size, loff_t *pos)
976{
977 struct amdgpu_device *adev = file_inode(f)->i_private;
978 int idx, x, outsize, r, valuesize;
979 uint32_t values[16];
980
981 if (size & 3 || *pos & 0x3)
982 return -EINVAL;
983
984 if (!adev->pm.dpm_enabled)
985 return -EINVAL;
986
987 /* convert offset to sensor number */
988 idx = *pos >> 2;
989
990 valuesize = sizeof(values);
991
992 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
993 if (r < 0) {
994 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
995 return r;
996 }
997
998 r = amdgpu_virt_enable_access_debugfs(adev);
999 if (r < 0) {
1000 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1001 return r;
1002 }
1003
1004 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
1005
1006 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1007 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1008
1009 if (r) {
1010 amdgpu_virt_disable_access_debugfs(adev);
1011 return r;
1012 }
1013
1014 if (size > valuesize) {
1015 amdgpu_virt_disable_access_debugfs(adev);
1016 return -EINVAL;
1017 }
1018
1019 outsize = 0;
1020 x = 0;
1021 if (!r) {
1022 while (size) {
1023 r = put_user(values[x++], (int32_t *)buf);
1024 buf += 4;
1025 size -= 4;
1026 outsize += 4;
1027 }
1028 }
1029
1030 amdgpu_virt_disable_access_debugfs(adev);
1031 return !r ? outsize : r;
1032}
1033
1034/** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1035 *
1036 * @f: open file handle
1037 * @buf: User buffer to store read data in
1038 * @size: Number of bytes to read
1039 * @pos: Offset to seek to
1040 *
1041 * The offset being sought changes which wave that the status data
1042 * will be returned for. The bits are used as follows:
1043 *
1044 * Bits 0..6: Byte offset into data
1045 * Bits 7..14: SE selector
1046 * Bits 15..22: SH/SA selector
1047 * Bits 23..30: CU/{WGP+SIMD} selector
1048 * Bits 31..36: WAVE ID selector
1049 * Bits 37..44: SIMD ID selector
1050 *
1051 * The returned data begins with one DWORD of version information
1052 * Followed by WAVE STATUS registers relevant to the GFX IP version
1053 * being used. See gfx_v8_0_read_wave_data() for an example output.
1054 */
1055static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1056 size_t size, loff_t *pos)
1057{
1058 struct amdgpu_device *adev = f->f_inode->i_private;
1059 int r, x;
1060 ssize_t result = 0;
1061 uint32_t offset, se, sh, cu, wave, simd, data[32];
1062
1063 if (size & 3 || *pos & 3)
1064 return -EINVAL;
1065
1066 /* decode offset */
1067 offset = (*pos & GENMASK_ULL(6, 0));
1068 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1069 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1070 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1071 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1072 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1073
1074 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1075 if (r < 0) {
1076 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1077 return r;
1078 }
1079
1080 r = amdgpu_virt_enable_access_debugfs(adev);
1081 if (r < 0) {
1082 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1083 return r;
1084 }
1085
1086 /* switch to the specific se/sh/cu */
1087 mutex_lock(&adev->grbm_idx_mutex);
1088 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1089
1090 x = 0;
1091 if (adev->gfx.funcs->read_wave_data)
1092 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1093
1094 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1095 mutex_unlock(&adev->grbm_idx_mutex);
1096
1097 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1098 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1099
1100 if (!x) {
1101 amdgpu_virt_disable_access_debugfs(adev);
1102 return -EINVAL;
1103 }
1104
1105 while (size && (offset < x * 4)) {
1106 uint32_t value;
1107
1108 value = data[offset >> 2];
1109 r = put_user(value, (uint32_t *)buf);
1110 if (r) {
1111 amdgpu_virt_disable_access_debugfs(adev);
1112 return r;
1113 }
1114
1115 result += 4;
1116 buf += 4;
1117 offset += 4;
1118 size -= 4;
1119 }
1120
1121 amdgpu_virt_disable_access_debugfs(adev);
1122 return result;
1123}
1124
1125/** amdgpu_debugfs_gpr_read - Read wave gprs
1126 *
1127 * @f: open file handle
1128 * @buf: User buffer to store read data in
1129 * @size: Number of bytes to read
1130 * @pos: Offset to seek to
1131 *
1132 * The offset being sought changes which wave that the status data
1133 * will be returned for. The bits are used as follows:
1134 *
1135 * Bits 0..11: Byte offset into data
1136 * Bits 12..19: SE selector
1137 * Bits 20..27: SH/SA selector
1138 * Bits 28..35: CU/{WGP+SIMD} selector
1139 * Bits 36..43: WAVE ID selector
1140 * Bits 37..44: SIMD ID selector
1141 * Bits 52..59: Thread selector
1142 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1143 *
1144 * The return data comes from the SGPR or VGPR register bank for
1145 * the selected operational unit.
1146 */
1147static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1148 size_t size, loff_t *pos)
1149{
1150 struct amdgpu_device *adev = f->f_inode->i_private;
1151 int r;
1152 ssize_t result = 0;
1153 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1154
1155 if (size > 4096 || size & 3 || *pos & 3)
1156 return -EINVAL;
1157
1158 /* decode offset */
1159 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1160 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1161 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1162 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1163 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1164 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1165 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1166 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1167
1168 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1169 if (!data)
1170 return -ENOMEM;
1171
1172 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1173 if (r < 0)
1174 goto err;
1175
1176 r = amdgpu_virt_enable_access_debugfs(adev);
1177 if (r < 0)
1178 goto err;
1179
1180 /* switch to the specific se/sh/cu */
1181 mutex_lock(&adev->grbm_idx_mutex);
1182 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1183
1184 if (bank == 0) {
1185 if (adev->gfx.funcs->read_wave_vgprs)
1186 adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1187 } else {
1188 if (adev->gfx.funcs->read_wave_sgprs)
1189 adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1190 }
1191
1192 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1193 mutex_unlock(&adev->grbm_idx_mutex);
1194
1195 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1196 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1197
1198 while (size) {
1199 uint32_t value;
1200
1201 value = data[result >> 2];
1202 r = put_user(value, (uint32_t *)buf);
1203 if (r) {
1204 amdgpu_virt_disable_access_debugfs(adev);
1205 goto err;
1206 }
1207
1208 result += 4;
1209 buf += 4;
1210 size -= 4;
1211 }
1212
1213 kfree(data);
1214 amdgpu_virt_disable_access_debugfs(adev);
1215 return result;
1216
1217err:
1218 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1219 kfree(data);
1220 return r;
1221}
1222
1223/**
1224 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1225 *
1226 * @f: open file handle
1227 * @buf: User buffer to store read data in
1228 * @size: Number of bytes to read
1229 * @pos: Offset to seek to
1230 *
1231 * Read the last residency value logged. It doesn't auto update, one needs to
1232 * stop logging before getting the current value.
1233 */
1234static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1235 size_t size, loff_t *pos)
1236{
1237 struct amdgpu_device *adev = file_inode(f)->i_private;
1238 ssize_t result = 0;
1239 int r;
1240
1241 if (size & 0x3 || *pos & 0x3)
1242 return -EINVAL;
1243
1244 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1245 if (r < 0) {
1246 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1247 return r;
1248 }
1249
1250 while (size) {
1251 uint32_t value;
1252
1253 r = amdgpu_get_gfx_off_residency(adev, &value);
1254 if (r)
1255 goto out;
1256
1257 r = put_user(value, (uint32_t *)buf);
1258 if (r)
1259 goto out;
1260
1261 result += 4;
1262 buf += 4;
1263 *pos += 4;
1264 size -= 4;
1265 }
1266
1267 r = result;
1268out:
1269 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1270 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1271
1272 return r;
1273}
1274
1275/**
1276 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1277 *
1278 * @f: open file handle
1279 * @buf: User buffer to write data from
1280 * @size: Number of bytes to write
1281 * @pos: Offset to seek to
1282 *
1283 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1284 */
1285static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1286 size_t size, loff_t *pos)
1287{
1288 struct amdgpu_device *adev = file_inode(f)->i_private;
1289 ssize_t result = 0;
1290 int r;
1291
1292 if (size & 0x3 || *pos & 0x3)
1293 return -EINVAL;
1294
1295 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1296 if (r < 0) {
1297 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1298 return r;
1299 }
1300
1301 while (size) {
1302 u32 value;
1303
1304 r = get_user(value, (uint32_t *)buf);
1305 if (r)
1306 goto out;
1307
1308 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1309
1310 result += 4;
1311 buf += 4;
1312 *pos += 4;
1313 size -= 4;
1314 }
1315
1316 r = result;
1317out:
1318 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1319 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1320
1321 return r;
1322}
1323
1324
1325/**
1326 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1327 *
1328 * @f: open file handle
1329 * @buf: User buffer to store read data in
1330 * @size: Number of bytes to read
1331 * @pos: Offset to seek to
1332 */
1333static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1334 size_t size, loff_t *pos)
1335{
1336 struct amdgpu_device *adev = file_inode(f)->i_private;
1337 ssize_t result = 0;
1338 int r;
1339
1340 if (size & 0x3 || *pos & 0x3)
1341 return -EINVAL;
1342
1343 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1344 if (r < 0) {
1345 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1346 return r;
1347 }
1348
1349 while (size) {
1350 u64 value = 0;
1351
1352 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1353 if (r)
1354 goto out;
1355
1356 r = put_user(value, (u64 *)buf);
1357 if (r)
1358 goto out;
1359
1360 result += 4;
1361 buf += 4;
1362 *pos += 4;
1363 size -= 4;
1364 }
1365
1366 r = result;
1367out:
1368 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1369 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1370
1371 return r;
1372}
1373
1374/**
1375 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1376 *
1377 * @f: open file handle
1378 * @buf: User buffer to write data from
1379 * @size: Number of bytes to write
1380 * @pos: Offset to seek to
1381 *
1382 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1383 */
1384static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1385 size_t size, loff_t *pos)
1386{
1387 struct amdgpu_device *adev = file_inode(f)->i_private;
1388 ssize_t result = 0;
1389 int r;
1390
1391 if (size & 0x3 || *pos & 0x3)
1392 return -EINVAL;
1393
1394 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1395 if (r < 0) {
1396 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1397 return r;
1398 }
1399
1400 while (size) {
1401 uint32_t value;
1402
1403 r = get_user(value, (uint32_t *)buf);
1404 if (r)
1405 goto out;
1406
1407 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1408
1409 result += 4;
1410 buf += 4;
1411 *pos += 4;
1412 size -= 4;
1413 }
1414
1415 r = result;
1416out:
1417 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1418 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1419
1420 return r;
1421}
1422
1423
1424/**
1425 * amdgpu_debugfs_gfxoff_read - read gfxoff status
1426 *
1427 * @f: open file handle
1428 * @buf: User buffer to store read data in
1429 * @size: Number of bytes to read
1430 * @pos: Offset to seek to
1431 */
1432static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1433 size_t size, loff_t *pos)
1434{
1435 struct amdgpu_device *adev = file_inode(f)->i_private;
1436 ssize_t result = 0;
1437 int r;
1438
1439 if (size & 0x3 || *pos & 0x3)
1440 return -EINVAL;
1441
1442 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1443 if (r < 0) {
1444 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1445 return r;
1446 }
1447
1448 while (size) {
1449 u32 value = adev->gfx.gfx_off_state;
1450
1451 r = put_user(value, (u32 *)buf);
1452 if (r)
1453 goto out;
1454
1455 result += 4;
1456 buf += 4;
1457 *pos += 4;
1458 size -= 4;
1459 }
1460
1461 r = result;
1462out:
1463 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1464 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1465
1466 return r;
1467}
1468
1469static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1470 size_t size, loff_t *pos)
1471{
1472 struct amdgpu_device *adev = file_inode(f)->i_private;
1473 ssize_t result = 0;
1474 int r;
1475
1476 if (size & 0x3 || *pos & 0x3)
1477 return -EINVAL;
1478
1479 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1480 if (r < 0) {
1481 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1482 return r;
1483 }
1484
1485 while (size) {
1486 u32 value;
1487
1488 r = amdgpu_get_gfx_off_status(adev, &value);
1489 if (r)
1490 goto out;
1491
1492 r = put_user(value, (u32 *)buf);
1493 if (r)
1494 goto out;
1495
1496 result += 4;
1497 buf += 4;
1498 *pos += 4;
1499 size -= 4;
1500 }
1501
1502 r = result;
1503out:
1504 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1505 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1506
1507 return r;
1508}
1509
1510static const struct file_operations amdgpu_debugfs_regs2_fops = {
1511 .owner = THIS_MODULE,
1512 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1513 .read = amdgpu_debugfs_regs2_read,
1514 .write = amdgpu_debugfs_regs2_write,
1515 .open = amdgpu_debugfs_regs2_open,
1516 .release = amdgpu_debugfs_regs2_release,
1517 .llseek = default_llseek
1518};
1519
1520static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1521 .owner = THIS_MODULE,
1522 .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1523 .read = amdgpu_debugfs_gprwave_read,
1524 .open = amdgpu_debugfs_gprwave_open,
1525 .release = amdgpu_debugfs_gprwave_release,
1526 .llseek = default_llseek
1527};
1528
1529static const struct file_operations amdgpu_debugfs_regs_fops = {
1530 .owner = THIS_MODULE,
1531 .read = amdgpu_debugfs_regs_read,
1532 .write = amdgpu_debugfs_regs_write,
1533 .llseek = default_llseek
1534};
1535static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1536 .owner = THIS_MODULE,
1537 .read = amdgpu_debugfs_regs_didt_read,
1538 .write = amdgpu_debugfs_regs_didt_write,
1539 .llseek = default_llseek
1540};
1541static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1542 .owner = THIS_MODULE,
1543 .read = amdgpu_debugfs_regs_pcie_read,
1544 .write = amdgpu_debugfs_regs_pcie_write,
1545 .llseek = default_llseek
1546};
1547static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1548 .owner = THIS_MODULE,
1549 .read = amdgpu_debugfs_regs_smc_read,
1550 .write = amdgpu_debugfs_regs_smc_write,
1551 .llseek = default_llseek
1552};
1553
1554static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1555 .owner = THIS_MODULE,
1556 .read = amdgpu_debugfs_gca_config_read,
1557 .llseek = default_llseek
1558};
1559
1560static const struct file_operations amdgpu_debugfs_sensors_fops = {
1561 .owner = THIS_MODULE,
1562 .read = amdgpu_debugfs_sensor_read,
1563 .llseek = default_llseek
1564};
1565
1566static const struct file_operations amdgpu_debugfs_wave_fops = {
1567 .owner = THIS_MODULE,
1568 .read = amdgpu_debugfs_wave_read,
1569 .llseek = default_llseek
1570};
1571static const struct file_operations amdgpu_debugfs_gpr_fops = {
1572 .owner = THIS_MODULE,
1573 .read = amdgpu_debugfs_gpr_read,
1574 .llseek = default_llseek
1575};
1576
1577static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1578 .owner = THIS_MODULE,
1579 .read = amdgpu_debugfs_gfxoff_read,
1580 .write = amdgpu_debugfs_gfxoff_write,
1581 .llseek = default_llseek
1582};
1583
1584static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1585 .owner = THIS_MODULE,
1586 .read = amdgpu_debugfs_gfxoff_status_read,
1587 .llseek = default_llseek
1588};
1589
1590static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1591 .owner = THIS_MODULE,
1592 .read = amdgpu_debugfs_gfxoff_count_read,
1593 .llseek = default_llseek
1594};
1595
1596static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1597 .owner = THIS_MODULE,
1598 .read = amdgpu_debugfs_gfxoff_residency_read,
1599 .write = amdgpu_debugfs_gfxoff_residency_write,
1600 .llseek = default_llseek
1601};
1602
1603static const struct file_operations *debugfs_regs[] = {
1604 &amdgpu_debugfs_regs_fops,
1605 &amdgpu_debugfs_regs2_fops,
1606 &amdgpu_debugfs_gprwave_fops,
1607 &amdgpu_debugfs_regs_didt_fops,
1608 &amdgpu_debugfs_regs_pcie_fops,
1609 &amdgpu_debugfs_regs_smc_fops,
1610 &amdgpu_debugfs_gca_config_fops,
1611 &amdgpu_debugfs_sensors_fops,
1612 &amdgpu_debugfs_wave_fops,
1613 &amdgpu_debugfs_gpr_fops,
1614 &amdgpu_debugfs_gfxoff_fops,
1615 &amdgpu_debugfs_gfxoff_status_fops,
1616 &amdgpu_debugfs_gfxoff_count_fops,
1617 &amdgpu_debugfs_gfxoff_residency_fops,
1618};
1619
1620static const char * const debugfs_regs_names[] = {
1621 "amdgpu_regs",
1622 "amdgpu_regs2",
1623 "amdgpu_gprwave",
1624 "amdgpu_regs_didt",
1625 "amdgpu_regs_pcie",
1626 "amdgpu_regs_smc",
1627 "amdgpu_gca_config",
1628 "amdgpu_sensors",
1629 "amdgpu_wave",
1630 "amdgpu_gpr",
1631 "amdgpu_gfxoff",
1632 "amdgpu_gfxoff_status",
1633 "amdgpu_gfxoff_count",
1634 "amdgpu_gfxoff_residency",
1635};
1636
1637/**
1638 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1639 * register access.
1640 *
1641 * @adev: The device to attach the debugfs entries to
1642 */
1643int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1644{
1645 struct drm_minor *minor = adev_to_drm(adev)->primary;
1646 struct dentry *ent, *root = minor->debugfs_root;
1647 unsigned int i;
1648
1649 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1650 ent = debugfs_create_file(debugfs_regs_names[i],
1651 S_IFREG | 0444, root,
1652 adev, debugfs_regs[i]);
1653 if (!i && !IS_ERR_OR_NULL(ent))
1654 i_size_write(ent->d_inode, adev->rmmio_size);
1655 }
1656
1657 return 0;
1658}
1659
1660static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1661{
1662 struct amdgpu_device *adev = m->private;
1663 struct drm_device *dev = adev_to_drm(adev);
1664 int r = 0, i;
1665
1666 r = pm_runtime_get_sync(dev->dev);
1667 if (r < 0) {
1668 pm_runtime_put_autosuspend(dev->dev);
1669 return r;
1670 }
1671
1672 /* Avoid accidently unparking the sched thread during GPU reset */
1673 r = down_write_killable(&adev->reset_domain->sem);
1674 if (r)
1675 return r;
1676
1677 /* hold on the scheduler */
1678 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1679 struct amdgpu_ring *ring = adev->rings[i];
1680
1681 if (!amdgpu_ring_sched_ready(ring))
1682 continue;
1683 drm_sched_wqueue_stop(&ring->sched);
1684 }
1685
1686 seq_puts(m, "run ib test:\n");
1687 r = amdgpu_ib_ring_tests(adev);
1688 if (r)
1689 seq_printf(m, "ib ring tests failed (%d).\n", r);
1690 else
1691 seq_puts(m, "ib ring tests passed.\n");
1692
1693 /* go on the scheduler */
1694 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1695 struct amdgpu_ring *ring = adev->rings[i];
1696
1697 if (!amdgpu_ring_sched_ready(ring))
1698 continue;
1699 drm_sched_wqueue_start(&ring->sched);
1700 }
1701
1702 up_write(&adev->reset_domain->sem);
1703
1704 pm_runtime_mark_last_busy(dev->dev);
1705 pm_runtime_put_autosuspend(dev->dev);
1706
1707 return 0;
1708}
1709
1710static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1711{
1712 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1713 struct drm_device *dev = adev_to_drm(adev);
1714 int r;
1715
1716 r = pm_runtime_get_sync(dev->dev);
1717 if (r < 0) {
1718 pm_runtime_put_autosuspend(dev->dev);
1719 return r;
1720 }
1721
1722 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1723
1724 pm_runtime_mark_last_busy(dev->dev);
1725 pm_runtime_put_autosuspend(dev->dev);
1726
1727 return 0;
1728}
1729
1730
1731static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1732{
1733 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1734 struct drm_device *dev = adev_to_drm(adev);
1735 int r;
1736
1737 r = pm_runtime_get_sync(dev->dev);
1738 if (r < 0) {
1739 pm_runtime_put_autosuspend(dev->dev);
1740 return r;
1741 }
1742
1743 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1744
1745 pm_runtime_mark_last_busy(dev->dev);
1746 pm_runtime_put_autosuspend(dev->dev);
1747
1748 return 0;
1749}
1750
1751static int amdgpu_debugfs_benchmark(void *data, u64 val)
1752{
1753 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1754 struct drm_device *dev = adev_to_drm(adev);
1755 int r;
1756
1757 r = pm_runtime_get_sync(dev->dev);
1758 if (r < 0) {
1759 pm_runtime_put_autosuspend(dev->dev);
1760 return r;
1761 }
1762
1763 r = amdgpu_benchmark(adev, val);
1764
1765 pm_runtime_mark_last_busy(dev->dev);
1766 pm_runtime_put_autosuspend(dev->dev);
1767
1768 return r;
1769}
1770
1771static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1772{
1773 struct amdgpu_device *adev = m->private;
1774 struct drm_device *dev = adev_to_drm(adev);
1775 struct drm_file *file;
1776 int r;
1777
1778 r = mutex_lock_interruptible(&dev->filelist_mutex);
1779 if (r)
1780 return r;
1781
1782 list_for_each_entry(file, &dev->filelist, lhead) {
1783 struct amdgpu_fpriv *fpriv = file->driver_priv;
1784 struct amdgpu_vm *vm = &fpriv->vm;
1785 struct amdgpu_task_info *ti;
1786
1787 ti = amdgpu_vm_get_task_info_vm(vm);
1788 if (ti) {
1789 seq_printf(m, "pid:%d\tProcess:%s ----------\n", ti->pid, ti->process_name);
1790 amdgpu_vm_put_task_info(ti);
1791 }
1792
1793 r = amdgpu_bo_reserve(vm->root.bo, true);
1794 if (r)
1795 break;
1796 amdgpu_debugfs_vm_bo_info(vm, m);
1797 amdgpu_bo_unreserve(vm->root.bo);
1798 }
1799
1800 mutex_unlock(&dev->filelist_mutex);
1801
1802 return r;
1803}
1804
1805DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1806DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1807DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1808 NULL, "%lld\n");
1809DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1810 NULL, "%lld\n");
1811DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1812 "%lld\n");
1813
1814static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1815 struct dma_fence **fences)
1816{
1817 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1818 uint32_t sync_seq, last_seq;
1819
1820 last_seq = atomic_read(&ring->fence_drv.last_seq);
1821 sync_seq = ring->fence_drv.sync_seq;
1822
1823 last_seq &= drv->num_fences_mask;
1824 sync_seq &= drv->num_fences_mask;
1825
1826 do {
1827 struct dma_fence *fence, **ptr;
1828
1829 ++last_seq;
1830 last_seq &= drv->num_fences_mask;
1831 ptr = &drv->fences[last_seq];
1832
1833 fence = rcu_dereference_protected(*ptr, 1);
1834 RCU_INIT_POINTER(*ptr, NULL);
1835
1836 if (!fence)
1837 continue;
1838
1839 fences[last_seq] = fence;
1840
1841 } while (last_seq != sync_seq);
1842}
1843
1844static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1845 int length)
1846{
1847 int i;
1848 struct dma_fence *fence;
1849
1850 for (i = 0; i < length; i++) {
1851 fence = fences[i];
1852 if (!fence)
1853 continue;
1854 dma_fence_signal(fence);
1855 dma_fence_put(fence);
1856 }
1857}
1858
1859static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1860{
1861 struct drm_sched_job *s_job;
1862 struct dma_fence *fence;
1863
1864 spin_lock(&sched->job_list_lock);
1865 list_for_each_entry(s_job, &sched->pending_list, list) {
1866 fence = sched->ops->run_job(s_job);
1867 dma_fence_put(fence);
1868 }
1869 spin_unlock(&sched->job_list_lock);
1870}
1871
1872static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1873{
1874 struct amdgpu_job *job;
1875 struct drm_sched_job *s_job, *tmp;
1876 uint32_t preempt_seq;
1877 struct dma_fence *fence, **ptr;
1878 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1879 struct drm_gpu_scheduler *sched = &ring->sched;
1880 bool preempted = true;
1881
1882 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1883 return;
1884
1885 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1886 if (preempt_seq <= atomic_read(&drv->last_seq)) {
1887 preempted = false;
1888 goto no_preempt;
1889 }
1890
1891 preempt_seq &= drv->num_fences_mask;
1892 ptr = &drv->fences[preempt_seq];
1893 fence = rcu_dereference_protected(*ptr, 1);
1894
1895no_preempt:
1896 spin_lock(&sched->job_list_lock);
1897 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1898 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1899 /* remove job from ring_mirror_list */
1900 list_del_init(&s_job->list);
1901 sched->ops->free_job(s_job);
1902 continue;
1903 }
1904 job = to_amdgpu_job(s_job);
1905 if (preempted && (&job->hw_fence) == fence)
1906 /* mark the job as preempted */
1907 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1908 }
1909 spin_unlock(&sched->job_list_lock);
1910}
1911
1912static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1913{
1914 int r, length;
1915 struct amdgpu_ring *ring;
1916 struct dma_fence **fences = NULL;
1917 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1918
1919 if (val >= AMDGPU_MAX_RINGS)
1920 return -EINVAL;
1921
1922 ring = adev->rings[val];
1923
1924 if (!amdgpu_ring_sched_ready(ring) ||
1925 !ring->funcs->preempt_ib)
1926 return -EINVAL;
1927
1928 /* the last preemption failed */
1929 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1930 return -EBUSY;
1931
1932 length = ring->fence_drv.num_fences_mask + 1;
1933 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1934 if (!fences)
1935 return -ENOMEM;
1936
1937 /* Avoid accidently unparking the sched thread during GPU reset */
1938 r = down_read_killable(&adev->reset_domain->sem);
1939 if (r)
1940 goto pro_end;
1941
1942 /* stop the scheduler */
1943 drm_sched_wqueue_stop(&ring->sched);
1944
1945 /* preempt the IB */
1946 r = amdgpu_ring_preempt_ib(ring);
1947 if (r) {
1948 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1949 goto failure;
1950 }
1951
1952 amdgpu_fence_process(ring);
1953
1954 if (atomic_read(&ring->fence_drv.last_seq) !=
1955 ring->fence_drv.sync_seq) {
1956 DRM_INFO("ring %d was preempted\n", ring->idx);
1957
1958 amdgpu_ib_preempt_mark_partial_job(ring);
1959
1960 /* swap out the old fences */
1961 amdgpu_ib_preempt_fences_swap(ring, fences);
1962
1963 amdgpu_fence_driver_force_completion(ring);
1964
1965 /* resubmit unfinished jobs */
1966 amdgpu_ib_preempt_job_recovery(&ring->sched);
1967
1968 /* wait for jobs finished */
1969 amdgpu_fence_wait_empty(ring);
1970
1971 /* signal the old fences */
1972 amdgpu_ib_preempt_signal_fences(fences, length);
1973 }
1974
1975failure:
1976 /* restart the scheduler */
1977 drm_sched_wqueue_start(&ring->sched);
1978
1979 up_read(&adev->reset_domain->sem);
1980
1981pro_end:
1982 kfree(fences);
1983
1984 return r;
1985}
1986
1987static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1988{
1989 int ret = 0;
1990 uint32_t max_freq, min_freq;
1991 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1992
1993 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1994 return -EINVAL;
1995
1996 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1997 if (ret < 0) {
1998 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1999 return ret;
2000 }
2001
2002 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
2003 if (ret == -EOPNOTSUPP) {
2004 ret = 0;
2005 goto out;
2006 }
2007 if (ret || val > max_freq || val < min_freq) {
2008 ret = -EINVAL;
2009 goto out;
2010 }
2011
2012 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
2013 if (ret)
2014 ret = -EINVAL;
2015
2016out:
2017 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
2018 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2019
2020 return ret;
2021}
2022
2023DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
2024 amdgpu_debugfs_ib_preempt, "%llu\n");
2025
2026DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2027 amdgpu_debugfs_sclk_set, "%llu\n");
2028
2029static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
2030 char __user *buf, size_t size, loff_t *pos)
2031{
2032 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2033 char reg_offset[12];
2034 int i, ret, len = 0;
2035
2036 if (*pos)
2037 return 0;
2038
2039 memset(reg_offset, 0, 12);
2040 ret = down_read_killable(&adev->reset_domain->sem);
2041 if (ret)
2042 return ret;
2043
2044 for (i = 0; i < adev->reset_info.num_regs; i++) {
2045 sprintf(reg_offset, "0x%x\n", adev->reset_info.reset_dump_reg_list[i]);
2046 up_read(&adev->reset_domain->sem);
2047 if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
2048 return -EFAULT;
2049
2050 len += strlen(reg_offset);
2051 ret = down_read_killable(&adev->reset_domain->sem);
2052 if (ret)
2053 return ret;
2054 }
2055
2056 up_read(&adev->reset_domain->sem);
2057 *pos += len;
2058
2059 return len;
2060}
2061
2062static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
2063 const char __user *buf, size_t size, loff_t *pos)
2064{
2065 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2066 char reg_offset[11];
2067 uint32_t *new = NULL, *tmp = NULL;
2068 int ret, i = 0, len = 0;
2069
2070 do {
2071 memset(reg_offset, 0, 11);
2072 if (copy_from_user(reg_offset, buf + len,
2073 min(10, ((int)size-len)))) {
2074 ret = -EFAULT;
2075 goto error_free;
2076 }
2077
2078 new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
2079 if (!new) {
2080 ret = -ENOMEM;
2081 goto error_free;
2082 }
2083 tmp = new;
2084 if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
2085 ret = -EINVAL;
2086 goto error_free;
2087 }
2088
2089 len += ret;
2090 i++;
2091 } while (len < size);
2092
2093 new = kmalloc_array(i, sizeof(uint32_t), GFP_KERNEL);
2094 if (!new) {
2095 ret = -ENOMEM;
2096 goto error_free;
2097 }
2098 ret = down_write_killable(&adev->reset_domain->sem);
2099 if (ret)
2100 goto error_free;
2101
2102 swap(adev->reset_info.reset_dump_reg_list, tmp);
2103 swap(adev->reset_info.reset_dump_reg_value, new);
2104 adev->reset_info.num_regs = i;
2105 up_write(&adev->reset_domain->sem);
2106 ret = size;
2107
2108error_free:
2109 if (tmp != new)
2110 kfree(tmp);
2111 kfree(new);
2112 return ret;
2113}
2114
2115static const struct file_operations amdgpu_reset_dump_register_list = {
2116 .owner = THIS_MODULE,
2117 .read = amdgpu_reset_dump_register_list_read,
2118 .write = amdgpu_reset_dump_register_list_write,
2119 .llseek = default_llseek
2120};
2121
2122int amdgpu_debugfs_init(struct amdgpu_device *adev)
2123{
2124 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2125 struct dentry *ent;
2126 int r, i;
2127
2128 if (!debugfs_initialized())
2129 return 0;
2130
2131 debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2132 &adev->pm.smu_debug_mask);
2133
2134 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2135 &fops_ib_preempt);
2136 if (IS_ERR(ent)) {
2137 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
2138 return PTR_ERR(ent);
2139 }
2140
2141 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2142 &fops_sclk_set);
2143 if (IS_ERR(ent)) {
2144 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
2145 return PTR_ERR(ent);
2146 }
2147
2148 /* Register debugfs entries for amdgpu_ttm */
2149 amdgpu_ttm_debugfs_init(adev);
2150 amdgpu_debugfs_pm_init(adev);
2151 amdgpu_debugfs_sa_init(adev);
2152 amdgpu_debugfs_fence_init(adev);
2153 amdgpu_debugfs_gem_init(adev);
2154
2155 r = amdgpu_debugfs_regs_init(adev);
2156 if (r)
2157 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2158
2159 amdgpu_debugfs_firmware_init(adev);
2160 amdgpu_ta_if_debugfs_init(adev);
2161
2162 amdgpu_debugfs_mes_event_log_init(adev);
2163
2164#if defined(CONFIG_DRM_AMD_DC)
2165 if (adev->dc_enabled)
2166 dtn_debugfs_init(adev);
2167#endif
2168
2169 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2170 struct amdgpu_ring *ring = adev->rings[i];
2171
2172 if (!ring)
2173 continue;
2174
2175 amdgpu_debugfs_ring_init(adev, ring);
2176 }
2177
2178 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2179 if (!amdgpu_vcnfw_log)
2180 break;
2181
2182 if (adev->vcn.harvest_config & (1 << i))
2183 continue;
2184
2185 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2186 }
2187
2188 amdgpu_ras_debugfs_create_all(adev);
2189 amdgpu_rap_debugfs_init(adev);
2190 amdgpu_securedisplay_debugfs_init(adev);
2191 amdgpu_fw_attestation_debugfs_init(adev);
2192
2193 debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
2194 &amdgpu_evict_vram_fops);
2195 debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
2196 &amdgpu_evict_gtt_fops);
2197 debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2198 &amdgpu_debugfs_test_ib_fops);
2199 debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2200 &amdgpu_debugfs_vm_info_fops);
2201 debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2202 &amdgpu_benchmark_fops);
2203 debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
2204 &amdgpu_reset_dump_register_list);
2205
2206 adev->debugfs_vbios_blob.data = adev->bios;
2207 adev->debugfs_vbios_blob.size = adev->bios_size;
2208 debugfs_create_blob("amdgpu_vbios", 0444, root,
2209 &adev->debugfs_vbios_blob);
2210
2211 adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2212 adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2213 debugfs_create_blob("amdgpu_discovery", 0444, root,
2214 &adev->debugfs_discovery_blob);
2215
2216 return 0;
2217}
2218
2219#else
2220int amdgpu_debugfs_init(struct amdgpu_device *adev)
2221{
2222 return 0;
2223}
2224int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2225{
2226 return 0;
2227}
2228#endif
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26#include <linux/kthread.h>
27#include <linux/pci.h>
28#include <linux/uaccess.h>
29#include <linux/pm_runtime.h>
30#include <linux/poll.h>
31#include <drm/drm_debugfs.h>
32
33#include "amdgpu.h"
34#include "amdgpu_pm.h"
35#include "amdgpu_dm_debugfs.h"
36#include "amdgpu_ras.h"
37
38/**
39 * amdgpu_debugfs_add_files - Add simple debugfs entries
40 *
41 * @adev: Device to attach debugfs entries to
42 * @files: Array of function callbacks that respond to reads
43 * @nfiles: Number of callbacks to register
44 *
45 */
46int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
47 const struct drm_info_list *files,
48 unsigned nfiles)
49{
50 unsigned i;
51
52 for (i = 0; i < adev->debugfs_count; i++) {
53 if (adev->debugfs[i].files == files) {
54 /* Already registered */
55 return 0;
56 }
57 }
58
59 i = adev->debugfs_count + 1;
60 if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
61 DRM_ERROR("Reached maximum number of debugfs components.\n");
62 DRM_ERROR("Report so we increase "
63 "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
64 return -EINVAL;
65 }
66 adev->debugfs[adev->debugfs_count].files = files;
67 adev->debugfs[adev->debugfs_count].num_files = nfiles;
68 adev->debugfs_count = i;
69#if defined(CONFIG_DEBUG_FS)
70 drm_debugfs_create_files(files, nfiles,
71 adev->ddev->primary->debugfs_root,
72 adev->ddev->primary);
73#endif
74 return 0;
75}
76
77int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
78{
79#if defined(CONFIG_DEBUG_FS)
80 unsigned long timeout = 600 * HZ;
81 int ret;
82
83 wake_up_interruptible(&adev->autodump.gpu_hang);
84
85 ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
86 if (ret == 0) {
87 pr_err("autodump: timeout, move on to gpu recovery\n");
88 return -ETIMEDOUT;
89 }
90#endif
91 return 0;
92}
93
94#if defined(CONFIG_DEBUG_FS)
95
96static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
97{
98 struct amdgpu_device *adev = inode->i_private;
99 int ret;
100
101 file->private_data = adev;
102
103 mutex_lock(&adev->lock_reset);
104 if (adev->autodump.dumping.done) {
105 reinit_completion(&adev->autodump.dumping);
106 ret = 0;
107 } else {
108 ret = -EBUSY;
109 }
110 mutex_unlock(&adev->lock_reset);
111
112 return ret;
113}
114
115static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
116{
117 struct amdgpu_device *adev = file->private_data;
118
119 complete_all(&adev->autodump.dumping);
120 return 0;
121}
122
123static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
124{
125 struct amdgpu_device *adev = file->private_data;
126
127 poll_wait(file, &adev->autodump.gpu_hang, poll_table);
128
129 if (adev->in_gpu_reset)
130 return POLLIN | POLLRDNORM | POLLWRNORM;
131
132 return 0;
133}
134
135static const struct file_operations autodump_debug_fops = {
136 .owner = THIS_MODULE,
137 .open = amdgpu_debugfs_autodump_open,
138 .poll = amdgpu_debugfs_autodump_poll,
139 .release = amdgpu_debugfs_autodump_release,
140};
141
142static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
143{
144 init_completion(&adev->autodump.dumping);
145 complete_all(&adev->autodump.dumping);
146 init_waitqueue_head(&adev->autodump.gpu_hang);
147
148 debugfs_create_file("amdgpu_autodump", 0600,
149 adev->ddev->primary->debugfs_root,
150 adev, &autodump_debug_fops);
151}
152
153/**
154 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
155 *
156 * @read: True if reading
157 * @f: open file handle
158 * @buf: User buffer to write/read to
159 * @size: Number of bytes to write/read
160 * @pos: Offset to seek to
161 *
162 * This debugfs entry has special meaning on the offset being sought.
163 * Various bits have different meanings:
164 *
165 * Bit 62: Indicates a GRBM bank switch is needed
166 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
167 * zero)
168 * Bits 24..33: The SE or ME selector if needed
169 * Bits 34..43: The SH (or SA) or PIPE selector if needed
170 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
171 *
172 * Bit 23: Indicates that the PM power gating lock should be held
173 * This is necessary to read registers that might be
174 * unreliable during a power gating transistion.
175 *
176 * The lower bits are the BYTE offset of the register to read. This
177 * allows reading multiple registers in a single call and having
178 * the returned size reflect that.
179 */
180static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
181 char __user *buf, size_t size, loff_t *pos)
182{
183 struct amdgpu_device *adev = file_inode(f)->i_private;
184 ssize_t result = 0;
185 int r;
186 bool pm_pg_lock, use_bank, use_ring;
187 unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
188
189 pm_pg_lock = use_bank = use_ring = false;
190 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
191
192 if (size & 0x3 || *pos & 0x3 ||
193 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
194 return -EINVAL;
195
196 /* are we reading registers for which a PG lock is necessary? */
197 pm_pg_lock = (*pos >> 23) & 1;
198
199 if (*pos & (1ULL << 62)) {
200 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
201 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
202 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
203
204 if (se_bank == 0x3FF)
205 se_bank = 0xFFFFFFFF;
206 if (sh_bank == 0x3FF)
207 sh_bank = 0xFFFFFFFF;
208 if (instance_bank == 0x3FF)
209 instance_bank = 0xFFFFFFFF;
210 use_bank = true;
211 } else if (*pos & (1ULL << 61)) {
212
213 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
214 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
215 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
216 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
217
218 use_ring = true;
219 } else {
220 use_bank = use_ring = false;
221 }
222
223 *pos &= (1UL << 22) - 1;
224
225 r = pm_runtime_get_sync(adev->ddev->dev);
226 if (r < 0) {
227 pm_runtime_put_autosuspend(adev->ddev->dev);
228 return r;
229 }
230
231 r = amdgpu_virt_enable_access_debugfs(adev);
232 if (r < 0) {
233 pm_runtime_put_autosuspend(adev->ddev->dev);
234 return r;
235 }
236
237 if (use_bank) {
238 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
239 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
240 pm_runtime_mark_last_busy(adev->ddev->dev);
241 pm_runtime_put_autosuspend(adev->ddev->dev);
242 amdgpu_virt_disable_access_debugfs(adev);
243 return -EINVAL;
244 }
245 mutex_lock(&adev->grbm_idx_mutex);
246 amdgpu_gfx_select_se_sh(adev, se_bank,
247 sh_bank, instance_bank);
248 } else if (use_ring) {
249 mutex_lock(&adev->srbm_mutex);
250 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
251 }
252
253 if (pm_pg_lock)
254 mutex_lock(&adev->pm.mutex);
255
256 while (size) {
257 uint32_t value;
258
259 if (read) {
260 value = RREG32(*pos >> 2);
261 r = put_user(value, (uint32_t *)buf);
262 } else {
263 r = get_user(value, (uint32_t *)buf);
264 if (!r)
265 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
266 }
267 if (r) {
268 result = r;
269 goto end;
270 }
271
272 result += 4;
273 buf += 4;
274 *pos += 4;
275 size -= 4;
276 }
277
278end:
279 if (use_bank) {
280 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
281 mutex_unlock(&adev->grbm_idx_mutex);
282 } else if (use_ring) {
283 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
284 mutex_unlock(&adev->srbm_mutex);
285 }
286
287 if (pm_pg_lock)
288 mutex_unlock(&adev->pm.mutex);
289
290 pm_runtime_mark_last_busy(adev->ddev->dev);
291 pm_runtime_put_autosuspend(adev->ddev->dev);
292
293 amdgpu_virt_disable_access_debugfs(adev);
294 return result;
295}
296
297/**
298 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
299 */
300static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
301 size_t size, loff_t *pos)
302{
303 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
304}
305
306/**
307 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
308 */
309static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
310 size_t size, loff_t *pos)
311{
312 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
313}
314
315
316/**
317 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
318 *
319 * @f: open file handle
320 * @buf: User buffer to store read data in
321 * @size: Number of bytes to read
322 * @pos: Offset to seek to
323 *
324 * The lower bits are the BYTE offset of the register to read. This
325 * allows reading multiple registers in a single call and having
326 * the returned size reflect that.
327 */
328static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
329 size_t size, loff_t *pos)
330{
331 struct amdgpu_device *adev = file_inode(f)->i_private;
332 ssize_t result = 0;
333 int r;
334
335 if (size & 0x3 || *pos & 0x3)
336 return -EINVAL;
337
338 r = pm_runtime_get_sync(adev->ddev->dev);
339 if (r < 0) {
340 pm_runtime_put_autosuspend(adev->ddev->dev);
341 return r;
342 }
343
344 r = amdgpu_virt_enable_access_debugfs(adev);
345 if (r < 0) {
346 pm_runtime_put_autosuspend(adev->ddev->dev);
347 return r;
348 }
349
350 while (size) {
351 uint32_t value;
352
353 value = RREG32_PCIE(*pos >> 2);
354 r = put_user(value, (uint32_t *)buf);
355 if (r) {
356 pm_runtime_mark_last_busy(adev->ddev->dev);
357 pm_runtime_put_autosuspend(adev->ddev->dev);
358 amdgpu_virt_disable_access_debugfs(adev);
359 return r;
360 }
361
362 result += 4;
363 buf += 4;
364 *pos += 4;
365 size -= 4;
366 }
367
368 pm_runtime_mark_last_busy(adev->ddev->dev);
369 pm_runtime_put_autosuspend(adev->ddev->dev);
370
371 amdgpu_virt_disable_access_debugfs(adev);
372 return result;
373}
374
375/**
376 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
377 *
378 * @f: open file handle
379 * @buf: User buffer to write data from
380 * @size: Number of bytes to write
381 * @pos: Offset to seek to
382 *
383 * The lower bits are the BYTE offset of the register to write. This
384 * allows writing multiple registers in a single call and having
385 * the returned size reflect that.
386 */
387static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
388 size_t size, loff_t *pos)
389{
390 struct amdgpu_device *adev = file_inode(f)->i_private;
391 ssize_t result = 0;
392 int r;
393
394 if (size & 0x3 || *pos & 0x3)
395 return -EINVAL;
396
397 r = pm_runtime_get_sync(adev->ddev->dev);
398 if (r < 0) {
399 pm_runtime_put_autosuspend(adev->ddev->dev);
400 return r;
401 }
402
403 r = amdgpu_virt_enable_access_debugfs(adev);
404 if (r < 0) {
405 pm_runtime_put_autosuspend(adev->ddev->dev);
406 return r;
407 }
408
409 while (size) {
410 uint32_t value;
411
412 r = get_user(value, (uint32_t *)buf);
413 if (r) {
414 pm_runtime_mark_last_busy(adev->ddev->dev);
415 pm_runtime_put_autosuspend(adev->ddev->dev);
416 amdgpu_virt_disable_access_debugfs(adev);
417 return r;
418 }
419
420 WREG32_PCIE(*pos >> 2, value);
421
422 result += 4;
423 buf += 4;
424 *pos += 4;
425 size -= 4;
426 }
427
428 pm_runtime_mark_last_busy(adev->ddev->dev);
429 pm_runtime_put_autosuspend(adev->ddev->dev);
430
431 amdgpu_virt_disable_access_debugfs(adev);
432 return result;
433}
434
435/**
436 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
437 *
438 * @f: open file handle
439 * @buf: User buffer to store read data in
440 * @size: Number of bytes to read
441 * @pos: Offset to seek to
442 *
443 * The lower bits are the BYTE offset of the register to read. This
444 * allows reading multiple registers in a single call and having
445 * the returned size reflect that.
446 */
447static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
448 size_t size, loff_t *pos)
449{
450 struct amdgpu_device *adev = file_inode(f)->i_private;
451 ssize_t result = 0;
452 int r;
453
454 if (size & 0x3 || *pos & 0x3)
455 return -EINVAL;
456
457 r = pm_runtime_get_sync(adev->ddev->dev);
458 if (r < 0) {
459 pm_runtime_put_autosuspend(adev->ddev->dev);
460 return r;
461 }
462
463 r = amdgpu_virt_enable_access_debugfs(adev);
464 if (r < 0) {
465 pm_runtime_put_autosuspend(adev->ddev->dev);
466 return r;
467 }
468
469 while (size) {
470 uint32_t value;
471
472 value = RREG32_DIDT(*pos >> 2);
473 r = put_user(value, (uint32_t *)buf);
474 if (r) {
475 pm_runtime_mark_last_busy(adev->ddev->dev);
476 pm_runtime_put_autosuspend(adev->ddev->dev);
477 amdgpu_virt_disable_access_debugfs(adev);
478 return r;
479 }
480
481 result += 4;
482 buf += 4;
483 *pos += 4;
484 size -= 4;
485 }
486
487 pm_runtime_mark_last_busy(adev->ddev->dev);
488 pm_runtime_put_autosuspend(adev->ddev->dev);
489
490 amdgpu_virt_disable_access_debugfs(adev);
491 return result;
492}
493
494/**
495 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
496 *
497 * @f: open file handle
498 * @buf: User buffer to write data from
499 * @size: Number of bytes to write
500 * @pos: Offset to seek to
501 *
502 * The lower bits are the BYTE offset of the register to write. This
503 * allows writing multiple registers in a single call and having
504 * the returned size reflect that.
505 */
506static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
507 size_t size, loff_t *pos)
508{
509 struct amdgpu_device *adev = file_inode(f)->i_private;
510 ssize_t result = 0;
511 int r;
512
513 if (size & 0x3 || *pos & 0x3)
514 return -EINVAL;
515
516 r = pm_runtime_get_sync(adev->ddev->dev);
517 if (r < 0) {
518 pm_runtime_put_autosuspend(adev->ddev->dev);
519 return r;
520 }
521
522 r = amdgpu_virt_enable_access_debugfs(adev);
523 if (r < 0) {
524 pm_runtime_put_autosuspend(adev->ddev->dev);
525 return r;
526 }
527
528 while (size) {
529 uint32_t value;
530
531 r = get_user(value, (uint32_t *)buf);
532 if (r) {
533 pm_runtime_mark_last_busy(adev->ddev->dev);
534 pm_runtime_put_autosuspend(adev->ddev->dev);
535 amdgpu_virt_disable_access_debugfs(adev);
536 return r;
537 }
538
539 WREG32_DIDT(*pos >> 2, value);
540
541 result += 4;
542 buf += 4;
543 *pos += 4;
544 size -= 4;
545 }
546
547 pm_runtime_mark_last_busy(adev->ddev->dev);
548 pm_runtime_put_autosuspend(adev->ddev->dev);
549
550 amdgpu_virt_disable_access_debugfs(adev);
551 return result;
552}
553
554/**
555 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
556 *
557 * @f: open file handle
558 * @buf: User buffer to store read data in
559 * @size: Number of bytes to read
560 * @pos: Offset to seek to
561 *
562 * The lower bits are the BYTE offset of the register to read. This
563 * allows reading multiple registers in a single call and having
564 * the returned size reflect that.
565 */
566static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
567 size_t size, loff_t *pos)
568{
569 struct amdgpu_device *adev = file_inode(f)->i_private;
570 ssize_t result = 0;
571 int r;
572
573 if (size & 0x3 || *pos & 0x3)
574 return -EINVAL;
575
576 r = pm_runtime_get_sync(adev->ddev->dev);
577 if (r < 0) {
578 pm_runtime_put_autosuspend(adev->ddev->dev);
579 return r;
580 }
581
582 r = amdgpu_virt_enable_access_debugfs(adev);
583 if (r < 0) {
584 pm_runtime_put_autosuspend(adev->ddev->dev);
585 return r;
586 }
587
588 while (size) {
589 uint32_t value;
590
591 value = RREG32_SMC(*pos);
592 r = put_user(value, (uint32_t *)buf);
593 if (r) {
594 pm_runtime_mark_last_busy(adev->ddev->dev);
595 pm_runtime_put_autosuspend(adev->ddev->dev);
596 amdgpu_virt_disable_access_debugfs(adev);
597 return r;
598 }
599
600 result += 4;
601 buf += 4;
602 *pos += 4;
603 size -= 4;
604 }
605
606 pm_runtime_mark_last_busy(adev->ddev->dev);
607 pm_runtime_put_autosuspend(adev->ddev->dev);
608
609 amdgpu_virt_disable_access_debugfs(adev);
610 return result;
611}
612
613/**
614 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
615 *
616 * @f: open file handle
617 * @buf: User buffer to write data from
618 * @size: Number of bytes to write
619 * @pos: Offset to seek to
620 *
621 * The lower bits are the BYTE offset of the register to write. This
622 * allows writing multiple registers in a single call and having
623 * the returned size reflect that.
624 */
625static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
626 size_t size, loff_t *pos)
627{
628 struct amdgpu_device *adev = file_inode(f)->i_private;
629 ssize_t result = 0;
630 int r;
631
632 if (size & 0x3 || *pos & 0x3)
633 return -EINVAL;
634
635 r = pm_runtime_get_sync(adev->ddev->dev);
636 if (r < 0) {
637 pm_runtime_put_autosuspend(adev->ddev->dev);
638 return r;
639 }
640
641 r = amdgpu_virt_enable_access_debugfs(adev);
642 if (r < 0) {
643 pm_runtime_put_autosuspend(adev->ddev->dev);
644 return r;
645 }
646
647 while (size) {
648 uint32_t value;
649
650 r = get_user(value, (uint32_t *)buf);
651 if (r) {
652 pm_runtime_mark_last_busy(adev->ddev->dev);
653 pm_runtime_put_autosuspend(adev->ddev->dev);
654 amdgpu_virt_disable_access_debugfs(adev);
655 return r;
656 }
657
658 WREG32_SMC(*pos, value);
659
660 result += 4;
661 buf += 4;
662 *pos += 4;
663 size -= 4;
664 }
665
666 pm_runtime_mark_last_busy(adev->ddev->dev);
667 pm_runtime_put_autosuspend(adev->ddev->dev);
668
669 amdgpu_virt_disable_access_debugfs(adev);
670 return result;
671}
672
673/**
674 * amdgpu_debugfs_gca_config_read - Read from gfx config data
675 *
676 * @f: open file handle
677 * @buf: User buffer to store read data in
678 * @size: Number of bytes to read
679 * @pos: Offset to seek to
680 *
681 * This file is used to access configuration data in a somewhat
682 * stable fashion. The format is a series of DWORDs with the first
683 * indicating which revision it is. New content is appended to the
684 * end so that older software can still read the data.
685 */
686
687static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
688 size_t size, loff_t *pos)
689{
690 struct amdgpu_device *adev = file_inode(f)->i_private;
691 ssize_t result = 0;
692 int r;
693 uint32_t *config, no_regs = 0;
694
695 if (size & 0x3 || *pos & 0x3)
696 return -EINVAL;
697
698 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
699 if (!config)
700 return -ENOMEM;
701
702 /* version, increment each time something is added */
703 config[no_regs++] = 3;
704 config[no_regs++] = adev->gfx.config.max_shader_engines;
705 config[no_regs++] = adev->gfx.config.max_tile_pipes;
706 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
707 config[no_regs++] = adev->gfx.config.max_sh_per_se;
708 config[no_regs++] = adev->gfx.config.max_backends_per_se;
709 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
710 config[no_regs++] = adev->gfx.config.max_gprs;
711 config[no_regs++] = adev->gfx.config.max_gs_threads;
712 config[no_regs++] = adev->gfx.config.max_hw_contexts;
713 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
714 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
715 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
716 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
717 config[no_regs++] = adev->gfx.config.num_tile_pipes;
718 config[no_regs++] = adev->gfx.config.backend_enable_mask;
719 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
720 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
721 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
722 config[no_regs++] = adev->gfx.config.num_gpus;
723 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
724 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
725 config[no_regs++] = adev->gfx.config.gb_addr_config;
726 config[no_regs++] = adev->gfx.config.num_rbs;
727
728 /* rev==1 */
729 config[no_regs++] = adev->rev_id;
730 config[no_regs++] = adev->pg_flags;
731 config[no_regs++] = adev->cg_flags;
732
733 /* rev==2 */
734 config[no_regs++] = adev->family;
735 config[no_regs++] = adev->external_rev_id;
736
737 /* rev==3 */
738 config[no_regs++] = adev->pdev->device;
739 config[no_regs++] = adev->pdev->revision;
740 config[no_regs++] = adev->pdev->subsystem_device;
741 config[no_regs++] = adev->pdev->subsystem_vendor;
742
743 while (size && (*pos < no_regs * 4)) {
744 uint32_t value;
745
746 value = config[*pos >> 2];
747 r = put_user(value, (uint32_t *)buf);
748 if (r) {
749 kfree(config);
750 return r;
751 }
752
753 result += 4;
754 buf += 4;
755 *pos += 4;
756 size -= 4;
757 }
758
759 kfree(config);
760 return result;
761}
762
763/**
764 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
765 *
766 * @f: open file handle
767 * @buf: User buffer to store read data in
768 * @size: Number of bytes to read
769 * @pos: Offset to seek to
770 *
771 * The offset is treated as the BYTE address of one of the sensors
772 * enumerated in amd/include/kgd_pp_interface.h under the
773 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
774 * you would use the offset 3 * 4 = 12.
775 */
776static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
777 size_t size, loff_t *pos)
778{
779 struct amdgpu_device *adev = file_inode(f)->i_private;
780 int idx, x, outsize, r, valuesize;
781 uint32_t values[16];
782
783 if (size & 3 || *pos & 0x3)
784 return -EINVAL;
785
786 if (!adev->pm.dpm_enabled)
787 return -EINVAL;
788
789 /* convert offset to sensor number */
790 idx = *pos >> 2;
791
792 valuesize = sizeof(values);
793
794 r = pm_runtime_get_sync(adev->ddev->dev);
795 if (r < 0) {
796 pm_runtime_put_autosuspend(adev->ddev->dev);
797 return r;
798 }
799
800 r = amdgpu_virt_enable_access_debugfs(adev);
801 if (r < 0) {
802 pm_runtime_put_autosuspend(adev->ddev->dev);
803 return r;
804 }
805
806 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
807
808 pm_runtime_mark_last_busy(adev->ddev->dev);
809 pm_runtime_put_autosuspend(adev->ddev->dev);
810
811 if (r) {
812 amdgpu_virt_disable_access_debugfs(adev);
813 return r;
814 }
815
816 if (size > valuesize) {
817 amdgpu_virt_disable_access_debugfs(adev);
818 return -EINVAL;
819 }
820
821 outsize = 0;
822 x = 0;
823 if (!r) {
824 while (size) {
825 r = put_user(values[x++], (int32_t *)buf);
826 buf += 4;
827 size -= 4;
828 outsize += 4;
829 }
830 }
831
832 amdgpu_virt_disable_access_debugfs(adev);
833 return !r ? outsize : r;
834}
835
836/** amdgpu_debugfs_wave_read - Read WAVE STATUS data
837 *
838 * @f: open file handle
839 * @buf: User buffer to store read data in
840 * @size: Number of bytes to read
841 * @pos: Offset to seek to
842 *
843 * The offset being sought changes which wave that the status data
844 * will be returned for. The bits are used as follows:
845 *
846 * Bits 0..6: Byte offset into data
847 * Bits 7..14: SE selector
848 * Bits 15..22: SH/SA selector
849 * Bits 23..30: CU/{WGP+SIMD} selector
850 * Bits 31..36: WAVE ID selector
851 * Bits 37..44: SIMD ID selector
852 *
853 * The returned data begins with one DWORD of version information
854 * Followed by WAVE STATUS registers relevant to the GFX IP version
855 * being used. See gfx_v8_0_read_wave_data() for an example output.
856 */
857static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
858 size_t size, loff_t *pos)
859{
860 struct amdgpu_device *adev = f->f_inode->i_private;
861 int r, x;
862 ssize_t result=0;
863 uint32_t offset, se, sh, cu, wave, simd, data[32];
864
865 if (size & 3 || *pos & 3)
866 return -EINVAL;
867
868 /* decode offset */
869 offset = (*pos & GENMASK_ULL(6, 0));
870 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
871 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
872 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
873 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
874 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
875
876 r = pm_runtime_get_sync(adev->ddev->dev);
877 if (r < 0) {
878 pm_runtime_put_autosuspend(adev->ddev->dev);
879 return r;
880 }
881
882 r = amdgpu_virt_enable_access_debugfs(adev);
883 if (r < 0) {
884 pm_runtime_put_autosuspend(adev->ddev->dev);
885 return r;
886 }
887
888 /* switch to the specific se/sh/cu */
889 mutex_lock(&adev->grbm_idx_mutex);
890 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
891
892 x = 0;
893 if (adev->gfx.funcs->read_wave_data)
894 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
895
896 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
897 mutex_unlock(&adev->grbm_idx_mutex);
898
899 pm_runtime_mark_last_busy(adev->ddev->dev);
900 pm_runtime_put_autosuspend(adev->ddev->dev);
901
902 if (!x) {
903 amdgpu_virt_disable_access_debugfs(adev);
904 return -EINVAL;
905 }
906
907 while (size && (offset < x * 4)) {
908 uint32_t value;
909
910 value = data[offset >> 2];
911 r = put_user(value, (uint32_t *)buf);
912 if (r) {
913 amdgpu_virt_disable_access_debugfs(adev);
914 return r;
915 }
916
917 result += 4;
918 buf += 4;
919 offset += 4;
920 size -= 4;
921 }
922
923 amdgpu_virt_disable_access_debugfs(adev);
924 return result;
925}
926
927/** amdgpu_debugfs_gpr_read - Read wave gprs
928 *
929 * @f: open file handle
930 * @buf: User buffer to store read data in
931 * @size: Number of bytes to read
932 * @pos: Offset to seek to
933 *
934 * The offset being sought changes which wave that the status data
935 * will be returned for. The bits are used as follows:
936 *
937 * Bits 0..11: Byte offset into data
938 * Bits 12..19: SE selector
939 * Bits 20..27: SH/SA selector
940 * Bits 28..35: CU/{WGP+SIMD} selector
941 * Bits 36..43: WAVE ID selector
942 * Bits 37..44: SIMD ID selector
943 * Bits 52..59: Thread selector
944 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
945 *
946 * The return data comes from the SGPR or VGPR register bank for
947 * the selected operational unit.
948 */
949static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
950 size_t size, loff_t *pos)
951{
952 struct amdgpu_device *adev = f->f_inode->i_private;
953 int r;
954 ssize_t result = 0;
955 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
956
957 if (size > 4096 || size & 3 || *pos & 3)
958 return -EINVAL;
959
960 /* decode offset */
961 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
962 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
963 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
964 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
965 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
966 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
967 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
968 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
969
970 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
971 if (!data)
972 return -ENOMEM;
973
974 r = pm_runtime_get_sync(adev->ddev->dev);
975 if (r < 0)
976 goto err;
977
978 r = amdgpu_virt_enable_access_debugfs(adev);
979 if (r < 0)
980 goto err;
981
982 /* switch to the specific se/sh/cu */
983 mutex_lock(&adev->grbm_idx_mutex);
984 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
985
986 if (bank == 0) {
987 if (adev->gfx.funcs->read_wave_vgprs)
988 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
989 } else {
990 if (adev->gfx.funcs->read_wave_sgprs)
991 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
992 }
993
994 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
995 mutex_unlock(&adev->grbm_idx_mutex);
996
997 pm_runtime_mark_last_busy(adev->ddev->dev);
998 pm_runtime_put_autosuspend(adev->ddev->dev);
999
1000 while (size) {
1001 uint32_t value;
1002
1003 value = data[result >> 2];
1004 r = put_user(value, (uint32_t *)buf);
1005 if (r) {
1006 amdgpu_virt_disable_access_debugfs(adev);
1007 goto err;
1008 }
1009
1010 result += 4;
1011 buf += 4;
1012 size -= 4;
1013 }
1014
1015 kfree(data);
1016 amdgpu_virt_disable_access_debugfs(adev);
1017 return result;
1018
1019err:
1020 pm_runtime_put_autosuspend(adev->ddev->dev);
1021 kfree(data);
1022 return r;
1023}
1024
1025/**
1026 * amdgpu_debugfs_regs_gfxoff_write - Enable/disable GFXOFF
1027 *
1028 * @f: open file handle
1029 * @buf: User buffer to write data from
1030 * @size: Number of bytes to write
1031 * @pos: Offset to seek to
1032 *
1033 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1034 */
1035static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1036 size_t size, loff_t *pos)
1037{
1038 struct amdgpu_device *adev = file_inode(f)->i_private;
1039 ssize_t result = 0;
1040 int r;
1041
1042 if (size & 0x3 || *pos & 0x3)
1043 return -EINVAL;
1044
1045 r = pm_runtime_get_sync(adev->ddev->dev);
1046 if (r < 0) {
1047 pm_runtime_put_autosuspend(adev->ddev->dev);
1048 return r;
1049 }
1050
1051 while (size) {
1052 uint32_t value;
1053
1054 r = get_user(value, (uint32_t *)buf);
1055 if (r) {
1056 pm_runtime_mark_last_busy(adev->ddev->dev);
1057 pm_runtime_put_autosuspend(adev->ddev->dev);
1058 return r;
1059 }
1060
1061 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1062
1063 result += 4;
1064 buf += 4;
1065 *pos += 4;
1066 size -= 4;
1067 }
1068
1069 pm_runtime_mark_last_busy(adev->ddev->dev);
1070 pm_runtime_put_autosuspend(adev->ddev->dev);
1071
1072 return result;
1073}
1074
1075
1076/**
1077 * amdgpu_debugfs_regs_gfxoff_status - read gfxoff status
1078 *
1079 * @f: open file handle
1080 * @buf: User buffer to store read data in
1081 * @size: Number of bytes to read
1082 * @pos: Offset to seek to
1083 */
1084static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1085 size_t size, loff_t *pos)
1086{
1087 struct amdgpu_device *adev = file_inode(f)->i_private;
1088 ssize_t result = 0;
1089 int r;
1090
1091 if (size & 0x3 || *pos & 0x3)
1092 return -EINVAL;
1093
1094 r = pm_runtime_get_sync(adev->ddev->dev);
1095 if (r < 0)
1096 return r;
1097
1098 while (size) {
1099 uint32_t value;
1100
1101 r = amdgpu_get_gfx_off_status(adev, &value);
1102 if (r) {
1103 pm_runtime_mark_last_busy(adev->ddev->dev);
1104 pm_runtime_put_autosuspend(adev->ddev->dev);
1105 return r;
1106 }
1107
1108 r = put_user(value, (uint32_t *)buf);
1109 if (r) {
1110 pm_runtime_mark_last_busy(adev->ddev->dev);
1111 pm_runtime_put_autosuspend(adev->ddev->dev);
1112 return r;
1113 }
1114
1115 result += 4;
1116 buf += 4;
1117 *pos += 4;
1118 size -= 4;
1119 }
1120
1121 pm_runtime_mark_last_busy(adev->ddev->dev);
1122 pm_runtime_put_autosuspend(adev->ddev->dev);
1123
1124 return result;
1125}
1126
1127static const struct file_operations amdgpu_debugfs_regs_fops = {
1128 .owner = THIS_MODULE,
1129 .read = amdgpu_debugfs_regs_read,
1130 .write = amdgpu_debugfs_regs_write,
1131 .llseek = default_llseek
1132};
1133static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1134 .owner = THIS_MODULE,
1135 .read = amdgpu_debugfs_regs_didt_read,
1136 .write = amdgpu_debugfs_regs_didt_write,
1137 .llseek = default_llseek
1138};
1139static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1140 .owner = THIS_MODULE,
1141 .read = amdgpu_debugfs_regs_pcie_read,
1142 .write = amdgpu_debugfs_regs_pcie_write,
1143 .llseek = default_llseek
1144};
1145static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1146 .owner = THIS_MODULE,
1147 .read = amdgpu_debugfs_regs_smc_read,
1148 .write = amdgpu_debugfs_regs_smc_write,
1149 .llseek = default_llseek
1150};
1151
1152static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1153 .owner = THIS_MODULE,
1154 .read = amdgpu_debugfs_gca_config_read,
1155 .llseek = default_llseek
1156};
1157
1158static const struct file_operations amdgpu_debugfs_sensors_fops = {
1159 .owner = THIS_MODULE,
1160 .read = amdgpu_debugfs_sensor_read,
1161 .llseek = default_llseek
1162};
1163
1164static const struct file_operations amdgpu_debugfs_wave_fops = {
1165 .owner = THIS_MODULE,
1166 .read = amdgpu_debugfs_wave_read,
1167 .llseek = default_llseek
1168};
1169static const struct file_operations amdgpu_debugfs_gpr_fops = {
1170 .owner = THIS_MODULE,
1171 .read = amdgpu_debugfs_gpr_read,
1172 .llseek = default_llseek
1173};
1174
1175static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1176 .owner = THIS_MODULE,
1177 .read = amdgpu_debugfs_gfxoff_read,
1178 .write = amdgpu_debugfs_gfxoff_write,
1179 .llseek = default_llseek
1180};
1181
1182static const struct file_operations *debugfs_regs[] = {
1183 &amdgpu_debugfs_regs_fops,
1184 &amdgpu_debugfs_regs_didt_fops,
1185 &amdgpu_debugfs_regs_pcie_fops,
1186 &amdgpu_debugfs_regs_smc_fops,
1187 &amdgpu_debugfs_gca_config_fops,
1188 &amdgpu_debugfs_sensors_fops,
1189 &amdgpu_debugfs_wave_fops,
1190 &amdgpu_debugfs_gpr_fops,
1191 &amdgpu_debugfs_gfxoff_fops,
1192};
1193
1194static const char *debugfs_regs_names[] = {
1195 "amdgpu_regs",
1196 "amdgpu_regs_didt",
1197 "amdgpu_regs_pcie",
1198 "amdgpu_regs_smc",
1199 "amdgpu_gca_config",
1200 "amdgpu_sensors",
1201 "amdgpu_wave",
1202 "amdgpu_gpr",
1203 "amdgpu_gfxoff",
1204};
1205
1206/**
1207 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1208 * register access.
1209 *
1210 * @adev: The device to attach the debugfs entries to
1211 */
1212int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1213{
1214 struct drm_minor *minor = adev->ddev->primary;
1215 struct dentry *ent, *root = minor->debugfs_root;
1216 unsigned int i;
1217
1218 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1219 ent = debugfs_create_file(debugfs_regs_names[i],
1220 S_IFREG | S_IRUGO, root,
1221 adev, debugfs_regs[i]);
1222 if (!i && !IS_ERR_OR_NULL(ent))
1223 i_size_write(ent->d_inode, adev->rmmio_size);
1224 adev->debugfs_regs[i] = ent;
1225 }
1226
1227 return 0;
1228}
1229
1230static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
1231{
1232 struct drm_info_node *node = (struct drm_info_node *) m->private;
1233 struct drm_device *dev = node->minor->dev;
1234 struct amdgpu_device *adev = dev->dev_private;
1235 int r = 0, i;
1236
1237 r = pm_runtime_get_sync(dev->dev);
1238 if (r < 0) {
1239 pm_runtime_put_autosuspend(adev->ddev->dev);
1240 return r;
1241 }
1242
1243 /* Avoid accidently unparking the sched thread during GPU reset */
1244 mutex_lock(&adev->lock_reset);
1245
1246 /* hold on the scheduler */
1247 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1248 struct amdgpu_ring *ring = adev->rings[i];
1249
1250 if (!ring || !ring->sched.thread)
1251 continue;
1252 kthread_park(ring->sched.thread);
1253 }
1254
1255 seq_printf(m, "run ib test:\n");
1256 r = amdgpu_ib_ring_tests(adev);
1257 if (r)
1258 seq_printf(m, "ib ring tests failed (%d).\n", r);
1259 else
1260 seq_printf(m, "ib ring tests passed.\n");
1261
1262 /* go on the scheduler */
1263 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1264 struct amdgpu_ring *ring = adev->rings[i];
1265
1266 if (!ring || !ring->sched.thread)
1267 continue;
1268 kthread_unpark(ring->sched.thread);
1269 }
1270
1271 mutex_unlock(&adev->lock_reset);
1272
1273 pm_runtime_mark_last_busy(dev->dev);
1274 pm_runtime_put_autosuspend(dev->dev);
1275
1276 return 0;
1277}
1278
1279static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
1280{
1281 struct drm_info_node *node = (struct drm_info_node *) m->private;
1282 struct drm_device *dev = node->minor->dev;
1283 struct amdgpu_device *adev = dev->dev_private;
1284
1285 seq_write(m, adev->bios, adev->bios_size);
1286 return 0;
1287}
1288
1289static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data)
1290{
1291 struct drm_info_node *node = (struct drm_info_node *)m->private;
1292 struct drm_device *dev = node->minor->dev;
1293 struct amdgpu_device *adev = dev->dev_private;
1294 int r;
1295
1296 r = pm_runtime_get_sync(dev->dev);
1297 if (r < 0) {
1298 pm_runtime_put_autosuspend(adev->ddev->dev);
1299 return r;
1300 }
1301
1302 seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
1303
1304 pm_runtime_mark_last_busy(dev->dev);
1305 pm_runtime_put_autosuspend(dev->dev);
1306
1307 return 0;
1308}
1309
1310static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data)
1311{
1312 struct drm_info_node *node = (struct drm_info_node *)m->private;
1313 struct drm_device *dev = node->minor->dev;
1314 struct amdgpu_device *adev = dev->dev_private;
1315 int r;
1316
1317 r = pm_runtime_get_sync(dev->dev);
1318 if (r < 0) {
1319 pm_runtime_put_autosuspend(adev->ddev->dev);
1320 return r;
1321 }
1322
1323 seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT));
1324
1325 pm_runtime_mark_last_busy(dev->dev);
1326 pm_runtime_put_autosuspend(dev->dev);
1327
1328 return 0;
1329}
1330
1331static const struct drm_info_list amdgpu_debugfs_list[] = {
1332 {"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump},
1333 {"amdgpu_test_ib", &amdgpu_debugfs_test_ib},
1334 {"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram},
1335 {"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
1336};
1337
1338static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1339 struct dma_fence **fences)
1340{
1341 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1342 uint32_t sync_seq, last_seq;
1343
1344 last_seq = atomic_read(&ring->fence_drv.last_seq);
1345 sync_seq = ring->fence_drv.sync_seq;
1346
1347 last_seq &= drv->num_fences_mask;
1348 sync_seq &= drv->num_fences_mask;
1349
1350 do {
1351 struct dma_fence *fence, **ptr;
1352
1353 ++last_seq;
1354 last_seq &= drv->num_fences_mask;
1355 ptr = &drv->fences[last_seq];
1356
1357 fence = rcu_dereference_protected(*ptr, 1);
1358 RCU_INIT_POINTER(*ptr, NULL);
1359
1360 if (!fence)
1361 continue;
1362
1363 fences[last_seq] = fence;
1364
1365 } while (last_seq != sync_seq);
1366}
1367
1368static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1369 int length)
1370{
1371 int i;
1372 struct dma_fence *fence;
1373
1374 for (i = 0; i < length; i++) {
1375 fence = fences[i];
1376 if (!fence)
1377 continue;
1378 dma_fence_signal(fence);
1379 dma_fence_put(fence);
1380 }
1381}
1382
1383static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1384{
1385 struct drm_sched_job *s_job;
1386 struct dma_fence *fence;
1387
1388 spin_lock(&sched->job_list_lock);
1389 list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1390 fence = sched->ops->run_job(s_job);
1391 dma_fence_put(fence);
1392 }
1393 spin_unlock(&sched->job_list_lock);
1394}
1395
1396static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1397{
1398 struct amdgpu_job *job;
1399 struct drm_sched_job *s_job, *tmp;
1400 uint32_t preempt_seq;
1401 struct dma_fence *fence, **ptr;
1402 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1403 struct drm_gpu_scheduler *sched = &ring->sched;
1404 bool preempted = true;
1405
1406 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1407 return;
1408
1409 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1410 if (preempt_seq <= atomic_read(&drv->last_seq)) {
1411 preempted = false;
1412 goto no_preempt;
1413 }
1414
1415 preempt_seq &= drv->num_fences_mask;
1416 ptr = &drv->fences[preempt_seq];
1417 fence = rcu_dereference_protected(*ptr, 1);
1418
1419no_preempt:
1420 spin_lock(&sched->job_list_lock);
1421 list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
1422 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1423 /* remove job from ring_mirror_list */
1424 list_del_init(&s_job->node);
1425 sched->ops->free_job(s_job);
1426 continue;
1427 }
1428 job = to_amdgpu_job(s_job);
1429 if (preempted && job->fence == fence)
1430 /* mark the job as preempted */
1431 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1432 }
1433 spin_unlock(&sched->job_list_lock);
1434}
1435
1436static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1437{
1438 int r, resched, length;
1439 struct amdgpu_ring *ring;
1440 struct dma_fence **fences = NULL;
1441 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1442
1443 if (val >= AMDGPU_MAX_RINGS)
1444 return -EINVAL;
1445
1446 ring = adev->rings[val];
1447
1448 if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1449 return -EINVAL;
1450
1451 /* the last preemption failed */
1452 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1453 return -EBUSY;
1454
1455 length = ring->fence_drv.num_fences_mask + 1;
1456 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1457 if (!fences)
1458 return -ENOMEM;
1459
1460 /* Avoid accidently unparking the sched thread during GPU reset */
1461 mutex_lock(&adev->lock_reset);
1462
1463 /* stop the scheduler */
1464 kthread_park(ring->sched.thread);
1465
1466 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1467
1468 /* preempt the IB */
1469 r = amdgpu_ring_preempt_ib(ring);
1470 if (r) {
1471 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1472 goto failure;
1473 }
1474
1475 amdgpu_fence_process(ring);
1476
1477 if (atomic_read(&ring->fence_drv.last_seq) !=
1478 ring->fence_drv.sync_seq) {
1479 DRM_INFO("ring %d was preempted\n", ring->idx);
1480
1481 amdgpu_ib_preempt_mark_partial_job(ring);
1482
1483 /* swap out the old fences */
1484 amdgpu_ib_preempt_fences_swap(ring, fences);
1485
1486 amdgpu_fence_driver_force_completion(ring);
1487
1488 /* resubmit unfinished jobs */
1489 amdgpu_ib_preempt_job_recovery(&ring->sched);
1490
1491 /* wait for jobs finished */
1492 amdgpu_fence_wait_empty(ring);
1493
1494 /* signal the old fences */
1495 amdgpu_ib_preempt_signal_fences(fences, length);
1496 }
1497
1498failure:
1499 /* restart the scheduler */
1500 kthread_unpark(ring->sched.thread);
1501
1502 mutex_unlock(&adev->lock_reset);
1503
1504 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1505
1506 kfree(fences);
1507
1508 return 0;
1509}
1510
1511static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1512{
1513 int ret = 0;
1514 uint32_t max_freq, min_freq;
1515 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1516
1517 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1518 return -EINVAL;
1519
1520 ret = pm_runtime_get_sync(adev->ddev->dev);
1521 if (ret < 0) {
1522 pm_runtime_put_autosuspend(adev->ddev->dev);
1523 return ret;
1524 }
1525
1526 if (is_support_sw_smu(adev)) {
1527 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
1528 if (ret || val > max_freq || val < min_freq)
1529 return -EINVAL;
1530 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
1531 } else {
1532 return 0;
1533 }
1534
1535 pm_runtime_mark_last_busy(adev->ddev->dev);
1536 pm_runtime_put_autosuspend(adev->ddev->dev);
1537
1538 if (ret)
1539 return -EINVAL;
1540
1541 return 0;
1542}
1543
1544DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
1545 amdgpu_debugfs_ib_preempt, "%llu\n");
1546
1547DEFINE_SIMPLE_ATTRIBUTE(fops_sclk_set, NULL,
1548 amdgpu_debugfs_sclk_set, "%llu\n");
1549
1550int amdgpu_debugfs_init(struct amdgpu_device *adev)
1551{
1552 int r, i;
1553
1554 adev->debugfs_preempt =
1555 debugfs_create_file("amdgpu_preempt_ib", 0600,
1556 adev->ddev->primary->debugfs_root, adev,
1557 &fops_ib_preempt);
1558 if (!(adev->debugfs_preempt)) {
1559 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1560 return -EIO;
1561 }
1562
1563 adev->smu.debugfs_sclk =
1564 debugfs_create_file("amdgpu_force_sclk", 0200,
1565 adev->ddev->primary->debugfs_root, adev,
1566 &fops_sclk_set);
1567 if (!(adev->smu.debugfs_sclk)) {
1568 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1569 return -EIO;
1570 }
1571
1572 /* Register debugfs entries for amdgpu_ttm */
1573 r = amdgpu_ttm_debugfs_init(adev);
1574 if (r) {
1575 DRM_ERROR("Failed to init debugfs\n");
1576 return r;
1577 }
1578
1579 r = amdgpu_debugfs_pm_init(adev);
1580 if (r) {
1581 DRM_ERROR("Failed to register debugfs file for dpm!\n");
1582 return r;
1583 }
1584
1585 if (amdgpu_debugfs_sa_init(adev)) {
1586 dev_err(adev->dev, "failed to register debugfs file for SA\n");
1587 }
1588
1589 if (amdgpu_debugfs_fence_init(adev))
1590 dev_err(adev->dev, "fence debugfs file creation failed\n");
1591
1592 r = amdgpu_debugfs_gem_init(adev);
1593 if (r)
1594 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1595
1596 r = amdgpu_debugfs_regs_init(adev);
1597 if (r)
1598 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1599
1600 r = amdgpu_debugfs_firmware_init(adev);
1601 if (r)
1602 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
1603
1604#if defined(CONFIG_DRM_AMD_DC)
1605 if (amdgpu_device_has_dc_support(adev)) {
1606 if (dtn_debugfs_init(adev))
1607 DRM_ERROR("amdgpu: failed initialize dtn debugfs support.\n");
1608 }
1609#endif
1610
1611 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1612 struct amdgpu_ring *ring = adev->rings[i];
1613
1614 if (!ring)
1615 continue;
1616
1617 if (amdgpu_debugfs_ring_init(adev, ring)) {
1618 DRM_ERROR("Failed to register debugfs file for rings !\n");
1619 }
1620 }
1621
1622 amdgpu_ras_debugfs_create_all(adev);
1623
1624 amdgpu_debugfs_autodump_init(adev);
1625
1626 return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
1627 ARRAY_SIZE(amdgpu_debugfs_list));
1628}
1629
1630#else
1631int amdgpu_debugfs_init(struct amdgpu_device *adev)
1632{
1633 return 0;
1634}
1635int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1636{
1637 return 0;
1638}
1639#endif