Loading...
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/power_supply.h>
29#include <linux/kthread.h>
30#include <linux/module.h>
31#include <linux/console.h>
32#include <linux/slab.h>
33
34#include <drm/drm_atomic_helper.h>
35#include <drm/drm_probe_helper.h>
36#include <drm/amdgpu_drm.h>
37#include <linux/vgaarb.h>
38#include <linux/vga_switcheroo.h>
39#include <linux/efi.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_i2c.h"
43#include "atom.h"
44#include "amdgpu_atombios.h"
45#include "amdgpu_atomfirmware.h"
46#include "amd_pcie.h"
47#ifdef CONFIG_DRM_AMDGPU_SI
48#include "si.h"
49#endif
50#ifdef CONFIG_DRM_AMDGPU_CIK
51#include "cik.h"
52#endif
53#include "vi.h"
54#include "soc15.h"
55#include "nv.h"
56#include "bif/bif_4_1_d.h"
57#include <linux/pci.h>
58#include <linux/firmware.h>
59#include "amdgpu_vf_error.h"
60
61#include "amdgpu_amdkfd.h"
62#include "amdgpu_pm.h"
63
64#include "amdgpu_xgmi.h"
65#include "amdgpu_ras.h"
66#include "amdgpu_pmu.h"
67#include "amdgpu_fru_eeprom.h"
68#include "amdgpu_reset.h"
69
70#include <linux/suspend.h>
71#include <drm/task_barrier.h>
72#include <linux/pm_runtime.h>
73
74#include <drm/drm_drv.h>
75
76MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
77MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
78MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
79MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
80MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
81MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
82MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
83MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
84MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
85MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
86MODULE_FIRMWARE("amdgpu/vangogh_gpu_info.bin");
87MODULE_FIRMWARE("amdgpu/yellow_carp_gpu_info.bin");
88
89#define AMDGPU_RESUME_MS 2000
90
91const char *amdgpu_asic_name[] = {
92 "TAHITI",
93 "PITCAIRN",
94 "VERDE",
95 "OLAND",
96 "HAINAN",
97 "BONAIRE",
98 "KAVERI",
99 "KABINI",
100 "HAWAII",
101 "MULLINS",
102 "TOPAZ",
103 "TONGA",
104 "FIJI",
105 "CARRIZO",
106 "STONEY",
107 "POLARIS10",
108 "POLARIS11",
109 "POLARIS12",
110 "VEGAM",
111 "VEGA10",
112 "VEGA12",
113 "VEGA20",
114 "RAVEN",
115 "ARCTURUS",
116 "RENOIR",
117 "ALDEBARAN",
118 "NAVI10",
119 "NAVI14",
120 "NAVI12",
121 "SIENNA_CICHLID",
122 "NAVY_FLOUNDER",
123 "VANGOGH",
124 "DIMGREY_CAVEFISH",
125 "BEIGE_GOBY",
126 "YELLOW_CARP",
127 "LAST",
128};
129
130/**
131 * DOC: pcie_replay_count
132 *
133 * The amdgpu driver provides a sysfs API for reporting the total number
134 * of PCIe replays (NAKs)
135 * The file pcie_replay_count is used for this and returns the total
136 * number of replays as a sum of the NAKs generated and NAKs received
137 */
138
139static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
140 struct device_attribute *attr, char *buf)
141{
142 struct drm_device *ddev = dev_get_drvdata(dev);
143 struct amdgpu_device *adev = drm_to_adev(ddev);
144 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
145
146 return sysfs_emit(buf, "%llu\n", cnt);
147}
148
149static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
150 amdgpu_device_get_pcie_replay_count, NULL);
151
152static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
153
154/**
155 * DOC: product_name
156 *
157 * The amdgpu driver provides a sysfs API for reporting the product name
158 * for the device
159 * The file serial_number is used for this and returns the product name
160 * as returned from the FRU.
161 * NOTE: This is only available for certain server cards
162 */
163
164static ssize_t amdgpu_device_get_product_name(struct device *dev,
165 struct device_attribute *attr, char *buf)
166{
167 struct drm_device *ddev = dev_get_drvdata(dev);
168 struct amdgpu_device *adev = drm_to_adev(ddev);
169
170 return sysfs_emit(buf, "%s\n", adev->product_name);
171}
172
173static DEVICE_ATTR(product_name, S_IRUGO,
174 amdgpu_device_get_product_name, NULL);
175
176/**
177 * DOC: product_number
178 *
179 * The amdgpu driver provides a sysfs API for reporting the part number
180 * for the device
181 * The file serial_number is used for this and returns the part number
182 * as returned from the FRU.
183 * NOTE: This is only available for certain server cards
184 */
185
186static ssize_t amdgpu_device_get_product_number(struct device *dev,
187 struct device_attribute *attr, char *buf)
188{
189 struct drm_device *ddev = dev_get_drvdata(dev);
190 struct amdgpu_device *adev = drm_to_adev(ddev);
191
192 return sysfs_emit(buf, "%s\n", adev->product_number);
193}
194
195static DEVICE_ATTR(product_number, S_IRUGO,
196 amdgpu_device_get_product_number, NULL);
197
198/**
199 * DOC: serial_number
200 *
201 * The amdgpu driver provides a sysfs API for reporting the serial number
202 * for the device
203 * The file serial_number is used for this and returns the serial number
204 * as returned from the FRU.
205 * NOTE: This is only available for certain server cards
206 */
207
208static ssize_t amdgpu_device_get_serial_number(struct device *dev,
209 struct device_attribute *attr, char *buf)
210{
211 struct drm_device *ddev = dev_get_drvdata(dev);
212 struct amdgpu_device *adev = drm_to_adev(ddev);
213
214 return sysfs_emit(buf, "%s\n", adev->serial);
215}
216
217static DEVICE_ATTR(serial_number, S_IRUGO,
218 amdgpu_device_get_serial_number, NULL);
219
220/**
221 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
222 *
223 * @dev: drm_device pointer
224 *
225 * Returns true if the device is a dGPU with ATPX power control,
226 * otherwise return false.
227 */
228bool amdgpu_device_supports_px(struct drm_device *dev)
229{
230 struct amdgpu_device *adev = drm_to_adev(dev);
231
232 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
233 return true;
234 return false;
235}
236
237/**
238 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
239 *
240 * @dev: drm_device pointer
241 *
242 * Returns true if the device is a dGPU with ACPI power control,
243 * otherwise return false.
244 */
245bool amdgpu_device_supports_boco(struct drm_device *dev)
246{
247 struct amdgpu_device *adev = drm_to_adev(dev);
248
249 if (adev->has_pr3 ||
250 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
251 return true;
252 return false;
253}
254
255/**
256 * amdgpu_device_supports_baco - Does the device support BACO
257 *
258 * @dev: drm_device pointer
259 *
260 * Returns true if the device supporte BACO,
261 * otherwise return false.
262 */
263bool amdgpu_device_supports_baco(struct drm_device *dev)
264{
265 struct amdgpu_device *adev = drm_to_adev(dev);
266
267 return amdgpu_asic_supports_baco(adev);
268}
269
270/**
271 * amdgpu_device_supports_smart_shift - Is the device dGPU with
272 * smart shift support
273 *
274 * @dev: drm_device pointer
275 *
276 * Returns true if the device is a dGPU with Smart Shift support,
277 * otherwise returns false.
278 */
279bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
280{
281 return (amdgpu_device_supports_boco(dev) &&
282 amdgpu_acpi_is_power_shift_control_supported());
283}
284
285/*
286 * VRAM access helper functions
287 */
288
289/**
290 * amdgpu_device_vram_access - read/write a buffer in vram
291 *
292 * @adev: amdgpu_device pointer
293 * @pos: offset of the buffer in vram
294 * @buf: virtual address of the buffer in system memory
295 * @size: read/write size, sizeof(@buf) must > @size
296 * @write: true - write to vram, otherwise - read from vram
297 */
298void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
299 uint32_t *buf, size_t size, bool write)
300{
301 unsigned long flags;
302 uint32_t hi = ~0;
303 uint64_t last;
304 int idx;
305
306 if (!drm_dev_enter(&adev->ddev, &idx))
307 return;
308
309#ifdef CONFIG_64BIT
310 last = min(pos + size, adev->gmc.visible_vram_size);
311 if (last > pos) {
312 void __iomem *addr = adev->mman.aper_base_kaddr + pos;
313 size_t count = last - pos;
314
315 if (write) {
316 memcpy_toio(addr, buf, count);
317 mb();
318 amdgpu_device_flush_hdp(adev, NULL);
319 } else {
320 amdgpu_device_invalidate_hdp(adev, NULL);
321 mb();
322 memcpy_fromio(buf, addr, count);
323 }
324
325 if (count == size)
326 goto exit;
327
328 pos += count;
329 buf += count / 4;
330 size -= count;
331 }
332#endif
333
334 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
335 for (last = pos + size; pos < last; pos += 4) {
336 uint32_t tmp = pos >> 31;
337
338 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
339 if (tmp != hi) {
340 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
341 hi = tmp;
342 }
343 if (write)
344 WREG32_NO_KIQ(mmMM_DATA, *buf++);
345 else
346 *buf++ = RREG32_NO_KIQ(mmMM_DATA);
347 }
348 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
349
350#ifdef CONFIG_64BIT
351exit:
352#endif
353 drm_dev_exit(idx);
354}
355
356/*
357 * register access helper functions.
358 */
359
360/* Check if hw access should be skipped because of hotplug or device error */
361bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
362{
363 if (adev->no_hw_access)
364 return true;
365
366#ifdef CONFIG_LOCKDEP
367 /*
368 * This is a bit complicated to understand, so worth a comment. What we assert
369 * here is that the GPU reset is not running on another thread in parallel.
370 *
371 * For this we trylock the read side of the reset semaphore, if that succeeds
372 * we know that the reset is not running in paralell.
373 *
374 * If the trylock fails we assert that we are either already holding the read
375 * side of the lock or are the reset thread itself and hold the write side of
376 * the lock.
377 */
378 if (in_task()) {
379 if (down_read_trylock(&adev->reset_sem))
380 up_read(&adev->reset_sem);
381 else
382 lockdep_assert_held(&adev->reset_sem);
383 }
384#endif
385 return false;
386}
387
388/**
389 * amdgpu_device_rreg - read a memory mapped IO or indirect register
390 *
391 * @adev: amdgpu_device pointer
392 * @reg: dword aligned register offset
393 * @acc_flags: access flags which require special behavior
394 *
395 * Returns the 32 bit value from the offset specified.
396 */
397uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
398 uint32_t reg, uint32_t acc_flags)
399{
400 uint32_t ret;
401
402 if (amdgpu_device_skip_hw_access(adev))
403 return 0;
404
405 if ((reg * 4) < adev->rmmio_size) {
406 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
407 amdgpu_sriov_runtime(adev) &&
408 down_read_trylock(&adev->reset_sem)) {
409 ret = amdgpu_kiq_rreg(adev, reg);
410 up_read(&adev->reset_sem);
411 } else {
412 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
413 }
414 } else {
415 ret = adev->pcie_rreg(adev, reg * 4);
416 }
417
418 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
419
420 return ret;
421}
422
423/*
424 * MMIO register read with bytes helper functions
425 * @offset:bytes offset from MMIO start
426 *
427*/
428
429/**
430 * amdgpu_mm_rreg8 - read a memory mapped IO register
431 *
432 * @adev: amdgpu_device pointer
433 * @offset: byte aligned register offset
434 *
435 * Returns the 8 bit value from the offset specified.
436 */
437uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
438{
439 if (amdgpu_device_skip_hw_access(adev))
440 return 0;
441
442 if (offset < adev->rmmio_size)
443 return (readb(adev->rmmio + offset));
444 BUG();
445}
446
447/*
448 * MMIO register write with bytes helper functions
449 * @offset:bytes offset from MMIO start
450 * @value: the value want to be written to the register
451 *
452*/
453/**
454 * amdgpu_mm_wreg8 - read a memory mapped IO register
455 *
456 * @adev: amdgpu_device pointer
457 * @offset: byte aligned register offset
458 * @value: 8 bit value to write
459 *
460 * Writes the value specified to the offset specified.
461 */
462void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
463{
464 if (amdgpu_device_skip_hw_access(adev))
465 return;
466
467 if (offset < adev->rmmio_size)
468 writeb(value, adev->rmmio + offset);
469 else
470 BUG();
471}
472
473/**
474 * amdgpu_device_wreg - write to a memory mapped IO or indirect register
475 *
476 * @adev: amdgpu_device pointer
477 * @reg: dword aligned register offset
478 * @v: 32 bit value to write to the register
479 * @acc_flags: access flags which require special behavior
480 *
481 * Writes the value specified to the offset specified.
482 */
483void amdgpu_device_wreg(struct amdgpu_device *adev,
484 uint32_t reg, uint32_t v,
485 uint32_t acc_flags)
486{
487 if (amdgpu_device_skip_hw_access(adev))
488 return;
489
490 if ((reg * 4) < adev->rmmio_size) {
491 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
492 amdgpu_sriov_runtime(adev) &&
493 down_read_trylock(&adev->reset_sem)) {
494 amdgpu_kiq_wreg(adev, reg, v);
495 up_read(&adev->reset_sem);
496 } else {
497 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
498 }
499 } else {
500 adev->pcie_wreg(adev, reg * 4, v);
501 }
502
503 trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
504}
505
506/*
507 * amdgpu_mm_wreg_mmio_rlc - write register either with mmio or with RLC path if in range
508 *
509 * this function is invoked only the debugfs register access
510 * */
511void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
512 uint32_t reg, uint32_t v)
513{
514 if (amdgpu_device_skip_hw_access(adev))
515 return;
516
517 if (amdgpu_sriov_fullaccess(adev) &&
518 adev->gfx.rlc.funcs &&
519 adev->gfx.rlc.funcs->is_rlcg_access_range) {
520 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
521 return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v, 0, 0);
522 } else {
523 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
524 }
525}
526
527/**
528 * amdgpu_mm_rdoorbell - read a doorbell dword
529 *
530 * @adev: amdgpu_device pointer
531 * @index: doorbell index
532 *
533 * Returns the value in the doorbell aperture at the
534 * requested doorbell index (CIK).
535 */
536u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
537{
538 if (amdgpu_device_skip_hw_access(adev))
539 return 0;
540
541 if (index < adev->doorbell.num_doorbells) {
542 return readl(adev->doorbell.ptr + index);
543 } else {
544 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
545 return 0;
546 }
547}
548
549/**
550 * amdgpu_mm_wdoorbell - write a doorbell dword
551 *
552 * @adev: amdgpu_device pointer
553 * @index: doorbell index
554 * @v: value to write
555 *
556 * Writes @v to the doorbell aperture at the
557 * requested doorbell index (CIK).
558 */
559void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
560{
561 if (amdgpu_device_skip_hw_access(adev))
562 return;
563
564 if (index < adev->doorbell.num_doorbells) {
565 writel(v, adev->doorbell.ptr + index);
566 } else {
567 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
568 }
569}
570
571/**
572 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
573 *
574 * @adev: amdgpu_device pointer
575 * @index: doorbell index
576 *
577 * Returns the value in the doorbell aperture at the
578 * requested doorbell index (VEGA10+).
579 */
580u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
581{
582 if (amdgpu_device_skip_hw_access(adev))
583 return 0;
584
585 if (index < adev->doorbell.num_doorbells) {
586 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
587 } else {
588 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
589 return 0;
590 }
591}
592
593/**
594 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
595 *
596 * @adev: amdgpu_device pointer
597 * @index: doorbell index
598 * @v: value to write
599 *
600 * Writes @v to the doorbell aperture at the
601 * requested doorbell index (VEGA10+).
602 */
603void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
604{
605 if (amdgpu_device_skip_hw_access(adev))
606 return;
607
608 if (index < adev->doorbell.num_doorbells) {
609 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
610 } else {
611 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
612 }
613}
614
615/**
616 * amdgpu_device_indirect_rreg - read an indirect register
617 *
618 * @adev: amdgpu_device pointer
619 * @pcie_index: mmio register offset
620 * @pcie_data: mmio register offset
621 * @reg_addr: indirect register address to read from
622 *
623 * Returns the value of indirect register @reg_addr
624 */
625u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
626 u32 pcie_index, u32 pcie_data,
627 u32 reg_addr)
628{
629 unsigned long flags;
630 u32 r;
631 void __iomem *pcie_index_offset;
632 void __iomem *pcie_data_offset;
633
634 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
635 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
636 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
637
638 writel(reg_addr, pcie_index_offset);
639 readl(pcie_index_offset);
640 r = readl(pcie_data_offset);
641 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
642
643 return r;
644}
645
646/**
647 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
648 *
649 * @adev: amdgpu_device pointer
650 * @pcie_index: mmio register offset
651 * @pcie_data: mmio register offset
652 * @reg_addr: indirect register address to read from
653 *
654 * Returns the value of indirect register @reg_addr
655 */
656u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
657 u32 pcie_index, u32 pcie_data,
658 u32 reg_addr)
659{
660 unsigned long flags;
661 u64 r;
662 void __iomem *pcie_index_offset;
663 void __iomem *pcie_data_offset;
664
665 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
666 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
667 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
668
669 /* read low 32 bits */
670 writel(reg_addr, pcie_index_offset);
671 readl(pcie_index_offset);
672 r = readl(pcie_data_offset);
673 /* read high 32 bits */
674 writel(reg_addr + 4, pcie_index_offset);
675 readl(pcie_index_offset);
676 r |= ((u64)readl(pcie_data_offset) << 32);
677 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
678
679 return r;
680}
681
682/**
683 * amdgpu_device_indirect_wreg - write an indirect register address
684 *
685 * @adev: amdgpu_device pointer
686 * @pcie_index: mmio register offset
687 * @pcie_data: mmio register offset
688 * @reg_addr: indirect register offset
689 * @reg_data: indirect register data
690 *
691 */
692void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
693 u32 pcie_index, u32 pcie_data,
694 u32 reg_addr, u32 reg_data)
695{
696 unsigned long flags;
697 void __iomem *pcie_index_offset;
698 void __iomem *pcie_data_offset;
699
700 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
701 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
702 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
703
704 writel(reg_addr, pcie_index_offset);
705 readl(pcie_index_offset);
706 writel(reg_data, pcie_data_offset);
707 readl(pcie_data_offset);
708 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
709}
710
711/**
712 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
713 *
714 * @adev: amdgpu_device pointer
715 * @pcie_index: mmio register offset
716 * @pcie_data: mmio register offset
717 * @reg_addr: indirect register offset
718 * @reg_data: indirect register data
719 *
720 */
721void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
722 u32 pcie_index, u32 pcie_data,
723 u32 reg_addr, u64 reg_data)
724{
725 unsigned long flags;
726 void __iomem *pcie_index_offset;
727 void __iomem *pcie_data_offset;
728
729 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
730 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
731 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
732
733 /* write low 32 bits */
734 writel(reg_addr, pcie_index_offset);
735 readl(pcie_index_offset);
736 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
737 readl(pcie_data_offset);
738 /* write high 32 bits */
739 writel(reg_addr + 4, pcie_index_offset);
740 readl(pcie_index_offset);
741 writel((u32)(reg_data >> 32), pcie_data_offset);
742 readl(pcie_data_offset);
743 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
744}
745
746/**
747 * amdgpu_invalid_rreg - dummy reg read function
748 *
749 * @adev: amdgpu_device pointer
750 * @reg: offset of register
751 *
752 * Dummy register read function. Used for register blocks
753 * that certain asics don't have (all asics).
754 * Returns the value in the register.
755 */
756static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
757{
758 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
759 BUG();
760 return 0;
761}
762
763/**
764 * amdgpu_invalid_wreg - dummy reg write function
765 *
766 * @adev: amdgpu_device pointer
767 * @reg: offset of register
768 * @v: value to write to the register
769 *
770 * Dummy register read function. Used for register blocks
771 * that certain asics don't have (all asics).
772 */
773static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
774{
775 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
776 reg, v);
777 BUG();
778}
779
780/**
781 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
782 *
783 * @adev: amdgpu_device pointer
784 * @reg: offset of register
785 *
786 * Dummy register read function. Used for register blocks
787 * that certain asics don't have (all asics).
788 * Returns the value in the register.
789 */
790static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
791{
792 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
793 BUG();
794 return 0;
795}
796
797/**
798 * amdgpu_invalid_wreg64 - dummy reg write function
799 *
800 * @adev: amdgpu_device pointer
801 * @reg: offset of register
802 * @v: value to write to the register
803 *
804 * Dummy register read function. Used for register blocks
805 * that certain asics don't have (all asics).
806 */
807static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
808{
809 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
810 reg, v);
811 BUG();
812}
813
814/**
815 * amdgpu_block_invalid_rreg - dummy reg read function
816 *
817 * @adev: amdgpu_device pointer
818 * @block: offset of instance
819 * @reg: offset of register
820 *
821 * Dummy register read function. Used for register blocks
822 * that certain asics don't have (all asics).
823 * Returns the value in the register.
824 */
825static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
826 uint32_t block, uint32_t reg)
827{
828 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
829 reg, block);
830 BUG();
831 return 0;
832}
833
834/**
835 * amdgpu_block_invalid_wreg - dummy reg write function
836 *
837 * @adev: amdgpu_device pointer
838 * @block: offset of instance
839 * @reg: offset of register
840 * @v: value to write to the register
841 *
842 * Dummy register read function. Used for register blocks
843 * that certain asics don't have (all asics).
844 */
845static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
846 uint32_t block,
847 uint32_t reg, uint32_t v)
848{
849 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
850 reg, block, v);
851 BUG();
852}
853
854/**
855 * amdgpu_device_asic_init - Wrapper for atom asic_init
856 *
857 * @adev: amdgpu_device pointer
858 *
859 * Does any asic specific work and then calls atom asic init.
860 */
861static int amdgpu_device_asic_init(struct amdgpu_device *adev)
862{
863 amdgpu_asic_pre_asic_init(adev);
864
865 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
866}
867
868/**
869 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
870 *
871 * @adev: amdgpu_device pointer
872 *
873 * Allocates a scratch page of VRAM for use by various things in the
874 * driver.
875 */
876static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
877{
878 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
879 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
880 &adev->vram_scratch.robj,
881 &adev->vram_scratch.gpu_addr,
882 (void **)&adev->vram_scratch.ptr);
883}
884
885/**
886 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
887 *
888 * @adev: amdgpu_device pointer
889 *
890 * Frees the VRAM scratch page.
891 */
892static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
893{
894 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
895}
896
897/**
898 * amdgpu_device_program_register_sequence - program an array of registers.
899 *
900 * @adev: amdgpu_device pointer
901 * @registers: pointer to the register array
902 * @array_size: size of the register array
903 *
904 * Programs an array or registers with and and or masks.
905 * This is a helper for setting golden registers.
906 */
907void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
908 const u32 *registers,
909 const u32 array_size)
910{
911 u32 tmp, reg, and_mask, or_mask;
912 int i;
913
914 if (array_size % 3)
915 return;
916
917 for (i = 0; i < array_size; i +=3) {
918 reg = registers[i + 0];
919 and_mask = registers[i + 1];
920 or_mask = registers[i + 2];
921
922 if (and_mask == 0xffffffff) {
923 tmp = or_mask;
924 } else {
925 tmp = RREG32(reg);
926 tmp &= ~and_mask;
927 if (adev->family >= AMDGPU_FAMILY_AI)
928 tmp |= (or_mask & and_mask);
929 else
930 tmp |= or_mask;
931 }
932 WREG32(reg, tmp);
933 }
934}
935
936/**
937 * amdgpu_device_pci_config_reset - reset the GPU
938 *
939 * @adev: amdgpu_device pointer
940 *
941 * Resets the GPU using the pci config reset sequence.
942 * Only applicable to asics prior to vega10.
943 */
944void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
945{
946 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
947}
948
949/**
950 * amdgpu_device_pci_reset - reset the GPU using generic PCI means
951 *
952 * @adev: amdgpu_device pointer
953 *
954 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
955 */
956int amdgpu_device_pci_reset(struct amdgpu_device *adev)
957{
958 return pci_reset_function(adev->pdev);
959}
960
961/*
962 * GPU doorbell aperture helpers function.
963 */
964/**
965 * amdgpu_device_doorbell_init - Init doorbell driver information.
966 *
967 * @adev: amdgpu_device pointer
968 *
969 * Init doorbell driver information (CIK)
970 * Returns 0 on success, error on failure.
971 */
972static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
973{
974
975 /* No doorbell on SI hardware generation */
976 if (adev->asic_type < CHIP_BONAIRE) {
977 adev->doorbell.base = 0;
978 adev->doorbell.size = 0;
979 adev->doorbell.num_doorbells = 0;
980 adev->doorbell.ptr = NULL;
981 return 0;
982 }
983
984 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
985 return -EINVAL;
986
987 amdgpu_asic_init_doorbell_index(adev);
988
989 /* doorbell bar mapping */
990 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
991 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
992
993 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
994 adev->doorbell_index.max_assignment+1);
995 if (adev->doorbell.num_doorbells == 0)
996 return -EINVAL;
997
998 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
999 * paging queue doorbell use the second page. The
1000 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1001 * doorbells are in the first page. So with paging queue enabled,
1002 * the max num_doorbells should + 1 page (0x400 in dword)
1003 */
1004 if (adev->asic_type >= CHIP_VEGA10)
1005 adev->doorbell.num_doorbells += 0x400;
1006
1007 adev->doorbell.ptr = ioremap(adev->doorbell.base,
1008 adev->doorbell.num_doorbells *
1009 sizeof(u32));
1010 if (adev->doorbell.ptr == NULL)
1011 return -ENOMEM;
1012
1013 return 0;
1014}
1015
1016/**
1017 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1018 *
1019 * @adev: amdgpu_device pointer
1020 *
1021 * Tear down doorbell driver information (CIK)
1022 */
1023static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1024{
1025 iounmap(adev->doorbell.ptr);
1026 adev->doorbell.ptr = NULL;
1027}
1028
1029
1030
1031/*
1032 * amdgpu_device_wb_*()
1033 * Writeback is the method by which the GPU updates special pages in memory
1034 * with the status of certain GPU events (fences, ring pointers,etc.).
1035 */
1036
1037/**
1038 * amdgpu_device_wb_fini - Disable Writeback and free memory
1039 *
1040 * @adev: amdgpu_device pointer
1041 *
1042 * Disables Writeback and frees the Writeback memory (all asics).
1043 * Used at driver shutdown.
1044 */
1045static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1046{
1047 if (adev->wb.wb_obj) {
1048 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1049 &adev->wb.gpu_addr,
1050 (void **)&adev->wb.wb);
1051 adev->wb.wb_obj = NULL;
1052 }
1053}
1054
1055/**
1056 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
1057 *
1058 * @adev: amdgpu_device pointer
1059 *
1060 * Initializes writeback and allocates writeback memory (all asics).
1061 * Used at driver startup.
1062 * Returns 0 on success or an -error on failure.
1063 */
1064static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1065{
1066 int r;
1067
1068 if (adev->wb.wb_obj == NULL) {
1069 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1070 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1071 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1072 &adev->wb.wb_obj, &adev->wb.gpu_addr,
1073 (void **)&adev->wb.wb);
1074 if (r) {
1075 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1076 return r;
1077 }
1078
1079 adev->wb.num_wb = AMDGPU_MAX_WB;
1080 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1081
1082 /* clear wb memory */
1083 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1084 }
1085
1086 return 0;
1087}
1088
1089/**
1090 * amdgpu_device_wb_get - Allocate a wb entry
1091 *
1092 * @adev: amdgpu_device pointer
1093 * @wb: wb index
1094 *
1095 * Allocate a wb slot for use by the driver (all asics).
1096 * Returns 0 on success or -EINVAL on failure.
1097 */
1098int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1099{
1100 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1101
1102 if (offset < adev->wb.num_wb) {
1103 __set_bit(offset, adev->wb.used);
1104 *wb = offset << 3; /* convert to dw offset */
1105 return 0;
1106 } else {
1107 return -EINVAL;
1108 }
1109}
1110
1111/**
1112 * amdgpu_device_wb_free - Free a wb entry
1113 *
1114 * @adev: amdgpu_device pointer
1115 * @wb: wb index
1116 *
1117 * Free a wb slot allocated for use by the driver (all asics)
1118 */
1119void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1120{
1121 wb >>= 3;
1122 if (wb < adev->wb.num_wb)
1123 __clear_bit(wb, adev->wb.used);
1124}
1125
1126/**
1127 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1128 *
1129 * @adev: amdgpu_device pointer
1130 *
1131 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1132 * to fail, but if any of the BARs is not accessible after the size we abort
1133 * driver loading by returning -ENODEV.
1134 */
1135int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1136{
1137 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1138 struct pci_bus *root;
1139 struct resource *res;
1140 unsigned i;
1141 u16 cmd;
1142 int r;
1143
1144 /* Bypass for VF */
1145 if (amdgpu_sriov_vf(adev))
1146 return 0;
1147
1148 /* skip if the bios has already enabled large BAR */
1149 if (adev->gmc.real_vram_size &&
1150 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1151 return 0;
1152
1153 /* Check if the root BUS has 64bit memory resources */
1154 root = adev->pdev->bus;
1155 while (root->parent)
1156 root = root->parent;
1157
1158 pci_bus_for_each_resource(root, res, i) {
1159 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1160 res->start > 0x100000000ull)
1161 break;
1162 }
1163
1164 /* Trying to resize is pointless without a root hub window above 4GB */
1165 if (!res)
1166 return 0;
1167
1168 /* Limit the BAR size to what is available */
1169 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1170 rbar_size);
1171
1172 /* Disable memory decoding while we change the BAR addresses and size */
1173 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1174 pci_write_config_word(adev->pdev, PCI_COMMAND,
1175 cmd & ~PCI_COMMAND_MEMORY);
1176
1177 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1178 amdgpu_device_doorbell_fini(adev);
1179 if (adev->asic_type >= CHIP_BONAIRE)
1180 pci_release_resource(adev->pdev, 2);
1181
1182 pci_release_resource(adev->pdev, 0);
1183
1184 r = pci_resize_resource(adev->pdev, 0, rbar_size);
1185 if (r == -ENOSPC)
1186 DRM_INFO("Not enough PCI address space for a large BAR.");
1187 else if (r && r != -ENOTSUPP)
1188 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1189
1190 pci_assign_unassigned_bus_resources(adev->pdev->bus);
1191
1192 /* When the doorbell or fb BAR isn't available we have no chance of
1193 * using the device.
1194 */
1195 r = amdgpu_device_doorbell_init(adev);
1196 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1197 return -ENODEV;
1198
1199 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1200
1201 return 0;
1202}
1203
1204/*
1205 * GPU helpers function.
1206 */
1207/**
1208 * amdgpu_device_need_post - check if the hw need post or not
1209 *
1210 * @adev: amdgpu_device pointer
1211 *
1212 * Check if the asic has been initialized (all asics) at driver startup
1213 * or post is needed if hw reset is performed.
1214 * Returns true if need or false if not.
1215 */
1216bool amdgpu_device_need_post(struct amdgpu_device *adev)
1217{
1218 uint32_t reg;
1219
1220 if (amdgpu_sriov_vf(adev))
1221 return false;
1222
1223 if (amdgpu_passthrough(adev)) {
1224 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1225 * some old smc fw still need driver do vPost otherwise gpu hang, while
1226 * those smc fw version above 22.15 doesn't have this flaw, so we force
1227 * vpost executed for smc version below 22.15
1228 */
1229 if (adev->asic_type == CHIP_FIJI) {
1230 int err;
1231 uint32_t fw_ver;
1232 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1233 /* force vPost if error occured */
1234 if (err)
1235 return true;
1236
1237 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1238 if (fw_ver < 0x00160e00)
1239 return true;
1240 }
1241 }
1242
1243 /* Don't post if we need to reset whole hive on init */
1244 if (adev->gmc.xgmi.pending_reset)
1245 return false;
1246
1247 if (adev->has_hw_reset) {
1248 adev->has_hw_reset = false;
1249 return true;
1250 }
1251
1252 /* bios scratch used on CIK+ */
1253 if (adev->asic_type >= CHIP_BONAIRE)
1254 return amdgpu_atombios_scratch_need_asic_init(adev);
1255
1256 /* check MEM_SIZE for older asics */
1257 reg = amdgpu_asic_get_config_memsize(adev);
1258
1259 if ((reg != 0) && (reg != 0xffffffff))
1260 return false;
1261
1262 return true;
1263}
1264
1265/* if we get transitioned to only one device, take VGA back */
1266/**
1267 * amdgpu_device_vga_set_decode - enable/disable vga decode
1268 *
1269 * @cookie: amdgpu_device pointer
1270 * @state: enable/disable vga decode
1271 *
1272 * Enable/disable vga decode (all asics).
1273 * Returns VGA resource flags.
1274 */
1275static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
1276{
1277 struct amdgpu_device *adev = cookie;
1278 amdgpu_asic_set_vga_state(adev, state);
1279 if (state)
1280 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1281 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1282 else
1283 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1284}
1285
1286/**
1287 * amdgpu_device_check_block_size - validate the vm block size
1288 *
1289 * @adev: amdgpu_device pointer
1290 *
1291 * Validates the vm block size specified via module parameter.
1292 * The vm block size defines number of bits in page table versus page directory,
1293 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1294 * page table and the remaining bits are in the page directory.
1295 */
1296static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1297{
1298 /* defines number of bits in page table versus page directory,
1299 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1300 * page table and the remaining bits are in the page directory */
1301 if (amdgpu_vm_block_size == -1)
1302 return;
1303
1304 if (amdgpu_vm_block_size < 9) {
1305 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1306 amdgpu_vm_block_size);
1307 amdgpu_vm_block_size = -1;
1308 }
1309}
1310
1311/**
1312 * amdgpu_device_check_vm_size - validate the vm size
1313 *
1314 * @adev: amdgpu_device pointer
1315 *
1316 * Validates the vm size in GB specified via module parameter.
1317 * The VM size is the size of the GPU virtual memory space in GB.
1318 */
1319static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1320{
1321 /* no need to check the default value */
1322 if (amdgpu_vm_size == -1)
1323 return;
1324
1325 if (amdgpu_vm_size < 1) {
1326 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1327 amdgpu_vm_size);
1328 amdgpu_vm_size = -1;
1329 }
1330}
1331
1332static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1333{
1334 struct sysinfo si;
1335 bool is_os_64 = (sizeof(void *) == 8);
1336 uint64_t total_memory;
1337 uint64_t dram_size_seven_GB = 0x1B8000000;
1338 uint64_t dram_size_three_GB = 0xB8000000;
1339
1340 if (amdgpu_smu_memory_pool_size == 0)
1341 return;
1342
1343 if (!is_os_64) {
1344 DRM_WARN("Not 64-bit OS, feature not supported\n");
1345 goto def_value;
1346 }
1347 si_meminfo(&si);
1348 total_memory = (uint64_t)si.totalram * si.mem_unit;
1349
1350 if ((amdgpu_smu_memory_pool_size == 1) ||
1351 (amdgpu_smu_memory_pool_size == 2)) {
1352 if (total_memory < dram_size_three_GB)
1353 goto def_value1;
1354 } else if ((amdgpu_smu_memory_pool_size == 4) ||
1355 (amdgpu_smu_memory_pool_size == 8)) {
1356 if (total_memory < dram_size_seven_GB)
1357 goto def_value1;
1358 } else {
1359 DRM_WARN("Smu memory pool size not supported\n");
1360 goto def_value;
1361 }
1362 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1363
1364 return;
1365
1366def_value1:
1367 DRM_WARN("No enough system memory\n");
1368def_value:
1369 adev->pm.smu_prv_buffer_size = 0;
1370}
1371
1372static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1373{
1374 if (!(adev->flags & AMD_IS_APU) ||
1375 adev->asic_type < CHIP_RAVEN)
1376 return 0;
1377
1378 switch (adev->asic_type) {
1379 case CHIP_RAVEN:
1380 if (adev->pdev->device == 0x15dd)
1381 adev->apu_flags |= AMD_APU_IS_RAVEN;
1382 if (adev->pdev->device == 0x15d8)
1383 adev->apu_flags |= AMD_APU_IS_PICASSO;
1384 break;
1385 case CHIP_RENOIR:
1386 if ((adev->pdev->device == 0x1636) ||
1387 (adev->pdev->device == 0x164c))
1388 adev->apu_flags |= AMD_APU_IS_RENOIR;
1389 else
1390 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1391 break;
1392 case CHIP_VANGOGH:
1393 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1394 break;
1395 case CHIP_YELLOW_CARP:
1396 break;
1397 default:
1398 return -EINVAL;
1399 }
1400
1401 return 0;
1402}
1403
1404/**
1405 * amdgpu_device_check_arguments - validate module params
1406 *
1407 * @adev: amdgpu_device pointer
1408 *
1409 * Validates certain module parameters and updates
1410 * the associated values used by the driver (all asics).
1411 */
1412static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1413{
1414 if (amdgpu_sched_jobs < 4) {
1415 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1416 amdgpu_sched_jobs);
1417 amdgpu_sched_jobs = 4;
1418 } else if (!is_power_of_2(amdgpu_sched_jobs)){
1419 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1420 amdgpu_sched_jobs);
1421 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1422 }
1423
1424 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1425 /* gart size must be greater or equal to 32M */
1426 dev_warn(adev->dev, "gart size (%d) too small\n",
1427 amdgpu_gart_size);
1428 amdgpu_gart_size = -1;
1429 }
1430
1431 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1432 /* gtt size must be greater or equal to 32M */
1433 dev_warn(adev->dev, "gtt size (%d) too small\n",
1434 amdgpu_gtt_size);
1435 amdgpu_gtt_size = -1;
1436 }
1437
1438 /* valid range is between 4 and 9 inclusive */
1439 if (amdgpu_vm_fragment_size != -1 &&
1440 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1441 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1442 amdgpu_vm_fragment_size = -1;
1443 }
1444
1445 if (amdgpu_sched_hw_submission < 2) {
1446 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1447 amdgpu_sched_hw_submission);
1448 amdgpu_sched_hw_submission = 2;
1449 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1450 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1451 amdgpu_sched_hw_submission);
1452 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1453 }
1454
1455 amdgpu_device_check_smu_prv_buffer_size(adev);
1456
1457 amdgpu_device_check_vm_size(adev);
1458
1459 amdgpu_device_check_block_size(adev);
1460
1461 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1462
1463 amdgpu_gmc_tmz_set(adev);
1464
1465 amdgpu_gmc_noretry_set(adev);
1466
1467 return 0;
1468}
1469
1470/**
1471 * amdgpu_switcheroo_set_state - set switcheroo state
1472 *
1473 * @pdev: pci dev pointer
1474 * @state: vga_switcheroo state
1475 *
1476 * Callback for the switcheroo driver. Suspends or resumes the
1477 * the asics before or after it is powered up using ACPI methods.
1478 */
1479static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1480 enum vga_switcheroo_state state)
1481{
1482 struct drm_device *dev = pci_get_drvdata(pdev);
1483 int r;
1484
1485 if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1486 return;
1487
1488 if (state == VGA_SWITCHEROO_ON) {
1489 pr_info("switched on\n");
1490 /* don't suspend or resume card normally */
1491 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1492
1493 pci_set_power_state(pdev, PCI_D0);
1494 amdgpu_device_load_pci_state(pdev);
1495 r = pci_enable_device(pdev);
1496 if (r)
1497 DRM_WARN("pci_enable_device failed (%d)\n", r);
1498 amdgpu_device_resume(dev, true);
1499
1500 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1501 } else {
1502 pr_info("switched off\n");
1503 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1504 amdgpu_device_suspend(dev, true);
1505 amdgpu_device_cache_pci_state(pdev);
1506 /* Shut down the device */
1507 pci_disable_device(pdev);
1508 pci_set_power_state(pdev, PCI_D3cold);
1509 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1510 }
1511}
1512
1513/**
1514 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1515 *
1516 * @pdev: pci dev pointer
1517 *
1518 * Callback for the switcheroo driver. Check of the switcheroo
1519 * state can be changed.
1520 * Returns true if the state can be changed, false if not.
1521 */
1522static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1523{
1524 struct drm_device *dev = pci_get_drvdata(pdev);
1525
1526 /*
1527 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1528 * locking inversion with the driver load path. And the access here is
1529 * completely racy anyway. So don't bother with locking for now.
1530 */
1531 return atomic_read(&dev->open_count) == 0;
1532}
1533
1534static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1535 .set_gpu_state = amdgpu_switcheroo_set_state,
1536 .reprobe = NULL,
1537 .can_switch = amdgpu_switcheroo_can_switch,
1538};
1539
1540/**
1541 * amdgpu_device_ip_set_clockgating_state - set the CG state
1542 *
1543 * @dev: amdgpu_device pointer
1544 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1545 * @state: clockgating state (gate or ungate)
1546 *
1547 * Sets the requested clockgating state for all instances of
1548 * the hardware IP specified.
1549 * Returns the error code from the last instance.
1550 */
1551int amdgpu_device_ip_set_clockgating_state(void *dev,
1552 enum amd_ip_block_type block_type,
1553 enum amd_clockgating_state state)
1554{
1555 struct amdgpu_device *adev = dev;
1556 int i, r = 0;
1557
1558 for (i = 0; i < adev->num_ip_blocks; i++) {
1559 if (!adev->ip_blocks[i].status.valid)
1560 continue;
1561 if (adev->ip_blocks[i].version->type != block_type)
1562 continue;
1563 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1564 continue;
1565 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1566 (void *)adev, state);
1567 if (r)
1568 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1569 adev->ip_blocks[i].version->funcs->name, r);
1570 }
1571 return r;
1572}
1573
1574/**
1575 * amdgpu_device_ip_set_powergating_state - set the PG state
1576 *
1577 * @dev: amdgpu_device pointer
1578 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1579 * @state: powergating state (gate or ungate)
1580 *
1581 * Sets the requested powergating state for all instances of
1582 * the hardware IP specified.
1583 * Returns the error code from the last instance.
1584 */
1585int amdgpu_device_ip_set_powergating_state(void *dev,
1586 enum amd_ip_block_type block_type,
1587 enum amd_powergating_state state)
1588{
1589 struct amdgpu_device *adev = dev;
1590 int i, r = 0;
1591
1592 for (i = 0; i < adev->num_ip_blocks; i++) {
1593 if (!adev->ip_blocks[i].status.valid)
1594 continue;
1595 if (adev->ip_blocks[i].version->type != block_type)
1596 continue;
1597 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1598 continue;
1599 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1600 (void *)adev, state);
1601 if (r)
1602 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1603 adev->ip_blocks[i].version->funcs->name, r);
1604 }
1605 return r;
1606}
1607
1608/**
1609 * amdgpu_device_ip_get_clockgating_state - get the CG state
1610 *
1611 * @adev: amdgpu_device pointer
1612 * @flags: clockgating feature flags
1613 *
1614 * Walks the list of IPs on the device and updates the clockgating
1615 * flags for each IP.
1616 * Updates @flags with the feature flags for each hardware IP where
1617 * clockgating is enabled.
1618 */
1619void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1620 u32 *flags)
1621{
1622 int i;
1623
1624 for (i = 0; i < adev->num_ip_blocks; i++) {
1625 if (!adev->ip_blocks[i].status.valid)
1626 continue;
1627 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1628 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1629 }
1630}
1631
1632/**
1633 * amdgpu_device_ip_wait_for_idle - wait for idle
1634 *
1635 * @adev: amdgpu_device pointer
1636 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1637 *
1638 * Waits for the request hardware IP to be idle.
1639 * Returns 0 for success or a negative error code on failure.
1640 */
1641int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1642 enum amd_ip_block_type block_type)
1643{
1644 int i, r;
1645
1646 for (i = 0; i < adev->num_ip_blocks; i++) {
1647 if (!adev->ip_blocks[i].status.valid)
1648 continue;
1649 if (adev->ip_blocks[i].version->type == block_type) {
1650 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1651 if (r)
1652 return r;
1653 break;
1654 }
1655 }
1656 return 0;
1657
1658}
1659
1660/**
1661 * amdgpu_device_ip_is_idle - is the hardware IP idle
1662 *
1663 * @adev: amdgpu_device pointer
1664 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1665 *
1666 * Check if the hardware IP is idle or not.
1667 * Returns true if it the IP is idle, false if not.
1668 */
1669bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1670 enum amd_ip_block_type block_type)
1671{
1672 int i;
1673
1674 for (i = 0; i < adev->num_ip_blocks; i++) {
1675 if (!adev->ip_blocks[i].status.valid)
1676 continue;
1677 if (adev->ip_blocks[i].version->type == block_type)
1678 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1679 }
1680 return true;
1681
1682}
1683
1684/**
1685 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1686 *
1687 * @adev: amdgpu_device pointer
1688 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1689 *
1690 * Returns a pointer to the hardware IP block structure
1691 * if it exists for the asic, otherwise NULL.
1692 */
1693struct amdgpu_ip_block *
1694amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1695 enum amd_ip_block_type type)
1696{
1697 int i;
1698
1699 for (i = 0; i < adev->num_ip_blocks; i++)
1700 if (adev->ip_blocks[i].version->type == type)
1701 return &adev->ip_blocks[i];
1702
1703 return NULL;
1704}
1705
1706/**
1707 * amdgpu_device_ip_block_version_cmp
1708 *
1709 * @adev: amdgpu_device pointer
1710 * @type: enum amd_ip_block_type
1711 * @major: major version
1712 * @minor: minor version
1713 *
1714 * return 0 if equal or greater
1715 * return 1 if smaller or the ip_block doesn't exist
1716 */
1717int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1718 enum amd_ip_block_type type,
1719 u32 major, u32 minor)
1720{
1721 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1722
1723 if (ip_block && ((ip_block->version->major > major) ||
1724 ((ip_block->version->major == major) &&
1725 (ip_block->version->minor >= minor))))
1726 return 0;
1727
1728 return 1;
1729}
1730
1731/**
1732 * amdgpu_device_ip_block_add
1733 *
1734 * @adev: amdgpu_device pointer
1735 * @ip_block_version: pointer to the IP to add
1736 *
1737 * Adds the IP block driver information to the collection of IPs
1738 * on the asic.
1739 */
1740int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1741 const struct amdgpu_ip_block_version *ip_block_version)
1742{
1743 if (!ip_block_version)
1744 return -EINVAL;
1745
1746 switch (ip_block_version->type) {
1747 case AMD_IP_BLOCK_TYPE_VCN:
1748 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1749 return 0;
1750 break;
1751 case AMD_IP_BLOCK_TYPE_JPEG:
1752 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1753 return 0;
1754 break;
1755 default:
1756 break;
1757 }
1758
1759 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1760 ip_block_version->funcs->name);
1761
1762 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1763
1764 return 0;
1765}
1766
1767/**
1768 * amdgpu_device_enable_virtual_display - enable virtual display feature
1769 *
1770 * @adev: amdgpu_device pointer
1771 *
1772 * Enabled the virtual display feature if the user has enabled it via
1773 * the module parameter virtual_display. This feature provides a virtual
1774 * display hardware on headless boards or in virtualized environments.
1775 * This function parses and validates the configuration string specified by
1776 * the user and configues the virtual display configuration (number of
1777 * virtual connectors, crtcs, etc.) specified.
1778 */
1779static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1780{
1781 adev->enable_virtual_display = false;
1782
1783 if (amdgpu_virtual_display) {
1784 const char *pci_address_name = pci_name(adev->pdev);
1785 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1786
1787 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1788 pciaddstr_tmp = pciaddstr;
1789 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1790 pciaddname = strsep(&pciaddname_tmp, ",");
1791 if (!strcmp("all", pciaddname)
1792 || !strcmp(pci_address_name, pciaddname)) {
1793 long num_crtc;
1794 int res = -1;
1795
1796 adev->enable_virtual_display = true;
1797
1798 if (pciaddname_tmp)
1799 res = kstrtol(pciaddname_tmp, 10,
1800 &num_crtc);
1801
1802 if (!res) {
1803 if (num_crtc < 1)
1804 num_crtc = 1;
1805 if (num_crtc > 6)
1806 num_crtc = 6;
1807 adev->mode_info.num_crtc = num_crtc;
1808 } else {
1809 adev->mode_info.num_crtc = 1;
1810 }
1811 break;
1812 }
1813 }
1814
1815 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1816 amdgpu_virtual_display, pci_address_name,
1817 adev->enable_virtual_display, adev->mode_info.num_crtc);
1818
1819 kfree(pciaddstr);
1820 }
1821}
1822
1823/**
1824 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1825 *
1826 * @adev: amdgpu_device pointer
1827 *
1828 * Parses the asic configuration parameters specified in the gpu info
1829 * firmware and makes them availale to the driver for use in configuring
1830 * the asic.
1831 * Returns 0 on success, -EINVAL on failure.
1832 */
1833static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1834{
1835 const char *chip_name;
1836 char fw_name[40];
1837 int err;
1838 const struct gpu_info_firmware_header_v1_0 *hdr;
1839
1840 adev->firmware.gpu_info_fw = NULL;
1841
1842 if (adev->mman.discovery_bin) {
1843 amdgpu_discovery_get_gfx_info(adev);
1844
1845 /*
1846 * FIXME: The bounding box is still needed by Navi12, so
1847 * temporarily read it from gpu_info firmware. Should be droped
1848 * when DAL no longer needs it.
1849 */
1850 if (adev->asic_type != CHIP_NAVI12)
1851 return 0;
1852 }
1853
1854 switch (adev->asic_type) {
1855#ifdef CONFIG_DRM_AMDGPU_SI
1856 case CHIP_VERDE:
1857 case CHIP_TAHITI:
1858 case CHIP_PITCAIRN:
1859 case CHIP_OLAND:
1860 case CHIP_HAINAN:
1861#endif
1862#ifdef CONFIG_DRM_AMDGPU_CIK
1863 case CHIP_BONAIRE:
1864 case CHIP_HAWAII:
1865 case CHIP_KAVERI:
1866 case CHIP_KABINI:
1867 case CHIP_MULLINS:
1868#endif
1869 case CHIP_TOPAZ:
1870 case CHIP_TONGA:
1871 case CHIP_FIJI:
1872 case CHIP_POLARIS10:
1873 case CHIP_POLARIS11:
1874 case CHIP_POLARIS12:
1875 case CHIP_VEGAM:
1876 case CHIP_CARRIZO:
1877 case CHIP_STONEY:
1878 case CHIP_VEGA20:
1879 case CHIP_ALDEBARAN:
1880 case CHIP_SIENNA_CICHLID:
1881 case CHIP_NAVY_FLOUNDER:
1882 case CHIP_DIMGREY_CAVEFISH:
1883 case CHIP_BEIGE_GOBY:
1884 default:
1885 return 0;
1886 case CHIP_VEGA10:
1887 chip_name = "vega10";
1888 break;
1889 case CHIP_VEGA12:
1890 chip_name = "vega12";
1891 break;
1892 case CHIP_RAVEN:
1893 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1894 chip_name = "raven2";
1895 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1896 chip_name = "picasso";
1897 else
1898 chip_name = "raven";
1899 break;
1900 case CHIP_ARCTURUS:
1901 chip_name = "arcturus";
1902 break;
1903 case CHIP_RENOIR:
1904 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1905 chip_name = "renoir";
1906 else
1907 chip_name = "green_sardine";
1908 break;
1909 case CHIP_NAVI10:
1910 chip_name = "navi10";
1911 break;
1912 case CHIP_NAVI14:
1913 chip_name = "navi14";
1914 break;
1915 case CHIP_NAVI12:
1916 chip_name = "navi12";
1917 break;
1918 case CHIP_VANGOGH:
1919 chip_name = "vangogh";
1920 break;
1921 case CHIP_YELLOW_CARP:
1922 chip_name = "yellow_carp";
1923 break;
1924 }
1925
1926 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1927 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1928 if (err) {
1929 dev_err(adev->dev,
1930 "Failed to load gpu_info firmware \"%s\"\n",
1931 fw_name);
1932 goto out;
1933 }
1934 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1935 if (err) {
1936 dev_err(adev->dev,
1937 "Failed to validate gpu_info firmware \"%s\"\n",
1938 fw_name);
1939 goto out;
1940 }
1941
1942 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1943 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1944
1945 switch (hdr->version_major) {
1946 case 1:
1947 {
1948 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1949 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1950 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1951
1952 /*
1953 * Should be droped when DAL no longer needs it.
1954 */
1955 if (adev->asic_type == CHIP_NAVI12)
1956 goto parse_soc_bounding_box;
1957
1958 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1959 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1960 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1961 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1962 adev->gfx.config.max_texture_channel_caches =
1963 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1964 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1965 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1966 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1967 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1968 adev->gfx.config.double_offchip_lds_buf =
1969 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1970 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1971 adev->gfx.cu_info.max_waves_per_simd =
1972 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1973 adev->gfx.cu_info.max_scratch_slots_per_cu =
1974 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1975 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1976 if (hdr->version_minor >= 1) {
1977 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1978 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1979 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1980 adev->gfx.config.num_sc_per_sh =
1981 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1982 adev->gfx.config.num_packer_per_sc =
1983 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1984 }
1985
1986parse_soc_bounding_box:
1987 /*
1988 * soc bounding box info is not integrated in disocovery table,
1989 * we always need to parse it from gpu info firmware if needed.
1990 */
1991 if (hdr->version_minor == 2) {
1992 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1993 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1994 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1995 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1996 }
1997 break;
1998 }
1999 default:
2000 dev_err(adev->dev,
2001 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2002 err = -EINVAL;
2003 goto out;
2004 }
2005out:
2006 return err;
2007}
2008
2009/**
2010 * amdgpu_device_ip_early_init - run early init for hardware IPs
2011 *
2012 * @adev: amdgpu_device pointer
2013 *
2014 * Early initialization pass for hardware IPs. The hardware IPs that make
2015 * up each asic are discovered each IP's early_init callback is run. This
2016 * is the first stage in initializing the asic.
2017 * Returns 0 on success, negative error code on failure.
2018 */
2019static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2020{
2021 int i, r;
2022
2023 amdgpu_device_enable_virtual_display(adev);
2024
2025 if (amdgpu_sriov_vf(adev)) {
2026 r = amdgpu_virt_request_full_gpu(adev, true);
2027 if (r)
2028 return r;
2029 }
2030
2031 switch (adev->asic_type) {
2032#ifdef CONFIG_DRM_AMDGPU_SI
2033 case CHIP_VERDE:
2034 case CHIP_TAHITI:
2035 case CHIP_PITCAIRN:
2036 case CHIP_OLAND:
2037 case CHIP_HAINAN:
2038 adev->family = AMDGPU_FAMILY_SI;
2039 r = si_set_ip_blocks(adev);
2040 if (r)
2041 return r;
2042 break;
2043#endif
2044#ifdef CONFIG_DRM_AMDGPU_CIK
2045 case CHIP_BONAIRE:
2046 case CHIP_HAWAII:
2047 case CHIP_KAVERI:
2048 case CHIP_KABINI:
2049 case CHIP_MULLINS:
2050 if (adev->flags & AMD_IS_APU)
2051 adev->family = AMDGPU_FAMILY_KV;
2052 else
2053 adev->family = AMDGPU_FAMILY_CI;
2054
2055 r = cik_set_ip_blocks(adev);
2056 if (r)
2057 return r;
2058 break;
2059#endif
2060 case CHIP_TOPAZ:
2061 case CHIP_TONGA:
2062 case CHIP_FIJI:
2063 case CHIP_POLARIS10:
2064 case CHIP_POLARIS11:
2065 case CHIP_POLARIS12:
2066 case CHIP_VEGAM:
2067 case CHIP_CARRIZO:
2068 case CHIP_STONEY:
2069 if (adev->flags & AMD_IS_APU)
2070 adev->family = AMDGPU_FAMILY_CZ;
2071 else
2072 adev->family = AMDGPU_FAMILY_VI;
2073
2074 r = vi_set_ip_blocks(adev);
2075 if (r)
2076 return r;
2077 break;
2078 case CHIP_VEGA10:
2079 case CHIP_VEGA12:
2080 case CHIP_VEGA20:
2081 case CHIP_RAVEN:
2082 case CHIP_ARCTURUS:
2083 case CHIP_RENOIR:
2084 case CHIP_ALDEBARAN:
2085 if (adev->flags & AMD_IS_APU)
2086 adev->family = AMDGPU_FAMILY_RV;
2087 else
2088 adev->family = AMDGPU_FAMILY_AI;
2089
2090 r = soc15_set_ip_blocks(adev);
2091 if (r)
2092 return r;
2093 break;
2094 case CHIP_NAVI10:
2095 case CHIP_NAVI14:
2096 case CHIP_NAVI12:
2097 case CHIP_SIENNA_CICHLID:
2098 case CHIP_NAVY_FLOUNDER:
2099 case CHIP_DIMGREY_CAVEFISH:
2100 case CHIP_BEIGE_GOBY:
2101 case CHIP_VANGOGH:
2102 case CHIP_YELLOW_CARP:
2103 if (adev->asic_type == CHIP_VANGOGH)
2104 adev->family = AMDGPU_FAMILY_VGH;
2105 else if (adev->asic_type == CHIP_YELLOW_CARP)
2106 adev->family = AMDGPU_FAMILY_YC;
2107 else
2108 adev->family = AMDGPU_FAMILY_NV;
2109
2110 r = nv_set_ip_blocks(adev);
2111 if (r)
2112 return r;
2113 break;
2114 default:
2115 /* FIXME: not supported yet */
2116 return -EINVAL;
2117 }
2118
2119 amdgpu_amdkfd_device_probe(adev);
2120
2121 adev->pm.pp_feature = amdgpu_pp_feature_mask;
2122 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2123 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2124 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2125 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2126
2127 for (i = 0; i < adev->num_ip_blocks; i++) {
2128 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2129 DRM_ERROR("disabled ip block: %d <%s>\n",
2130 i, adev->ip_blocks[i].version->funcs->name);
2131 adev->ip_blocks[i].status.valid = false;
2132 } else {
2133 if (adev->ip_blocks[i].version->funcs->early_init) {
2134 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2135 if (r == -ENOENT) {
2136 adev->ip_blocks[i].status.valid = false;
2137 } else if (r) {
2138 DRM_ERROR("early_init of IP block <%s> failed %d\n",
2139 adev->ip_blocks[i].version->funcs->name, r);
2140 return r;
2141 } else {
2142 adev->ip_blocks[i].status.valid = true;
2143 }
2144 } else {
2145 adev->ip_blocks[i].status.valid = true;
2146 }
2147 }
2148 /* get the vbios after the asic_funcs are set up */
2149 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2150 r = amdgpu_device_parse_gpu_info_fw(adev);
2151 if (r)
2152 return r;
2153
2154 /* Read BIOS */
2155 if (!amdgpu_get_bios(adev))
2156 return -EINVAL;
2157
2158 r = amdgpu_atombios_init(adev);
2159 if (r) {
2160 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2161 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2162 return r;
2163 }
2164
2165 /*get pf2vf msg info at it's earliest time*/
2166 if (amdgpu_sriov_vf(adev))
2167 amdgpu_virt_init_data_exchange(adev);
2168
2169 }
2170 }
2171
2172 adev->cg_flags &= amdgpu_cg_mask;
2173 adev->pg_flags &= amdgpu_pg_mask;
2174
2175 return 0;
2176}
2177
2178static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2179{
2180 int i, r;
2181
2182 for (i = 0; i < adev->num_ip_blocks; i++) {
2183 if (!adev->ip_blocks[i].status.sw)
2184 continue;
2185 if (adev->ip_blocks[i].status.hw)
2186 continue;
2187 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2188 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2189 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2190 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2191 if (r) {
2192 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2193 adev->ip_blocks[i].version->funcs->name, r);
2194 return r;
2195 }
2196 adev->ip_blocks[i].status.hw = true;
2197 }
2198 }
2199
2200 return 0;
2201}
2202
2203static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2204{
2205 int i, r;
2206
2207 for (i = 0; i < adev->num_ip_blocks; i++) {
2208 if (!adev->ip_blocks[i].status.sw)
2209 continue;
2210 if (adev->ip_blocks[i].status.hw)
2211 continue;
2212 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2213 if (r) {
2214 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2215 adev->ip_blocks[i].version->funcs->name, r);
2216 return r;
2217 }
2218 adev->ip_blocks[i].status.hw = true;
2219 }
2220
2221 return 0;
2222}
2223
2224static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2225{
2226 int r = 0;
2227 int i;
2228 uint32_t smu_version;
2229
2230 if (adev->asic_type >= CHIP_VEGA10) {
2231 for (i = 0; i < adev->num_ip_blocks; i++) {
2232 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2233 continue;
2234
2235 if (!adev->ip_blocks[i].status.sw)
2236 continue;
2237
2238 /* no need to do the fw loading again if already done*/
2239 if (adev->ip_blocks[i].status.hw == true)
2240 break;
2241
2242 if (amdgpu_in_reset(adev) || adev->in_suspend) {
2243 r = adev->ip_blocks[i].version->funcs->resume(adev);
2244 if (r) {
2245 DRM_ERROR("resume of IP block <%s> failed %d\n",
2246 adev->ip_blocks[i].version->funcs->name, r);
2247 return r;
2248 }
2249 } else {
2250 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2251 if (r) {
2252 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2253 adev->ip_blocks[i].version->funcs->name, r);
2254 return r;
2255 }
2256 }
2257
2258 adev->ip_blocks[i].status.hw = true;
2259 break;
2260 }
2261 }
2262
2263 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2264 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2265
2266 return r;
2267}
2268
2269/**
2270 * amdgpu_device_ip_init - run init for hardware IPs
2271 *
2272 * @adev: amdgpu_device pointer
2273 *
2274 * Main initialization pass for hardware IPs. The list of all the hardware
2275 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2276 * are run. sw_init initializes the software state associated with each IP
2277 * and hw_init initializes the hardware associated with each IP.
2278 * Returns 0 on success, negative error code on failure.
2279 */
2280static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2281{
2282 int i, r;
2283
2284 r = amdgpu_ras_init(adev);
2285 if (r)
2286 return r;
2287
2288 for (i = 0; i < adev->num_ip_blocks; i++) {
2289 if (!adev->ip_blocks[i].status.valid)
2290 continue;
2291 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2292 if (r) {
2293 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2294 adev->ip_blocks[i].version->funcs->name, r);
2295 goto init_failed;
2296 }
2297 adev->ip_blocks[i].status.sw = true;
2298
2299 /* need to do gmc hw init early so we can allocate gpu mem */
2300 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2301 r = amdgpu_device_vram_scratch_init(adev);
2302 if (r) {
2303 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2304 goto init_failed;
2305 }
2306 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2307 if (r) {
2308 DRM_ERROR("hw_init %d failed %d\n", i, r);
2309 goto init_failed;
2310 }
2311 r = amdgpu_device_wb_init(adev);
2312 if (r) {
2313 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2314 goto init_failed;
2315 }
2316 adev->ip_blocks[i].status.hw = true;
2317
2318 /* right after GMC hw init, we create CSA */
2319 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
2320 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2321 AMDGPU_GEM_DOMAIN_VRAM,
2322 AMDGPU_CSA_SIZE);
2323 if (r) {
2324 DRM_ERROR("allocate CSA failed %d\n", r);
2325 goto init_failed;
2326 }
2327 }
2328 }
2329 }
2330
2331 if (amdgpu_sriov_vf(adev))
2332 amdgpu_virt_init_data_exchange(adev);
2333
2334 r = amdgpu_ib_pool_init(adev);
2335 if (r) {
2336 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2337 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2338 goto init_failed;
2339 }
2340
2341 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2342 if (r)
2343 goto init_failed;
2344
2345 r = amdgpu_device_ip_hw_init_phase1(adev);
2346 if (r)
2347 goto init_failed;
2348
2349 r = amdgpu_device_fw_loading(adev);
2350 if (r)
2351 goto init_failed;
2352
2353 r = amdgpu_device_ip_hw_init_phase2(adev);
2354 if (r)
2355 goto init_failed;
2356
2357 /*
2358 * retired pages will be loaded from eeprom and reserved here,
2359 * it should be called after amdgpu_device_ip_hw_init_phase2 since
2360 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2361 * for I2C communication which only true at this point.
2362 *
2363 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2364 * failure from bad gpu situation and stop amdgpu init process
2365 * accordingly. For other failed cases, it will still release all
2366 * the resource and print error message, rather than returning one
2367 * negative value to upper level.
2368 *
2369 * Note: theoretically, this should be called before all vram allocations
2370 * to protect retired page from abusing
2371 */
2372 r = amdgpu_ras_recovery_init(adev);
2373 if (r)
2374 goto init_failed;
2375
2376 if (adev->gmc.xgmi.num_physical_nodes > 1)
2377 amdgpu_xgmi_add_device(adev);
2378
2379 /* Don't init kfd if whole hive need to be reset during init */
2380 if (!adev->gmc.xgmi.pending_reset)
2381 amdgpu_amdkfd_device_init(adev);
2382
2383 r = amdgpu_amdkfd_resume_iommu(adev);
2384 if (r)
2385 goto init_failed;
2386
2387 amdgpu_fru_get_product_info(adev);
2388
2389init_failed:
2390 if (amdgpu_sriov_vf(adev))
2391 amdgpu_virt_release_full_gpu(adev, true);
2392
2393 return r;
2394}
2395
2396/**
2397 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2398 *
2399 * @adev: amdgpu_device pointer
2400 *
2401 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
2402 * this function before a GPU reset. If the value is retained after a
2403 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
2404 */
2405static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2406{
2407 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2408}
2409
2410/**
2411 * amdgpu_device_check_vram_lost - check if vram is valid
2412 *
2413 * @adev: amdgpu_device pointer
2414 *
2415 * Checks the reset magic value written to the gart pointer in VRAM.
2416 * The driver calls this after a GPU reset to see if the contents of
2417 * VRAM is lost or now.
2418 * returns true if vram is lost, false if not.
2419 */
2420static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2421{
2422 if (memcmp(adev->gart.ptr, adev->reset_magic,
2423 AMDGPU_RESET_MAGIC_NUM))
2424 return true;
2425
2426 if (!amdgpu_in_reset(adev))
2427 return false;
2428
2429 /*
2430 * For all ASICs with baco/mode1 reset, the VRAM is
2431 * always assumed to be lost.
2432 */
2433 switch (amdgpu_asic_reset_method(adev)) {
2434 case AMD_RESET_METHOD_BACO:
2435 case AMD_RESET_METHOD_MODE1:
2436 return true;
2437 default:
2438 return false;
2439 }
2440}
2441
2442/**
2443 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2444 *
2445 * @adev: amdgpu_device pointer
2446 * @state: clockgating state (gate or ungate)
2447 *
2448 * The list of all the hardware IPs that make up the asic is walked and the
2449 * set_clockgating_state callbacks are run.
2450 * Late initialization pass enabling clockgating for hardware IPs.
2451 * Fini or suspend, pass disabling clockgating for hardware IPs.
2452 * Returns 0 on success, negative error code on failure.
2453 */
2454
2455int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2456 enum amd_clockgating_state state)
2457{
2458 int i, j, r;
2459
2460 if (amdgpu_emu_mode == 1)
2461 return 0;
2462
2463 for (j = 0; j < adev->num_ip_blocks; j++) {
2464 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2465 if (!adev->ip_blocks[i].status.late_initialized)
2466 continue;
2467 /* skip CG for GFX on S0ix */
2468 if (adev->in_s0ix &&
2469 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2470 continue;
2471 /* skip CG for VCE/UVD, it's handled specially */
2472 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2473 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2474 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2475 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2476 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2477 /* enable clockgating to save power */
2478 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2479 state);
2480 if (r) {
2481 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2482 adev->ip_blocks[i].version->funcs->name, r);
2483 return r;
2484 }
2485 }
2486 }
2487
2488 return 0;
2489}
2490
2491int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2492 enum amd_powergating_state state)
2493{
2494 int i, j, r;
2495
2496 if (amdgpu_emu_mode == 1)
2497 return 0;
2498
2499 for (j = 0; j < adev->num_ip_blocks; j++) {
2500 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2501 if (!adev->ip_blocks[i].status.late_initialized)
2502 continue;
2503 /* skip PG for GFX on S0ix */
2504 if (adev->in_s0ix &&
2505 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2506 continue;
2507 /* skip CG for VCE/UVD, it's handled specially */
2508 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2509 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2510 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2511 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2512 adev->ip_blocks[i].version->funcs->set_powergating_state) {
2513 /* enable powergating to save power */
2514 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2515 state);
2516 if (r) {
2517 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2518 adev->ip_blocks[i].version->funcs->name, r);
2519 return r;
2520 }
2521 }
2522 }
2523 return 0;
2524}
2525
2526static int amdgpu_device_enable_mgpu_fan_boost(void)
2527{
2528 struct amdgpu_gpu_instance *gpu_ins;
2529 struct amdgpu_device *adev;
2530 int i, ret = 0;
2531
2532 mutex_lock(&mgpu_info.mutex);
2533
2534 /*
2535 * MGPU fan boost feature should be enabled
2536 * only when there are two or more dGPUs in
2537 * the system
2538 */
2539 if (mgpu_info.num_dgpu < 2)
2540 goto out;
2541
2542 for (i = 0; i < mgpu_info.num_dgpu; i++) {
2543 gpu_ins = &(mgpu_info.gpu_ins[i]);
2544 adev = gpu_ins->adev;
2545 if (!(adev->flags & AMD_IS_APU) &&
2546 !gpu_ins->mgpu_fan_enabled) {
2547 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2548 if (ret)
2549 break;
2550
2551 gpu_ins->mgpu_fan_enabled = 1;
2552 }
2553 }
2554
2555out:
2556 mutex_unlock(&mgpu_info.mutex);
2557
2558 return ret;
2559}
2560
2561/**
2562 * amdgpu_device_ip_late_init - run late init for hardware IPs
2563 *
2564 * @adev: amdgpu_device pointer
2565 *
2566 * Late initialization pass for hardware IPs. The list of all the hardware
2567 * IPs that make up the asic is walked and the late_init callbacks are run.
2568 * late_init covers any special initialization that an IP requires
2569 * after all of the have been initialized or something that needs to happen
2570 * late in the init process.
2571 * Returns 0 on success, negative error code on failure.
2572 */
2573static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2574{
2575 struct amdgpu_gpu_instance *gpu_instance;
2576 int i = 0, r;
2577
2578 for (i = 0; i < adev->num_ip_blocks; i++) {
2579 if (!adev->ip_blocks[i].status.hw)
2580 continue;
2581 if (adev->ip_blocks[i].version->funcs->late_init) {
2582 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2583 if (r) {
2584 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2585 adev->ip_blocks[i].version->funcs->name, r);
2586 return r;
2587 }
2588 }
2589 adev->ip_blocks[i].status.late_initialized = true;
2590 }
2591
2592 amdgpu_ras_set_error_query_ready(adev, true);
2593
2594 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2595 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2596
2597 amdgpu_device_fill_reset_magic(adev);
2598
2599 r = amdgpu_device_enable_mgpu_fan_boost();
2600 if (r)
2601 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2602
2603 /* For XGMI + passthrough configuration on arcturus, enable light SBR */
2604 if (adev->asic_type == CHIP_ARCTURUS &&
2605 amdgpu_passthrough(adev) &&
2606 adev->gmc.xgmi.num_physical_nodes > 1)
2607 smu_set_light_sbr(&adev->smu, true);
2608
2609 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2610 mutex_lock(&mgpu_info.mutex);
2611
2612 /*
2613 * Reset device p-state to low as this was booted with high.
2614 *
2615 * This should be performed only after all devices from the same
2616 * hive get initialized.
2617 *
2618 * However, it's unknown how many device in the hive in advance.
2619 * As this is counted one by one during devices initializations.
2620 *
2621 * So, we wait for all XGMI interlinked devices initialized.
2622 * This may bring some delays as those devices may come from
2623 * different hives. But that should be OK.
2624 */
2625 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2626 for (i = 0; i < mgpu_info.num_gpu; i++) {
2627 gpu_instance = &(mgpu_info.gpu_ins[i]);
2628 if (gpu_instance->adev->flags & AMD_IS_APU)
2629 continue;
2630
2631 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2632 AMDGPU_XGMI_PSTATE_MIN);
2633 if (r) {
2634 DRM_ERROR("pstate setting failed (%d).\n", r);
2635 break;
2636 }
2637 }
2638 }
2639
2640 mutex_unlock(&mgpu_info.mutex);
2641 }
2642
2643 return 0;
2644}
2645
2646static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2647{
2648 int i, r;
2649
2650 for (i = 0; i < adev->num_ip_blocks; i++) {
2651 if (!adev->ip_blocks[i].version->funcs->early_fini)
2652 continue;
2653
2654 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2655 if (r) {
2656 DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2657 adev->ip_blocks[i].version->funcs->name, r);
2658 }
2659 }
2660
2661 amdgpu_amdkfd_suspend(adev, false);
2662
2663 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2664 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2665
2666 /* need to disable SMC first */
2667 for (i = 0; i < adev->num_ip_blocks; i++) {
2668 if (!adev->ip_blocks[i].status.hw)
2669 continue;
2670 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2671 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2672 /* XXX handle errors */
2673 if (r) {
2674 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2675 adev->ip_blocks[i].version->funcs->name, r);
2676 }
2677 adev->ip_blocks[i].status.hw = false;
2678 break;
2679 }
2680 }
2681
2682 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2683 if (!adev->ip_blocks[i].status.hw)
2684 continue;
2685
2686 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2687 /* XXX handle errors */
2688 if (r) {
2689 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2690 adev->ip_blocks[i].version->funcs->name, r);
2691 }
2692
2693 adev->ip_blocks[i].status.hw = false;
2694 }
2695
2696 return 0;
2697}
2698
2699/**
2700 * amdgpu_device_ip_fini - run fini for hardware IPs
2701 *
2702 * @adev: amdgpu_device pointer
2703 *
2704 * Main teardown pass for hardware IPs. The list of all the hardware
2705 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2706 * are run. hw_fini tears down the hardware associated with each IP
2707 * and sw_fini tears down any software state associated with each IP.
2708 * Returns 0 on success, negative error code on failure.
2709 */
2710static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2711{
2712 int i, r;
2713
2714 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2715 amdgpu_virt_release_ras_err_handler_data(adev);
2716
2717 amdgpu_ras_pre_fini(adev);
2718
2719 if (adev->gmc.xgmi.num_physical_nodes > 1)
2720 amdgpu_xgmi_remove_device(adev);
2721
2722 amdgpu_amdkfd_device_fini_sw(adev);
2723
2724 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2725 if (!adev->ip_blocks[i].status.sw)
2726 continue;
2727
2728 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2729 amdgpu_ucode_free_bo(adev);
2730 amdgpu_free_static_csa(&adev->virt.csa_obj);
2731 amdgpu_device_wb_fini(adev);
2732 amdgpu_device_vram_scratch_fini(adev);
2733 amdgpu_ib_pool_fini(adev);
2734 }
2735
2736 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2737 /* XXX handle errors */
2738 if (r) {
2739 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2740 adev->ip_blocks[i].version->funcs->name, r);
2741 }
2742 adev->ip_blocks[i].status.sw = false;
2743 adev->ip_blocks[i].status.valid = false;
2744 }
2745
2746 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2747 if (!adev->ip_blocks[i].status.late_initialized)
2748 continue;
2749 if (adev->ip_blocks[i].version->funcs->late_fini)
2750 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2751 adev->ip_blocks[i].status.late_initialized = false;
2752 }
2753
2754 amdgpu_ras_fini(adev);
2755
2756 if (amdgpu_sriov_vf(adev))
2757 if (amdgpu_virt_release_full_gpu(adev, false))
2758 DRM_ERROR("failed to release exclusive mode on fini\n");
2759
2760 return 0;
2761}
2762
2763/**
2764 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2765 *
2766 * @work: work_struct.
2767 */
2768static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2769{
2770 struct amdgpu_device *adev =
2771 container_of(work, struct amdgpu_device, delayed_init_work.work);
2772 int r;
2773
2774 r = amdgpu_ib_ring_tests(adev);
2775 if (r)
2776 DRM_ERROR("ib ring test failed (%d).\n", r);
2777}
2778
2779static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2780{
2781 struct amdgpu_device *adev =
2782 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2783
2784 WARN_ON_ONCE(adev->gfx.gfx_off_state);
2785 WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2786
2787 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2788 adev->gfx.gfx_off_state = true;
2789}
2790
2791/**
2792 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2793 *
2794 * @adev: amdgpu_device pointer
2795 *
2796 * Main suspend function for hardware IPs. The list of all the hardware
2797 * IPs that make up the asic is walked, clockgating is disabled and the
2798 * suspend callbacks are run. suspend puts the hardware and software state
2799 * in each IP into a state suitable for suspend.
2800 * Returns 0 on success, negative error code on failure.
2801 */
2802static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2803{
2804 int i, r;
2805
2806 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2807 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2808
2809 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2810 if (!adev->ip_blocks[i].status.valid)
2811 continue;
2812
2813 /* displays are handled separately */
2814 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2815 continue;
2816
2817 /* XXX handle errors */
2818 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2819 /* XXX handle errors */
2820 if (r) {
2821 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2822 adev->ip_blocks[i].version->funcs->name, r);
2823 return r;
2824 }
2825
2826 adev->ip_blocks[i].status.hw = false;
2827 }
2828
2829 return 0;
2830}
2831
2832/**
2833 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2834 *
2835 * @adev: amdgpu_device pointer
2836 *
2837 * Main suspend function for hardware IPs. The list of all the hardware
2838 * IPs that make up the asic is walked, clockgating is disabled and the
2839 * suspend callbacks are run. suspend puts the hardware and software state
2840 * in each IP into a state suitable for suspend.
2841 * Returns 0 on success, negative error code on failure.
2842 */
2843static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2844{
2845 int i, r;
2846
2847 if (adev->in_s0ix)
2848 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry);
2849
2850 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2851 if (!adev->ip_blocks[i].status.valid)
2852 continue;
2853 /* displays are handled in phase1 */
2854 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2855 continue;
2856 /* PSP lost connection when err_event_athub occurs */
2857 if (amdgpu_ras_intr_triggered() &&
2858 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2859 adev->ip_blocks[i].status.hw = false;
2860 continue;
2861 }
2862
2863 /* skip unnecessary suspend if we do not initialize them yet */
2864 if (adev->gmc.xgmi.pending_reset &&
2865 !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2866 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
2867 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2868 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
2869 adev->ip_blocks[i].status.hw = false;
2870 continue;
2871 }
2872
2873 /* skip suspend of gfx and psp for S0ix
2874 * gfx is in gfxoff state, so on resume it will exit gfxoff just
2875 * like at runtime. PSP is also part of the always on hardware
2876 * so no need to suspend it.
2877 */
2878 if (adev->in_s0ix &&
2879 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
2880 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX))
2881 continue;
2882
2883 /* XXX handle errors */
2884 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2885 /* XXX handle errors */
2886 if (r) {
2887 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2888 adev->ip_blocks[i].version->funcs->name, r);
2889 }
2890 adev->ip_blocks[i].status.hw = false;
2891 /* handle putting the SMC in the appropriate state */
2892 if(!amdgpu_sriov_vf(adev)){
2893 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2894 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
2895 if (r) {
2896 DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2897 adev->mp1_state, r);
2898 return r;
2899 }
2900 }
2901 }
2902 }
2903
2904 return 0;
2905}
2906
2907/**
2908 * amdgpu_device_ip_suspend - run suspend for hardware IPs
2909 *
2910 * @adev: amdgpu_device pointer
2911 *
2912 * Main suspend function for hardware IPs. The list of all the hardware
2913 * IPs that make up the asic is walked, clockgating is disabled and the
2914 * suspend callbacks are run. suspend puts the hardware and software state
2915 * in each IP into a state suitable for suspend.
2916 * Returns 0 on success, negative error code on failure.
2917 */
2918int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2919{
2920 int r;
2921
2922 if (amdgpu_sriov_vf(adev)) {
2923 amdgpu_virt_fini_data_exchange(adev);
2924 amdgpu_virt_request_full_gpu(adev, false);
2925 }
2926
2927 r = amdgpu_device_ip_suspend_phase1(adev);
2928 if (r)
2929 return r;
2930 r = amdgpu_device_ip_suspend_phase2(adev);
2931
2932 if (amdgpu_sriov_vf(adev))
2933 amdgpu_virt_release_full_gpu(adev, false);
2934
2935 return r;
2936}
2937
2938static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2939{
2940 int i, r;
2941
2942 static enum amd_ip_block_type ip_order[] = {
2943 AMD_IP_BLOCK_TYPE_GMC,
2944 AMD_IP_BLOCK_TYPE_COMMON,
2945 AMD_IP_BLOCK_TYPE_PSP,
2946 AMD_IP_BLOCK_TYPE_IH,
2947 };
2948
2949 for (i = 0; i < adev->num_ip_blocks; i++) {
2950 int j;
2951 struct amdgpu_ip_block *block;
2952
2953 block = &adev->ip_blocks[i];
2954 block->status.hw = false;
2955
2956 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
2957
2958 if (block->version->type != ip_order[j] ||
2959 !block->status.valid)
2960 continue;
2961
2962 r = block->version->funcs->hw_init(adev);
2963 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2964 if (r)
2965 return r;
2966 block->status.hw = true;
2967 }
2968 }
2969
2970 return 0;
2971}
2972
2973static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2974{
2975 int i, r;
2976
2977 static enum amd_ip_block_type ip_order[] = {
2978 AMD_IP_BLOCK_TYPE_SMC,
2979 AMD_IP_BLOCK_TYPE_DCE,
2980 AMD_IP_BLOCK_TYPE_GFX,
2981 AMD_IP_BLOCK_TYPE_SDMA,
2982 AMD_IP_BLOCK_TYPE_UVD,
2983 AMD_IP_BLOCK_TYPE_VCE,
2984 AMD_IP_BLOCK_TYPE_VCN
2985 };
2986
2987 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2988 int j;
2989 struct amdgpu_ip_block *block;
2990
2991 for (j = 0; j < adev->num_ip_blocks; j++) {
2992 block = &adev->ip_blocks[j];
2993
2994 if (block->version->type != ip_order[i] ||
2995 !block->status.valid ||
2996 block->status.hw)
2997 continue;
2998
2999 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3000 r = block->version->funcs->resume(adev);
3001 else
3002 r = block->version->funcs->hw_init(adev);
3003
3004 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3005 if (r)
3006 return r;
3007 block->status.hw = true;
3008 }
3009 }
3010
3011 return 0;
3012}
3013
3014/**
3015 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3016 *
3017 * @adev: amdgpu_device pointer
3018 *
3019 * First resume function for hardware IPs. The list of all the hardware
3020 * IPs that make up the asic is walked and the resume callbacks are run for
3021 * COMMON, GMC, and IH. resume puts the hardware into a functional state
3022 * after a suspend and updates the software state as necessary. This
3023 * function is also used for restoring the GPU after a GPU reset.
3024 * Returns 0 on success, negative error code on failure.
3025 */
3026static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3027{
3028 int i, r;
3029
3030 for (i = 0; i < adev->num_ip_blocks; i++) {
3031 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3032 continue;
3033 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3034 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3035 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
3036
3037 r = adev->ip_blocks[i].version->funcs->resume(adev);
3038 if (r) {
3039 DRM_ERROR("resume of IP block <%s> failed %d\n",
3040 adev->ip_blocks[i].version->funcs->name, r);
3041 return r;
3042 }
3043 adev->ip_blocks[i].status.hw = true;
3044 }
3045 }
3046
3047 return 0;
3048}
3049
3050/**
3051 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3052 *
3053 * @adev: amdgpu_device pointer
3054 *
3055 * First resume function for hardware IPs. The list of all the hardware
3056 * IPs that make up the asic is walked and the resume callbacks are run for
3057 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
3058 * functional state after a suspend and updates the software state as
3059 * necessary. This function is also used for restoring the GPU after a GPU
3060 * reset.
3061 * Returns 0 on success, negative error code on failure.
3062 */
3063static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3064{
3065 int i, r;
3066
3067 for (i = 0; i < adev->num_ip_blocks; i++) {
3068 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3069 continue;
3070 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3071 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3072 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3073 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3074 continue;
3075 r = adev->ip_blocks[i].version->funcs->resume(adev);
3076 if (r) {
3077 DRM_ERROR("resume of IP block <%s> failed %d\n",
3078 adev->ip_blocks[i].version->funcs->name, r);
3079 return r;
3080 }
3081 adev->ip_blocks[i].status.hw = true;
3082 }
3083
3084 return 0;
3085}
3086
3087/**
3088 * amdgpu_device_ip_resume - run resume for hardware IPs
3089 *
3090 * @adev: amdgpu_device pointer
3091 *
3092 * Main resume function for hardware IPs. The hardware IPs
3093 * are split into two resume functions because they are
3094 * are also used in in recovering from a GPU reset and some additional
3095 * steps need to be take between them. In this case (S3/S4) they are
3096 * run sequentially.
3097 * Returns 0 on success, negative error code on failure.
3098 */
3099static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3100{
3101 int r;
3102
3103 r = amdgpu_amdkfd_resume_iommu(adev);
3104 if (r)
3105 return r;
3106
3107 r = amdgpu_device_ip_resume_phase1(adev);
3108 if (r)
3109 return r;
3110
3111 r = amdgpu_device_fw_loading(adev);
3112 if (r)
3113 return r;
3114
3115 r = amdgpu_device_ip_resume_phase2(adev);
3116
3117 return r;
3118}
3119
3120/**
3121 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3122 *
3123 * @adev: amdgpu_device pointer
3124 *
3125 * Query the VBIOS data tables to determine if the board supports SR-IOV.
3126 */
3127static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3128{
3129 if (amdgpu_sriov_vf(adev)) {
3130 if (adev->is_atom_fw) {
3131 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3132 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3133 } else {
3134 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3135 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3136 }
3137
3138 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3139 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3140 }
3141}
3142
3143/**
3144 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3145 *
3146 * @asic_type: AMD asic type
3147 *
3148 * Check if there is DC (new modesetting infrastructre) support for an asic.
3149 * returns true if DC has support, false if not.
3150 */
3151bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3152{
3153 switch (asic_type) {
3154#if defined(CONFIG_DRM_AMD_DC)
3155#if defined(CONFIG_DRM_AMD_DC_SI)
3156 case CHIP_TAHITI:
3157 case CHIP_PITCAIRN:
3158 case CHIP_VERDE:
3159 case CHIP_OLAND:
3160#endif
3161 case CHIP_BONAIRE:
3162 case CHIP_KAVERI:
3163 case CHIP_KABINI:
3164 case CHIP_MULLINS:
3165 /*
3166 * We have systems in the wild with these ASICs that require
3167 * LVDS and VGA support which is not supported with DC.
3168 *
3169 * Fallback to the non-DC driver here by default so as not to
3170 * cause regressions.
3171 */
3172 return amdgpu_dc > 0;
3173 case CHIP_HAWAII:
3174 case CHIP_CARRIZO:
3175 case CHIP_STONEY:
3176 case CHIP_POLARIS10:
3177 case CHIP_POLARIS11:
3178 case CHIP_POLARIS12:
3179 case CHIP_VEGAM:
3180 case CHIP_TONGA:
3181 case CHIP_FIJI:
3182 case CHIP_VEGA10:
3183 case CHIP_VEGA12:
3184 case CHIP_VEGA20:
3185#if defined(CONFIG_DRM_AMD_DC_DCN)
3186 case CHIP_RAVEN:
3187 case CHIP_NAVI10:
3188 case CHIP_NAVI14:
3189 case CHIP_NAVI12:
3190 case CHIP_RENOIR:
3191 case CHIP_SIENNA_CICHLID:
3192 case CHIP_NAVY_FLOUNDER:
3193 case CHIP_DIMGREY_CAVEFISH:
3194 case CHIP_BEIGE_GOBY:
3195 case CHIP_VANGOGH:
3196 case CHIP_YELLOW_CARP:
3197#endif
3198 return amdgpu_dc != 0;
3199#endif
3200 default:
3201 if (amdgpu_dc > 0)
3202 DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3203 "but isn't supported by ASIC, ignoring\n");
3204 return false;
3205 }
3206}
3207
3208/**
3209 * amdgpu_device_has_dc_support - check if dc is supported
3210 *
3211 * @adev: amdgpu_device pointer
3212 *
3213 * Returns true for supported, false for not supported
3214 */
3215bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3216{
3217 if (amdgpu_sriov_vf(adev) ||
3218 adev->enable_virtual_display ||
3219 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3220 return false;
3221
3222 return amdgpu_device_asic_has_dc_support(adev->asic_type);
3223}
3224
3225static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3226{
3227 struct amdgpu_device *adev =
3228 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3229 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3230
3231 /* It's a bug to not have a hive within this function */
3232 if (WARN_ON(!hive))
3233 return;
3234
3235 /*
3236 * Use task barrier to synchronize all xgmi reset works across the
3237 * hive. task_barrier_enter and task_barrier_exit will block
3238 * until all the threads running the xgmi reset works reach
3239 * those points. task_barrier_full will do both blocks.
3240 */
3241 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3242
3243 task_barrier_enter(&hive->tb);
3244 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3245
3246 if (adev->asic_reset_res)
3247 goto fail;
3248
3249 task_barrier_exit(&hive->tb);
3250 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3251
3252 if (adev->asic_reset_res)
3253 goto fail;
3254
3255 if (adev->mmhub.ras_funcs &&
3256 adev->mmhub.ras_funcs->reset_ras_error_count)
3257 adev->mmhub.ras_funcs->reset_ras_error_count(adev);
3258 } else {
3259
3260 task_barrier_full(&hive->tb);
3261 adev->asic_reset_res = amdgpu_asic_reset(adev);
3262 }
3263
3264fail:
3265 if (adev->asic_reset_res)
3266 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3267 adev->asic_reset_res, adev_to_drm(adev)->unique);
3268 amdgpu_put_xgmi_hive(hive);
3269}
3270
3271static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3272{
3273 char *input = amdgpu_lockup_timeout;
3274 char *timeout_setting = NULL;
3275 int index = 0;
3276 long timeout;
3277 int ret = 0;
3278
3279 /*
3280 * By default timeout for non compute jobs is 10000
3281 * and 60000 for compute jobs.
3282 * In SR-IOV or passthrough mode, timeout for compute
3283 * jobs are 60000 by default.
3284 */
3285 adev->gfx_timeout = msecs_to_jiffies(10000);
3286 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3287 if (amdgpu_sriov_vf(adev))
3288 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3289 msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3290 else
3291 adev->compute_timeout = msecs_to_jiffies(60000);
3292
3293 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3294 while ((timeout_setting = strsep(&input, ",")) &&
3295 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3296 ret = kstrtol(timeout_setting, 0, &timeout);
3297 if (ret)
3298 return ret;
3299
3300 if (timeout == 0) {
3301 index++;
3302 continue;
3303 } else if (timeout < 0) {
3304 timeout = MAX_SCHEDULE_TIMEOUT;
3305 } else {
3306 timeout = msecs_to_jiffies(timeout);
3307 }
3308
3309 switch (index++) {
3310 case 0:
3311 adev->gfx_timeout = timeout;
3312 break;
3313 case 1:
3314 adev->compute_timeout = timeout;
3315 break;
3316 case 2:
3317 adev->sdma_timeout = timeout;
3318 break;
3319 case 3:
3320 adev->video_timeout = timeout;
3321 break;
3322 default:
3323 break;
3324 }
3325 }
3326 /*
3327 * There is only one value specified and
3328 * it should apply to all non-compute jobs.
3329 */
3330 if (index == 1) {
3331 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3332 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3333 adev->compute_timeout = adev->gfx_timeout;
3334 }
3335 }
3336
3337 return ret;
3338}
3339
3340static const struct attribute *amdgpu_dev_attributes[] = {
3341 &dev_attr_product_name.attr,
3342 &dev_attr_product_number.attr,
3343 &dev_attr_serial_number.attr,
3344 &dev_attr_pcie_replay_count.attr,
3345 NULL
3346};
3347
3348/**
3349 * amdgpu_device_init - initialize the driver
3350 *
3351 * @adev: amdgpu_device pointer
3352 * @flags: driver flags
3353 *
3354 * Initializes the driver info and hw (all asics).
3355 * Returns 0 for success or an error on failure.
3356 * Called at driver startup.
3357 */
3358int amdgpu_device_init(struct amdgpu_device *adev,
3359 uint32_t flags)
3360{
3361 struct drm_device *ddev = adev_to_drm(adev);
3362 struct pci_dev *pdev = adev->pdev;
3363 int r, i;
3364 bool px = false;
3365 u32 max_MBps;
3366
3367 adev->shutdown = false;
3368 adev->flags = flags;
3369
3370 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3371 adev->asic_type = amdgpu_force_asic_type;
3372 else
3373 adev->asic_type = flags & AMD_ASIC_MASK;
3374
3375 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3376 if (amdgpu_emu_mode == 1)
3377 adev->usec_timeout *= 10;
3378 adev->gmc.gart_size = 512 * 1024 * 1024;
3379 adev->accel_working = false;
3380 adev->num_rings = 0;
3381 adev->mman.buffer_funcs = NULL;
3382 adev->mman.buffer_funcs_ring = NULL;
3383 adev->vm_manager.vm_pte_funcs = NULL;
3384 adev->vm_manager.vm_pte_num_scheds = 0;
3385 adev->gmc.gmc_funcs = NULL;
3386 adev->harvest_ip_mask = 0x0;
3387 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3388 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3389
3390 adev->smc_rreg = &amdgpu_invalid_rreg;
3391 adev->smc_wreg = &amdgpu_invalid_wreg;
3392 adev->pcie_rreg = &amdgpu_invalid_rreg;
3393 adev->pcie_wreg = &amdgpu_invalid_wreg;
3394 adev->pciep_rreg = &amdgpu_invalid_rreg;
3395 adev->pciep_wreg = &amdgpu_invalid_wreg;
3396 adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3397 adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3398 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3399 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3400 adev->didt_rreg = &amdgpu_invalid_rreg;
3401 adev->didt_wreg = &amdgpu_invalid_wreg;
3402 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3403 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3404 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3405 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3406
3407 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3408 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3409 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3410
3411 /* mutex initialization are all done here so we
3412 * can recall function without having locking issues */
3413 mutex_init(&adev->firmware.mutex);
3414 mutex_init(&adev->pm.mutex);
3415 mutex_init(&adev->gfx.gpu_clock_mutex);
3416 mutex_init(&adev->srbm_mutex);
3417 mutex_init(&adev->gfx.pipe_reserve_mutex);
3418 mutex_init(&adev->gfx.gfx_off_mutex);
3419 mutex_init(&adev->grbm_idx_mutex);
3420 mutex_init(&adev->mn_lock);
3421 mutex_init(&adev->virt.vf_errors.lock);
3422 hash_init(adev->mn_hash);
3423 atomic_set(&adev->in_gpu_reset, 0);
3424 init_rwsem(&adev->reset_sem);
3425 mutex_init(&adev->psp.mutex);
3426 mutex_init(&adev->notifier_lock);
3427
3428 r = amdgpu_device_init_apu_flags(adev);
3429 if (r)
3430 return r;
3431
3432 r = amdgpu_device_check_arguments(adev);
3433 if (r)
3434 return r;
3435
3436 spin_lock_init(&adev->mmio_idx_lock);
3437 spin_lock_init(&adev->smc_idx_lock);
3438 spin_lock_init(&adev->pcie_idx_lock);
3439 spin_lock_init(&adev->uvd_ctx_idx_lock);
3440 spin_lock_init(&adev->didt_idx_lock);
3441 spin_lock_init(&adev->gc_cac_idx_lock);
3442 spin_lock_init(&adev->se_cac_idx_lock);
3443 spin_lock_init(&adev->audio_endpt_idx_lock);
3444 spin_lock_init(&adev->mm_stats.lock);
3445
3446 INIT_LIST_HEAD(&adev->shadow_list);
3447 mutex_init(&adev->shadow_list_lock);
3448
3449 INIT_LIST_HEAD(&adev->reset_list);
3450
3451 INIT_DELAYED_WORK(&adev->delayed_init_work,
3452 amdgpu_device_delayed_init_work_handler);
3453 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3454 amdgpu_device_delay_enable_gfx_off);
3455
3456 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3457
3458 adev->gfx.gfx_off_req_count = 1;
3459 adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3460
3461 atomic_set(&adev->throttling_logging_enabled, 1);
3462 /*
3463 * If throttling continues, logging will be performed every minute
3464 * to avoid log flooding. "-1" is subtracted since the thermal
3465 * throttling interrupt comes every second. Thus, the total logging
3466 * interval is 59 seconds(retelimited printk interval) + 1(waiting
3467 * for throttling interrupt) = 60 seconds.
3468 */
3469 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3470 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3471
3472 /* Registers mapping */
3473 /* TODO: block userspace mapping of io register */
3474 if (adev->asic_type >= CHIP_BONAIRE) {
3475 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3476 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3477 } else {
3478 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3479 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3480 }
3481
3482 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3483 if (adev->rmmio == NULL) {
3484 return -ENOMEM;
3485 }
3486 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3487 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3488
3489 /* enable PCIE atomic ops */
3490 r = pci_enable_atomic_ops_to_root(adev->pdev,
3491 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3492 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3493 if (r) {
3494 adev->have_atomics_support = false;
3495 DRM_INFO("PCIE atomic ops is not supported\n");
3496 } else {
3497 adev->have_atomics_support = true;
3498 }
3499
3500 amdgpu_device_get_pcie_info(adev);
3501
3502 if (amdgpu_mcbp)
3503 DRM_INFO("MCBP is enabled\n");
3504
3505 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
3506 adev->enable_mes = true;
3507
3508 /* detect hw virtualization here */
3509 amdgpu_detect_virtualization(adev);
3510
3511 r = amdgpu_device_get_job_timeout_settings(adev);
3512 if (r) {
3513 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3514 return r;
3515 }
3516
3517 /* early init functions */
3518 r = amdgpu_device_ip_early_init(adev);
3519 if (r)
3520 return r;
3521
3522 /* doorbell bar mapping and doorbell index init*/
3523 amdgpu_device_doorbell_init(adev);
3524
3525 if (amdgpu_emu_mode == 1) {
3526 /* post the asic on emulation mode */
3527 emu_soc_asic_init(adev);
3528 goto fence_driver_init;
3529 }
3530
3531 amdgpu_reset_init(adev);
3532
3533 /* detect if we are with an SRIOV vbios */
3534 amdgpu_device_detect_sriov_bios(adev);
3535
3536 /* check if we need to reset the asic
3537 * E.g., driver was not cleanly unloaded previously, etc.
3538 */
3539 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3540 if (adev->gmc.xgmi.num_physical_nodes) {
3541 dev_info(adev->dev, "Pending hive reset.\n");
3542 adev->gmc.xgmi.pending_reset = true;
3543 /* Only need to init necessary block for SMU to handle the reset */
3544 for (i = 0; i < adev->num_ip_blocks; i++) {
3545 if (!adev->ip_blocks[i].status.valid)
3546 continue;
3547 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3548 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3549 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3550 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3551 DRM_DEBUG("IP %s disabled for hw_init.\n",
3552 adev->ip_blocks[i].version->funcs->name);
3553 adev->ip_blocks[i].status.hw = true;
3554 }
3555 }
3556 } else {
3557 r = amdgpu_asic_reset(adev);
3558 if (r) {
3559 dev_err(adev->dev, "asic reset on init failed\n");
3560 goto failed;
3561 }
3562 }
3563 }
3564
3565 pci_enable_pcie_error_reporting(adev->pdev);
3566
3567 /* Post card if necessary */
3568 if (amdgpu_device_need_post(adev)) {
3569 if (!adev->bios) {
3570 dev_err(adev->dev, "no vBIOS found\n");
3571 r = -EINVAL;
3572 goto failed;
3573 }
3574 DRM_INFO("GPU posting now...\n");
3575 r = amdgpu_device_asic_init(adev);
3576 if (r) {
3577 dev_err(adev->dev, "gpu post error!\n");
3578 goto failed;
3579 }
3580 }
3581
3582 if (adev->is_atom_fw) {
3583 /* Initialize clocks */
3584 r = amdgpu_atomfirmware_get_clock_info(adev);
3585 if (r) {
3586 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3587 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3588 goto failed;
3589 }
3590 } else {
3591 /* Initialize clocks */
3592 r = amdgpu_atombios_get_clock_info(adev);
3593 if (r) {
3594 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3595 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3596 goto failed;
3597 }
3598 /* init i2c buses */
3599 if (!amdgpu_device_has_dc_support(adev))
3600 amdgpu_atombios_i2c_init(adev);
3601 }
3602
3603fence_driver_init:
3604 /* Fence driver */
3605 r = amdgpu_fence_driver_sw_init(adev);
3606 if (r) {
3607 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3608 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3609 goto failed;
3610 }
3611
3612 /* init the mode config */
3613 drm_mode_config_init(adev_to_drm(adev));
3614
3615 r = amdgpu_device_ip_init(adev);
3616 if (r) {
3617 /* failed in exclusive mode due to timeout */
3618 if (amdgpu_sriov_vf(adev) &&
3619 !amdgpu_sriov_runtime(adev) &&
3620 amdgpu_virt_mmio_blocked(adev) &&
3621 !amdgpu_virt_wait_reset(adev)) {
3622 dev_err(adev->dev, "VF exclusive mode timeout\n");
3623 /* Don't send request since VF is inactive. */
3624 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3625 adev->virt.ops = NULL;
3626 r = -EAGAIN;
3627 goto release_ras_con;
3628 }
3629 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3630 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3631 goto release_ras_con;
3632 }
3633
3634 amdgpu_fence_driver_hw_init(adev);
3635
3636 dev_info(adev->dev,
3637 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3638 adev->gfx.config.max_shader_engines,
3639 adev->gfx.config.max_sh_per_se,
3640 adev->gfx.config.max_cu_per_sh,
3641 adev->gfx.cu_info.number);
3642
3643 adev->accel_working = true;
3644
3645 amdgpu_vm_check_compute_bug(adev);
3646
3647 /* Initialize the buffer migration limit. */
3648 if (amdgpu_moverate >= 0)
3649 max_MBps = amdgpu_moverate;
3650 else
3651 max_MBps = 8; /* Allow 8 MB/s. */
3652 /* Get a log2 for easy divisions. */
3653 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3654
3655 amdgpu_fbdev_init(adev);
3656
3657 r = amdgpu_pm_sysfs_init(adev);
3658 if (r) {
3659 adev->pm_sysfs_en = false;
3660 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3661 } else
3662 adev->pm_sysfs_en = true;
3663
3664 r = amdgpu_ucode_sysfs_init(adev);
3665 if (r) {
3666 adev->ucode_sysfs_en = false;
3667 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3668 } else
3669 adev->ucode_sysfs_en = true;
3670
3671 if ((amdgpu_testing & 1)) {
3672 if (adev->accel_working)
3673 amdgpu_test_moves(adev);
3674 else
3675 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
3676 }
3677 if (amdgpu_benchmarking) {
3678 if (adev->accel_working)
3679 amdgpu_benchmark(adev, amdgpu_benchmarking);
3680 else
3681 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
3682 }
3683
3684 /*
3685 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3686 * Otherwise the mgpu fan boost feature will be skipped due to the
3687 * gpu instance is counted less.
3688 */
3689 amdgpu_register_gpu_instance(adev);
3690
3691 /* enable clockgating, etc. after ib tests, etc. since some blocks require
3692 * explicit gating rather than handling it automatically.
3693 */
3694 if (!adev->gmc.xgmi.pending_reset) {
3695 r = amdgpu_device_ip_late_init(adev);
3696 if (r) {
3697 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3698 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3699 goto release_ras_con;
3700 }
3701 /* must succeed. */
3702 amdgpu_ras_resume(adev);
3703 queue_delayed_work(system_wq, &adev->delayed_init_work,
3704 msecs_to_jiffies(AMDGPU_RESUME_MS));
3705 }
3706
3707 if (amdgpu_sriov_vf(adev))
3708 flush_delayed_work(&adev->delayed_init_work);
3709
3710 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3711 if (r)
3712 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3713
3714 if (IS_ENABLED(CONFIG_PERF_EVENTS))
3715 r = amdgpu_pmu_init(adev);
3716 if (r)
3717 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3718
3719 /* Have stored pci confspace at hand for restore in sudden PCI error */
3720 if (amdgpu_device_cache_pci_state(adev->pdev))
3721 pci_restore_state(pdev);
3722
3723 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3724 /* this will fail for cards that aren't VGA class devices, just
3725 * ignore it */
3726 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3727 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
3728
3729 if (amdgpu_device_supports_px(ddev)) {
3730 px = true;
3731 vga_switcheroo_register_client(adev->pdev,
3732 &amdgpu_switcheroo_ops, px);
3733 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3734 }
3735
3736 if (adev->gmc.xgmi.pending_reset)
3737 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3738 msecs_to_jiffies(AMDGPU_RESUME_MS));
3739
3740 return 0;
3741
3742release_ras_con:
3743 amdgpu_release_ras_context(adev);
3744
3745failed:
3746 amdgpu_vf_error_trans_all(adev);
3747
3748 return r;
3749}
3750
3751static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3752{
3753 /* Clear all CPU mappings pointing to this device */
3754 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3755
3756 /* Unmap all mapped bars - Doorbell, registers and VRAM */
3757 amdgpu_device_doorbell_fini(adev);
3758
3759 iounmap(adev->rmmio);
3760 adev->rmmio = NULL;
3761 if (adev->mman.aper_base_kaddr)
3762 iounmap(adev->mman.aper_base_kaddr);
3763 adev->mman.aper_base_kaddr = NULL;
3764
3765 /* Memory manager related */
3766 if (!adev->gmc.xgmi.connected_to_cpu) {
3767 arch_phys_wc_del(adev->gmc.vram_mtrr);
3768 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
3769 }
3770}
3771
3772/**
3773 * amdgpu_device_fini - tear down the driver
3774 *
3775 * @adev: amdgpu_device pointer
3776 *
3777 * Tear down the driver info (all asics).
3778 * Called at driver shutdown.
3779 */
3780void amdgpu_device_fini_hw(struct amdgpu_device *adev)
3781{
3782 dev_info(adev->dev, "amdgpu: finishing device.\n");
3783 flush_delayed_work(&adev->delayed_init_work);
3784 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3785 adev->shutdown = true;
3786
3787 /* make sure IB test finished before entering exclusive mode
3788 * to avoid preemption on IB test
3789 * */
3790 if (amdgpu_sriov_vf(adev)) {
3791 amdgpu_virt_request_full_gpu(adev, false);
3792 amdgpu_virt_fini_data_exchange(adev);
3793 }
3794
3795 /* disable all interrupts */
3796 amdgpu_irq_disable_all(adev);
3797 if (adev->mode_info.mode_config_initialized){
3798 if (!amdgpu_device_has_dc_support(adev))
3799 drm_helper_force_disable_all(adev_to_drm(adev));
3800 else
3801 drm_atomic_helper_shutdown(adev_to_drm(adev));
3802 }
3803 amdgpu_fence_driver_hw_fini(adev);
3804
3805 if (adev->pm_sysfs_en)
3806 amdgpu_pm_sysfs_fini(adev);
3807 if (adev->ucode_sysfs_en)
3808 amdgpu_ucode_sysfs_fini(adev);
3809 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
3810
3811 amdgpu_fbdev_fini(adev);
3812
3813 amdgpu_irq_fini_hw(adev);
3814
3815 amdgpu_device_ip_fini_early(adev);
3816
3817 amdgpu_gart_dummy_page_fini(adev);
3818
3819 amdgpu_device_unmap_mmio(adev);
3820}
3821
3822void amdgpu_device_fini_sw(struct amdgpu_device *adev)
3823{
3824 amdgpu_device_ip_fini(adev);
3825 amdgpu_fence_driver_sw_fini(adev);
3826 release_firmware(adev->firmware.gpu_info_fw);
3827 adev->firmware.gpu_info_fw = NULL;
3828 adev->accel_working = false;
3829
3830 amdgpu_reset_fini(adev);
3831
3832 /* free i2c buses */
3833 if (!amdgpu_device_has_dc_support(adev))
3834 amdgpu_i2c_fini(adev);
3835
3836 if (amdgpu_emu_mode != 1)
3837 amdgpu_atombios_fini(adev);
3838
3839 kfree(adev->bios);
3840 adev->bios = NULL;
3841 if (amdgpu_device_supports_px(adev_to_drm(adev))) {
3842 vga_switcheroo_unregister_client(adev->pdev);
3843 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3844 }
3845 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3846 vga_client_register(adev->pdev, NULL, NULL, NULL);
3847
3848 if (IS_ENABLED(CONFIG_PERF_EVENTS))
3849 amdgpu_pmu_fini(adev);
3850 if (adev->mman.discovery_bin)
3851 amdgpu_discovery_fini(adev);
3852
3853 kfree(adev->pci_state);
3854
3855}
3856
3857
3858/*
3859 * Suspend & resume.
3860 */
3861/**
3862 * amdgpu_device_suspend - initiate device suspend
3863 *
3864 * @dev: drm dev pointer
3865 * @fbcon : notify the fbdev of suspend
3866 *
3867 * Puts the hw in the suspend state (all asics).
3868 * Returns 0 for success or an error on failure.
3869 * Called at driver suspend.
3870 */
3871int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
3872{
3873 struct amdgpu_device *adev = drm_to_adev(dev);
3874
3875 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3876 return 0;
3877
3878 adev->in_suspend = true;
3879
3880 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
3881 DRM_WARN("smart shift update failed\n");
3882
3883 drm_kms_helper_poll_disable(dev);
3884
3885 if (fbcon)
3886 amdgpu_fbdev_set_suspend(adev, 1);
3887
3888 cancel_delayed_work_sync(&adev->delayed_init_work);
3889
3890 amdgpu_ras_suspend(adev);
3891
3892 amdgpu_device_ip_suspend_phase1(adev);
3893
3894 if (!adev->in_s0ix)
3895 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
3896
3897 /* evict vram memory */
3898 amdgpu_bo_evict_vram(adev);
3899
3900 amdgpu_fence_driver_hw_fini(adev);
3901
3902 amdgpu_device_ip_suspend_phase2(adev);
3903 /* evict remaining vram memory
3904 * This second call to evict vram is to evict the gart page table
3905 * using the CPU.
3906 */
3907 amdgpu_bo_evict_vram(adev);
3908
3909 return 0;
3910}
3911
3912/**
3913 * amdgpu_device_resume - initiate device resume
3914 *
3915 * @dev: drm dev pointer
3916 * @fbcon : notify the fbdev of resume
3917 *
3918 * Bring the hw back to operating state (all asics).
3919 * Returns 0 for success or an error on failure.
3920 * Called at driver resume.
3921 */
3922int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
3923{
3924 struct amdgpu_device *adev = drm_to_adev(dev);
3925 int r = 0;
3926
3927 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3928 return 0;
3929
3930 if (adev->in_s0ix)
3931 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D0Entry);
3932
3933 /* post card */
3934 if (amdgpu_device_need_post(adev)) {
3935 r = amdgpu_device_asic_init(adev);
3936 if (r)
3937 dev_err(adev->dev, "amdgpu asic init failed\n");
3938 }
3939
3940 r = amdgpu_device_ip_resume(adev);
3941 if (r) {
3942 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
3943 return r;
3944 }
3945 amdgpu_fence_driver_hw_init(adev);
3946
3947 r = amdgpu_device_ip_late_init(adev);
3948 if (r)
3949 return r;
3950
3951 queue_delayed_work(system_wq, &adev->delayed_init_work,
3952 msecs_to_jiffies(AMDGPU_RESUME_MS));
3953
3954 if (!adev->in_s0ix) {
3955 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
3956 if (r)
3957 return r;
3958 }
3959
3960 /* Make sure IB tests flushed */
3961 flush_delayed_work(&adev->delayed_init_work);
3962
3963 if (fbcon)
3964 amdgpu_fbdev_set_suspend(adev, 0);
3965
3966 drm_kms_helper_poll_enable(dev);
3967
3968 amdgpu_ras_resume(adev);
3969
3970 /*
3971 * Most of the connector probing functions try to acquire runtime pm
3972 * refs to ensure that the GPU is powered on when connector polling is
3973 * performed. Since we're calling this from a runtime PM callback,
3974 * trying to acquire rpm refs will cause us to deadlock.
3975 *
3976 * Since we're guaranteed to be holding the rpm lock, it's safe to
3977 * temporarily disable the rpm helpers so this doesn't deadlock us.
3978 */
3979#ifdef CONFIG_PM
3980 dev->dev->power.disable_depth++;
3981#endif
3982 if (!amdgpu_device_has_dc_support(adev))
3983 drm_helper_hpd_irq_event(dev);
3984 else
3985 drm_kms_helper_hotplug_event(dev);
3986#ifdef CONFIG_PM
3987 dev->dev->power.disable_depth--;
3988#endif
3989 adev->in_suspend = false;
3990
3991 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
3992 DRM_WARN("smart shift update failed\n");
3993
3994 return 0;
3995}
3996
3997/**
3998 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3999 *
4000 * @adev: amdgpu_device pointer
4001 *
4002 * The list of all the hardware IPs that make up the asic is walked and
4003 * the check_soft_reset callbacks are run. check_soft_reset determines
4004 * if the asic is still hung or not.
4005 * Returns true if any of the IPs are still in a hung state, false if not.
4006 */
4007static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4008{
4009 int i;
4010 bool asic_hang = false;
4011
4012 if (amdgpu_sriov_vf(adev))
4013 return true;
4014
4015 if (amdgpu_asic_need_full_reset(adev))
4016 return true;
4017
4018 for (i = 0; i < adev->num_ip_blocks; i++) {
4019 if (!adev->ip_blocks[i].status.valid)
4020 continue;
4021 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4022 adev->ip_blocks[i].status.hang =
4023 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4024 if (adev->ip_blocks[i].status.hang) {
4025 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4026 asic_hang = true;
4027 }
4028 }
4029 return asic_hang;
4030}
4031
4032/**
4033 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4034 *
4035 * @adev: amdgpu_device pointer
4036 *
4037 * The list of all the hardware IPs that make up the asic is walked and the
4038 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
4039 * handles any IP specific hardware or software state changes that are
4040 * necessary for a soft reset to succeed.
4041 * Returns 0 on success, negative error code on failure.
4042 */
4043static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4044{
4045 int i, r = 0;
4046
4047 for (i = 0; i < adev->num_ip_blocks; i++) {
4048 if (!adev->ip_blocks[i].status.valid)
4049 continue;
4050 if (adev->ip_blocks[i].status.hang &&
4051 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4052 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4053 if (r)
4054 return r;
4055 }
4056 }
4057
4058 return 0;
4059}
4060
4061/**
4062 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4063 *
4064 * @adev: amdgpu_device pointer
4065 *
4066 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
4067 * reset is necessary to recover.
4068 * Returns true if a full asic reset is required, false if not.
4069 */
4070static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4071{
4072 int i;
4073
4074 if (amdgpu_asic_need_full_reset(adev))
4075 return true;
4076
4077 for (i = 0; i < adev->num_ip_blocks; i++) {
4078 if (!adev->ip_blocks[i].status.valid)
4079 continue;
4080 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4081 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4082 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4083 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4084 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4085 if (adev->ip_blocks[i].status.hang) {
4086 dev_info(adev->dev, "Some block need full reset!\n");
4087 return true;
4088 }
4089 }
4090 }
4091 return false;
4092}
4093
4094/**
4095 * amdgpu_device_ip_soft_reset - do a soft reset
4096 *
4097 * @adev: amdgpu_device pointer
4098 *
4099 * The list of all the hardware IPs that make up the asic is walked and the
4100 * soft_reset callbacks are run if the block is hung. soft_reset handles any
4101 * IP specific hardware or software state changes that are necessary to soft
4102 * reset the IP.
4103 * Returns 0 on success, negative error code on failure.
4104 */
4105static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4106{
4107 int i, r = 0;
4108
4109 for (i = 0; i < adev->num_ip_blocks; i++) {
4110 if (!adev->ip_blocks[i].status.valid)
4111 continue;
4112 if (adev->ip_blocks[i].status.hang &&
4113 adev->ip_blocks[i].version->funcs->soft_reset) {
4114 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4115 if (r)
4116 return r;
4117 }
4118 }
4119
4120 return 0;
4121}
4122
4123/**
4124 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4125 *
4126 * @adev: amdgpu_device pointer
4127 *
4128 * The list of all the hardware IPs that make up the asic is walked and the
4129 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
4130 * handles any IP specific hardware or software state changes that are
4131 * necessary after the IP has been soft reset.
4132 * Returns 0 on success, negative error code on failure.
4133 */
4134static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4135{
4136 int i, r = 0;
4137
4138 for (i = 0; i < adev->num_ip_blocks; i++) {
4139 if (!adev->ip_blocks[i].status.valid)
4140 continue;
4141 if (adev->ip_blocks[i].status.hang &&
4142 adev->ip_blocks[i].version->funcs->post_soft_reset)
4143 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4144 if (r)
4145 return r;
4146 }
4147
4148 return 0;
4149}
4150
4151/**
4152 * amdgpu_device_recover_vram - Recover some VRAM contents
4153 *
4154 * @adev: amdgpu_device pointer
4155 *
4156 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
4157 * restore things like GPUVM page tables after a GPU reset where
4158 * the contents of VRAM might be lost.
4159 *
4160 * Returns:
4161 * 0 on success, negative error code on failure.
4162 */
4163static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4164{
4165 struct dma_fence *fence = NULL, *next = NULL;
4166 struct amdgpu_bo *shadow;
4167 struct amdgpu_bo_vm *vmbo;
4168 long r = 1, tmo;
4169
4170 if (amdgpu_sriov_runtime(adev))
4171 tmo = msecs_to_jiffies(8000);
4172 else
4173 tmo = msecs_to_jiffies(100);
4174
4175 dev_info(adev->dev, "recover vram bo from shadow start\n");
4176 mutex_lock(&adev->shadow_list_lock);
4177 list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4178 shadow = &vmbo->bo;
4179 /* No need to recover an evicted BO */
4180 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4181 shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4182 shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4183 continue;
4184
4185 r = amdgpu_bo_restore_shadow(shadow, &next);
4186 if (r)
4187 break;
4188
4189 if (fence) {
4190 tmo = dma_fence_wait_timeout(fence, false, tmo);
4191 dma_fence_put(fence);
4192 fence = next;
4193 if (tmo == 0) {
4194 r = -ETIMEDOUT;
4195 break;
4196 } else if (tmo < 0) {
4197 r = tmo;
4198 break;
4199 }
4200 } else {
4201 fence = next;
4202 }
4203 }
4204 mutex_unlock(&adev->shadow_list_lock);
4205
4206 if (fence)
4207 tmo = dma_fence_wait_timeout(fence, false, tmo);
4208 dma_fence_put(fence);
4209
4210 if (r < 0 || tmo <= 0) {
4211 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4212 return -EIO;
4213 }
4214
4215 dev_info(adev->dev, "recover vram bo from shadow done\n");
4216 return 0;
4217}
4218
4219
4220/**
4221 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4222 *
4223 * @adev: amdgpu_device pointer
4224 * @from_hypervisor: request from hypervisor
4225 *
4226 * do VF FLR and reinitialize Asic
4227 * return 0 means succeeded otherwise failed
4228 */
4229static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4230 bool from_hypervisor)
4231{
4232 int r;
4233
4234 if (from_hypervisor)
4235 r = amdgpu_virt_request_full_gpu(adev, true);
4236 else
4237 r = amdgpu_virt_reset_gpu(adev);
4238 if (r)
4239 return r;
4240
4241 amdgpu_amdkfd_pre_reset(adev);
4242
4243 /* Resume IP prior to SMC */
4244 r = amdgpu_device_ip_reinit_early_sriov(adev);
4245 if (r)
4246 goto error;
4247
4248 amdgpu_virt_init_data_exchange(adev);
4249 /* we need recover gart prior to run SMC/CP/SDMA resume */
4250 amdgpu_gtt_mgr_recover(ttm_manager_type(&adev->mman.bdev, TTM_PL_TT));
4251
4252 r = amdgpu_device_fw_loading(adev);
4253 if (r)
4254 return r;
4255
4256 /* now we are okay to resume SMC/CP/SDMA */
4257 r = amdgpu_device_ip_reinit_late_sriov(adev);
4258 if (r)
4259 goto error;
4260
4261 amdgpu_irq_gpu_reset_resume_helper(adev);
4262 r = amdgpu_ib_ring_tests(adev);
4263 amdgpu_amdkfd_post_reset(adev);
4264
4265error:
4266 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4267 amdgpu_inc_vram_lost(adev);
4268 r = amdgpu_device_recover_vram(adev);
4269 }
4270 amdgpu_virt_release_full_gpu(adev, true);
4271
4272 return r;
4273}
4274
4275/**
4276 * amdgpu_device_has_job_running - check if there is any job in mirror list
4277 *
4278 * @adev: amdgpu_device pointer
4279 *
4280 * check if there is any job in mirror list
4281 */
4282bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4283{
4284 int i;
4285 struct drm_sched_job *job;
4286
4287 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4288 struct amdgpu_ring *ring = adev->rings[i];
4289
4290 if (!ring || !ring->sched.thread)
4291 continue;
4292
4293 spin_lock(&ring->sched.job_list_lock);
4294 job = list_first_entry_or_null(&ring->sched.pending_list,
4295 struct drm_sched_job, list);
4296 spin_unlock(&ring->sched.job_list_lock);
4297 if (job)
4298 return true;
4299 }
4300 return false;
4301}
4302
4303/**
4304 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4305 *
4306 * @adev: amdgpu_device pointer
4307 *
4308 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4309 * a hung GPU.
4310 */
4311bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4312{
4313 if (!amdgpu_device_ip_check_soft_reset(adev)) {
4314 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n");
4315 return false;
4316 }
4317
4318 if (amdgpu_gpu_recovery == 0)
4319 goto disabled;
4320
4321 if (amdgpu_sriov_vf(adev))
4322 return true;
4323
4324 if (amdgpu_gpu_recovery == -1) {
4325 switch (adev->asic_type) {
4326 case CHIP_BONAIRE:
4327 case CHIP_HAWAII:
4328 case CHIP_TOPAZ:
4329 case CHIP_TONGA:
4330 case CHIP_FIJI:
4331 case CHIP_POLARIS10:
4332 case CHIP_POLARIS11:
4333 case CHIP_POLARIS12:
4334 case CHIP_VEGAM:
4335 case CHIP_VEGA20:
4336 case CHIP_VEGA10:
4337 case CHIP_VEGA12:
4338 case CHIP_RAVEN:
4339 case CHIP_ARCTURUS:
4340 case CHIP_RENOIR:
4341 case CHIP_NAVI10:
4342 case CHIP_NAVI14:
4343 case CHIP_NAVI12:
4344 case CHIP_SIENNA_CICHLID:
4345 case CHIP_NAVY_FLOUNDER:
4346 case CHIP_DIMGREY_CAVEFISH:
4347 case CHIP_BEIGE_GOBY:
4348 case CHIP_VANGOGH:
4349 case CHIP_ALDEBARAN:
4350 break;
4351 default:
4352 goto disabled;
4353 }
4354 }
4355
4356 return true;
4357
4358disabled:
4359 dev_info(adev->dev, "GPU recovery disabled.\n");
4360 return false;
4361}
4362
4363int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4364{
4365 u32 i;
4366 int ret = 0;
4367
4368 amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4369
4370 dev_info(adev->dev, "GPU mode1 reset\n");
4371
4372 /* disable BM */
4373 pci_clear_master(adev->pdev);
4374
4375 amdgpu_device_cache_pci_state(adev->pdev);
4376
4377 if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4378 dev_info(adev->dev, "GPU smu mode1 reset\n");
4379 ret = amdgpu_dpm_mode1_reset(adev);
4380 } else {
4381 dev_info(adev->dev, "GPU psp mode1 reset\n");
4382 ret = psp_gpu_reset(adev);
4383 }
4384
4385 if (ret)
4386 dev_err(adev->dev, "GPU mode1 reset failed\n");
4387
4388 amdgpu_device_load_pci_state(adev->pdev);
4389
4390 /* wait for asic to come out of reset */
4391 for (i = 0; i < adev->usec_timeout; i++) {
4392 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4393
4394 if (memsize != 0xffffffff)
4395 break;
4396 udelay(1);
4397 }
4398
4399 amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4400 return ret;
4401}
4402
4403int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4404 struct amdgpu_reset_context *reset_context)
4405{
4406 int i, r = 0;
4407 struct amdgpu_job *job = NULL;
4408 bool need_full_reset =
4409 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4410
4411 if (reset_context->reset_req_dev == adev)
4412 job = reset_context->job;
4413
4414 /* no need to dump if device is not in good state during probe period */
4415 if (!adev->gmc.xgmi.pending_reset)
4416 amdgpu_debugfs_wait_dump(adev);
4417
4418 if (amdgpu_sriov_vf(adev)) {
4419 /* stop the data exchange thread */
4420 amdgpu_virt_fini_data_exchange(adev);
4421 }
4422
4423 /* block all schedulers and reset given job's ring */
4424 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4425 struct amdgpu_ring *ring = adev->rings[i];
4426
4427 if (!ring || !ring->sched.thread)
4428 continue;
4429
4430 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4431 amdgpu_fence_driver_force_completion(ring);
4432 }
4433
4434 if(job)
4435 drm_sched_increase_karma(&job->base);
4436
4437 r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4438 /* If reset handler not implemented, continue; otherwise return */
4439 if (r == -ENOSYS)
4440 r = 0;
4441 else
4442 return r;
4443
4444 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4445 if (!amdgpu_sriov_vf(adev)) {
4446
4447 if (!need_full_reset)
4448 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4449
4450 if (!need_full_reset) {
4451 amdgpu_device_ip_pre_soft_reset(adev);
4452 r = amdgpu_device_ip_soft_reset(adev);
4453 amdgpu_device_ip_post_soft_reset(adev);
4454 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4455 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4456 need_full_reset = true;
4457 }
4458 }
4459
4460 if (need_full_reset)
4461 r = amdgpu_device_ip_suspend(adev);
4462 if (need_full_reset)
4463 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4464 else
4465 clear_bit(AMDGPU_NEED_FULL_RESET,
4466 &reset_context->flags);
4467 }
4468
4469 return r;
4470}
4471
4472int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4473 struct amdgpu_reset_context *reset_context)
4474{
4475 struct amdgpu_device *tmp_adev = NULL;
4476 bool need_full_reset, skip_hw_reset, vram_lost = false;
4477 int r = 0;
4478
4479 /* Try reset handler method first */
4480 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4481 reset_list);
4482 r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4483 /* If reset handler not implemented, continue; otherwise return */
4484 if (r == -ENOSYS)
4485 r = 0;
4486 else
4487 return r;
4488
4489 /* Reset handler not implemented, use the default method */
4490 need_full_reset =
4491 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4492 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4493
4494 /*
4495 * ASIC reset has to be done on all XGMI hive nodes ASAP
4496 * to allow proper links negotiation in FW (within 1 sec)
4497 */
4498 if (!skip_hw_reset && need_full_reset) {
4499 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4500 /* For XGMI run all resets in parallel to speed up the process */
4501 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4502 tmp_adev->gmc.xgmi.pending_reset = false;
4503 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4504 r = -EALREADY;
4505 } else
4506 r = amdgpu_asic_reset(tmp_adev);
4507
4508 if (r) {
4509 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4510 r, adev_to_drm(tmp_adev)->unique);
4511 break;
4512 }
4513 }
4514
4515 /* For XGMI wait for all resets to complete before proceed */
4516 if (!r) {
4517 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4518 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4519 flush_work(&tmp_adev->xgmi_reset_work);
4520 r = tmp_adev->asic_reset_res;
4521 if (r)
4522 break;
4523 }
4524 }
4525 }
4526 }
4527
4528 if (!r && amdgpu_ras_intr_triggered()) {
4529 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4530 if (tmp_adev->mmhub.ras_funcs &&
4531 tmp_adev->mmhub.ras_funcs->reset_ras_error_count)
4532 tmp_adev->mmhub.ras_funcs->reset_ras_error_count(tmp_adev);
4533 }
4534
4535 amdgpu_ras_intr_cleared();
4536 }
4537
4538 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4539 if (need_full_reset) {
4540 /* post card */
4541 r = amdgpu_device_asic_init(tmp_adev);
4542 if (r) {
4543 dev_warn(tmp_adev->dev, "asic atom init failed!");
4544 } else {
4545 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4546 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4547 if (r)
4548 goto out;
4549
4550 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4551 if (r)
4552 goto out;
4553
4554 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4555 if (vram_lost) {
4556 DRM_INFO("VRAM is lost due to GPU reset!\n");
4557 amdgpu_inc_vram_lost(tmp_adev);
4558 }
4559
4560 r = amdgpu_gtt_mgr_recover(ttm_manager_type(&tmp_adev->mman.bdev, TTM_PL_TT));
4561 if (r)
4562 goto out;
4563
4564 r = amdgpu_device_fw_loading(tmp_adev);
4565 if (r)
4566 return r;
4567
4568 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4569 if (r)
4570 goto out;
4571
4572 if (vram_lost)
4573 amdgpu_device_fill_reset_magic(tmp_adev);
4574
4575 /*
4576 * Add this ASIC as tracked as reset was already
4577 * complete successfully.
4578 */
4579 amdgpu_register_gpu_instance(tmp_adev);
4580
4581 if (!reset_context->hive &&
4582 tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4583 amdgpu_xgmi_add_device(tmp_adev);
4584
4585 r = amdgpu_device_ip_late_init(tmp_adev);
4586 if (r)
4587 goto out;
4588
4589 amdgpu_fbdev_set_suspend(tmp_adev, 0);
4590
4591 /*
4592 * The GPU enters bad state once faulty pages
4593 * by ECC has reached the threshold, and ras
4594 * recovery is scheduled next. So add one check
4595 * here to break recovery if it indeed exceeds
4596 * bad page threshold, and remind user to
4597 * retire this GPU or setting one bigger
4598 * bad_page_threshold value to fix this once
4599 * probing driver again.
4600 */
4601 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4602 /* must succeed. */
4603 amdgpu_ras_resume(tmp_adev);
4604 } else {
4605 r = -EINVAL;
4606 goto out;
4607 }
4608
4609 /* Update PSP FW topology after reset */
4610 if (reset_context->hive &&
4611 tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4612 r = amdgpu_xgmi_update_topology(
4613 reset_context->hive, tmp_adev);
4614 }
4615 }
4616
4617out:
4618 if (!r) {
4619 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4620 r = amdgpu_ib_ring_tests(tmp_adev);
4621 if (r) {
4622 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
4623 need_full_reset = true;
4624 r = -EAGAIN;
4625 goto end;
4626 }
4627 }
4628
4629 if (!r)
4630 r = amdgpu_device_recover_vram(tmp_adev);
4631 else
4632 tmp_adev->asic_reset_res = r;
4633 }
4634
4635end:
4636 if (need_full_reset)
4637 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4638 else
4639 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4640 return r;
4641}
4642
4643static bool amdgpu_device_lock_adev(struct amdgpu_device *adev,
4644 struct amdgpu_hive_info *hive)
4645{
4646 if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0)
4647 return false;
4648
4649 if (hive) {
4650 down_write_nest_lock(&adev->reset_sem, &hive->hive_lock);
4651 } else {
4652 down_write(&adev->reset_sem);
4653 }
4654
4655 switch (amdgpu_asic_reset_method(adev)) {
4656 case AMD_RESET_METHOD_MODE1:
4657 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
4658 break;
4659 case AMD_RESET_METHOD_MODE2:
4660 adev->mp1_state = PP_MP1_STATE_RESET;
4661 break;
4662 default:
4663 adev->mp1_state = PP_MP1_STATE_NONE;
4664 break;
4665 }
4666
4667 return true;
4668}
4669
4670static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
4671{
4672 amdgpu_vf_error_trans_all(adev);
4673 adev->mp1_state = PP_MP1_STATE_NONE;
4674 atomic_set(&adev->in_gpu_reset, 0);
4675 up_write(&adev->reset_sem);
4676}
4677
4678/*
4679 * to lockup a list of amdgpu devices in a hive safely, if not a hive
4680 * with multiple nodes, it will be similar as amdgpu_device_lock_adev.
4681 *
4682 * unlock won't require roll back.
4683 */
4684static int amdgpu_device_lock_hive_adev(struct amdgpu_device *adev, struct amdgpu_hive_info *hive)
4685{
4686 struct amdgpu_device *tmp_adev = NULL;
4687
4688 if (adev->gmc.xgmi.num_physical_nodes > 1) {
4689 if (!hive) {
4690 dev_err(adev->dev, "Hive is NULL while device has multiple xgmi nodes");
4691 return -ENODEV;
4692 }
4693 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4694 if (!amdgpu_device_lock_adev(tmp_adev, hive))
4695 goto roll_back;
4696 }
4697 } else if (!amdgpu_device_lock_adev(adev, hive))
4698 return -EAGAIN;
4699
4700 return 0;
4701roll_back:
4702 if (!list_is_first(&tmp_adev->gmc.xgmi.head, &hive->device_list)) {
4703 /*
4704 * if the lockup iteration break in the middle of a hive,
4705 * it may means there may has a race issue,
4706 * or a hive device locked up independently.
4707 * we may be in trouble and may not, so will try to roll back
4708 * the lock and give out a warnning.
4709 */
4710 dev_warn(tmp_adev->dev, "Hive lock iteration broke in the middle. Rolling back to unlock");
4711 list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4712 amdgpu_device_unlock_adev(tmp_adev);
4713 }
4714 }
4715 return -EAGAIN;
4716}
4717
4718static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
4719{
4720 struct pci_dev *p = NULL;
4721
4722 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4723 adev->pdev->bus->number, 1);
4724 if (p) {
4725 pm_runtime_enable(&(p->dev));
4726 pm_runtime_resume(&(p->dev));
4727 }
4728}
4729
4730static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
4731{
4732 enum amd_reset_method reset_method;
4733 struct pci_dev *p = NULL;
4734 u64 expires;
4735
4736 /*
4737 * For now, only BACO and mode1 reset are confirmed
4738 * to suffer the audio issue without proper suspended.
4739 */
4740 reset_method = amdgpu_asic_reset_method(adev);
4741 if ((reset_method != AMD_RESET_METHOD_BACO) &&
4742 (reset_method != AMD_RESET_METHOD_MODE1))
4743 return -EINVAL;
4744
4745 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4746 adev->pdev->bus->number, 1);
4747 if (!p)
4748 return -ENODEV;
4749
4750 expires = pm_runtime_autosuspend_expiration(&(p->dev));
4751 if (!expires)
4752 /*
4753 * If we cannot get the audio device autosuspend delay,
4754 * a fixed 4S interval will be used. Considering 3S is
4755 * the audio controller default autosuspend delay setting.
4756 * 4S used here is guaranteed to cover that.
4757 */
4758 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
4759
4760 while (!pm_runtime_status_suspended(&(p->dev))) {
4761 if (!pm_runtime_suspend(&(p->dev)))
4762 break;
4763
4764 if (expires < ktime_get_mono_fast_ns()) {
4765 dev_warn(adev->dev, "failed to suspend display audio\n");
4766 /* TODO: abort the succeeding gpu reset? */
4767 return -ETIMEDOUT;
4768 }
4769 }
4770
4771 pm_runtime_disable(&(p->dev));
4772
4773 return 0;
4774}
4775
4776static void amdgpu_device_recheck_guilty_jobs(
4777 struct amdgpu_device *adev, struct list_head *device_list_handle,
4778 struct amdgpu_reset_context *reset_context)
4779{
4780 int i, r = 0;
4781
4782 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4783 struct amdgpu_ring *ring = adev->rings[i];
4784 int ret = 0;
4785 struct drm_sched_job *s_job;
4786
4787 if (!ring || !ring->sched.thread)
4788 continue;
4789
4790 s_job = list_first_entry_or_null(&ring->sched.pending_list,
4791 struct drm_sched_job, list);
4792 if (s_job == NULL)
4793 continue;
4794
4795 /* clear job's guilty and depend the folowing step to decide the real one */
4796 drm_sched_reset_karma(s_job);
4797 drm_sched_resubmit_jobs_ext(&ring->sched, 1);
4798
4799 ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout);
4800 if (ret == 0) { /* timeout */
4801 DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n",
4802 ring->sched.name, s_job->id);
4803
4804 /* set guilty */
4805 drm_sched_increase_karma(s_job);
4806retry:
4807 /* do hw reset */
4808 if (amdgpu_sriov_vf(adev)) {
4809 amdgpu_virt_fini_data_exchange(adev);
4810 r = amdgpu_device_reset_sriov(adev, false);
4811 if (r)
4812 adev->asic_reset_res = r;
4813 } else {
4814 clear_bit(AMDGPU_SKIP_HW_RESET,
4815 &reset_context->flags);
4816 r = amdgpu_do_asic_reset(device_list_handle,
4817 reset_context);
4818 if (r && r == -EAGAIN)
4819 goto retry;
4820 }
4821
4822 /*
4823 * add reset counter so that the following
4824 * resubmitted job could flush vmid
4825 */
4826 atomic_inc(&adev->gpu_reset_counter);
4827 continue;
4828 }
4829
4830 /* got the hw fence, signal finished fence */
4831 atomic_dec(ring->sched.score);
4832 dma_fence_get(&s_job->s_fence->finished);
4833 dma_fence_signal(&s_job->s_fence->finished);
4834 dma_fence_put(&s_job->s_fence->finished);
4835
4836 /* remove node from list and free the job */
4837 spin_lock(&ring->sched.job_list_lock);
4838 list_del_init(&s_job->list);
4839 spin_unlock(&ring->sched.job_list_lock);
4840 ring->sched.ops->free_job(s_job);
4841 }
4842}
4843
4844/**
4845 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4846 *
4847 * @adev: amdgpu_device pointer
4848 * @job: which job trigger hang
4849 *
4850 * Attempt to reset the GPU if it has hung (all asics).
4851 * Attempt to do soft-reset or full-reset and reinitialize Asic
4852 * Returns 0 for success or an error on failure.
4853 */
4854
4855int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
4856 struct amdgpu_job *job)
4857{
4858 struct list_head device_list, *device_list_handle = NULL;
4859 bool job_signaled = false;
4860 struct amdgpu_hive_info *hive = NULL;
4861 struct amdgpu_device *tmp_adev = NULL;
4862 int i, r = 0;
4863 bool need_emergency_restart = false;
4864 bool audio_suspended = false;
4865 int tmp_vram_lost_counter;
4866 struct amdgpu_reset_context reset_context;
4867
4868 memset(&reset_context, 0, sizeof(reset_context));
4869
4870 /*
4871 * Special case: RAS triggered and full reset isn't supported
4872 */
4873 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
4874
4875 /*
4876 * Flush RAM to disk so that after reboot
4877 * the user can read log and see why the system rebooted.
4878 */
4879 if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
4880 DRM_WARN("Emergency reboot.");
4881
4882 ksys_sync_helper();
4883 emergency_restart();
4884 }
4885
4886 dev_info(adev->dev, "GPU %s begin!\n",
4887 need_emergency_restart ? "jobs stop":"reset");
4888
4889 /*
4890 * Here we trylock to avoid chain of resets executing from
4891 * either trigger by jobs on different adevs in XGMI hive or jobs on
4892 * different schedulers for same device while this TO handler is running.
4893 * We always reset all schedulers for device and all devices for XGMI
4894 * hive so that should take care of them too.
4895 */
4896 hive = amdgpu_get_xgmi_hive(adev);
4897 if (hive) {
4898 if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) {
4899 DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
4900 job ? job->base.id : -1, hive->hive_id);
4901 amdgpu_put_xgmi_hive(hive);
4902 if (job)
4903 drm_sched_increase_karma(&job->base);
4904 return 0;
4905 }
4906 mutex_lock(&hive->hive_lock);
4907 }
4908
4909 reset_context.method = AMD_RESET_METHOD_NONE;
4910 reset_context.reset_req_dev = adev;
4911 reset_context.job = job;
4912 reset_context.hive = hive;
4913 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
4914
4915 /*
4916 * lock the device before we try to operate the linked list
4917 * if didn't get the device lock, don't touch the linked list since
4918 * others may iterating it.
4919 */
4920 r = amdgpu_device_lock_hive_adev(adev, hive);
4921 if (r) {
4922 dev_info(adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress",
4923 job ? job->base.id : -1);
4924
4925 /* even we skipped this reset, still need to set the job to guilty */
4926 if (job)
4927 drm_sched_increase_karma(&job->base);
4928 goto skip_recovery;
4929 }
4930
4931 /*
4932 * Build list of devices to reset.
4933 * In case we are in XGMI hive mode, resort the device list
4934 * to put adev in the 1st position.
4935 */
4936 INIT_LIST_HEAD(&device_list);
4937 if (adev->gmc.xgmi.num_physical_nodes > 1) {
4938 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
4939 list_add_tail(&tmp_adev->reset_list, &device_list);
4940 if (!list_is_first(&adev->reset_list, &device_list))
4941 list_rotate_to_front(&adev->reset_list, &device_list);
4942 device_list_handle = &device_list;
4943 } else {
4944 list_add_tail(&adev->reset_list, &device_list);
4945 device_list_handle = &device_list;
4946 }
4947
4948 /* block all schedulers and reset given job's ring */
4949 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4950 /*
4951 * Try to put the audio codec into suspend state
4952 * before gpu reset started.
4953 *
4954 * Due to the power domain of the graphics device
4955 * is shared with AZ power domain. Without this,
4956 * we may change the audio hardware from behind
4957 * the audio driver's back. That will trigger
4958 * some audio codec errors.
4959 */
4960 if (!amdgpu_device_suspend_display_audio(tmp_adev))
4961 audio_suspended = true;
4962
4963 amdgpu_ras_set_error_query_ready(tmp_adev, false);
4964
4965 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
4966
4967 if (!amdgpu_sriov_vf(tmp_adev))
4968 amdgpu_amdkfd_pre_reset(tmp_adev);
4969
4970 /*
4971 * Mark these ASICs to be reseted as untracked first
4972 * And add them back after reset completed
4973 */
4974 amdgpu_unregister_gpu_instance(tmp_adev);
4975
4976 amdgpu_fbdev_set_suspend(tmp_adev, 1);
4977
4978 /* disable ras on ALL IPs */
4979 if (!need_emergency_restart &&
4980 amdgpu_device_ip_need_full_reset(tmp_adev))
4981 amdgpu_ras_suspend(tmp_adev);
4982
4983 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4984 struct amdgpu_ring *ring = tmp_adev->rings[i];
4985
4986 if (!ring || !ring->sched.thread)
4987 continue;
4988
4989 drm_sched_stop(&ring->sched, job ? &job->base : NULL);
4990
4991 if (need_emergency_restart)
4992 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
4993 }
4994 atomic_inc(&tmp_adev->gpu_reset_counter);
4995 }
4996
4997 if (need_emergency_restart)
4998 goto skip_sched_resume;
4999
5000 /*
5001 * Must check guilty signal here since after this point all old
5002 * HW fences are force signaled.
5003 *
5004 * job->base holds a reference to parent fence
5005 */
5006 if (job && job->base.s_fence->parent &&
5007 dma_fence_is_signaled(job->base.s_fence->parent)) {
5008 job_signaled = true;
5009 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5010 goto skip_hw_reset;
5011 }
5012
5013retry: /* Rest of adevs pre asic reset from XGMI hive. */
5014 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5015 r = amdgpu_device_pre_asic_reset(tmp_adev, &reset_context);
5016 /*TODO Should we stop ?*/
5017 if (r) {
5018 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5019 r, adev_to_drm(tmp_adev)->unique);
5020 tmp_adev->asic_reset_res = r;
5021 }
5022 }
5023
5024 tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));
5025 /* Actual ASIC resets if needed.*/
5026 /* TODO Implement XGMI hive reset logic for SRIOV */
5027 if (amdgpu_sriov_vf(adev)) {
5028 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5029 if (r)
5030 adev->asic_reset_res = r;
5031 } else {
5032 r = amdgpu_do_asic_reset(device_list_handle, &reset_context);
5033 if (r && r == -EAGAIN)
5034 goto retry;
5035 }
5036
5037skip_hw_reset:
5038
5039 /* Post ASIC reset for all devs .*/
5040 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5041
5042 /*
5043 * Sometimes a later bad compute job can block a good gfx job as gfx
5044 * and compute ring share internal GC HW mutually. We add an additional
5045 * guilty jobs recheck step to find the real guilty job, it synchronously
5046 * submits and pends for the first job being signaled. If it gets timeout,
5047 * we identify it as a real guilty job.
5048 */
5049 if (amdgpu_gpu_recovery == 2 &&
5050 !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter)))
5051 amdgpu_device_recheck_guilty_jobs(
5052 tmp_adev, device_list_handle, &reset_context);
5053
5054 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5055 struct amdgpu_ring *ring = tmp_adev->rings[i];
5056
5057 if (!ring || !ring->sched.thread)
5058 continue;
5059
5060 /* No point to resubmit jobs if we didn't HW reset*/
5061 if (!tmp_adev->asic_reset_res && !job_signaled)
5062 drm_sched_resubmit_jobs(&ring->sched);
5063
5064 drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
5065 }
5066
5067 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
5068 drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5069 }
5070
5071 tmp_adev->asic_reset_res = 0;
5072
5073 if (r) {
5074 /* bad news, how to tell it to userspace ? */
5075 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5076 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5077 } else {
5078 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5079 if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5080 DRM_WARN("smart shift update failed\n");
5081 }
5082 }
5083
5084skip_sched_resume:
5085 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5086 /* unlock kfd: SRIOV would do it separately */
5087 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5088 amdgpu_amdkfd_post_reset(tmp_adev);
5089
5090 /* kfd_post_reset will do nothing if kfd device is not initialized,
5091 * need to bring up kfd here if it's not be initialized before
5092 */
5093 if (!adev->kfd.init_complete)
5094 amdgpu_amdkfd_device_init(adev);
5095
5096 if (audio_suspended)
5097 amdgpu_device_resume_display_audio(tmp_adev);
5098 amdgpu_device_unlock_adev(tmp_adev);
5099 }
5100
5101skip_recovery:
5102 if (hive) {
5103 atomic_set(&hive->in_reset, 0);
5104 mutex_unlock(&hive->hive_lock);
5105 amdgpu_put_xgmi_hive(hive);
5106 }
5107
5108 if (r && r != -EAGAIN)
5109 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5110 return r;
5111}
5112
5113/**
5114 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5115 *
5116 * @adev: amdgpu_device pointer
5117 *
5118 * Fetchs and stores in the driver the PCIE capabilities (gen speed
5119 * and lanes) of the slot the device is in. Handles APUs and
5120 * virtualized environments where PCIE config space may not be available.
5121 */
5122static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5123{
5124 struct pci_dev *pdev;
5125 enum pci_bus_speed speed_cap, platform_speed_cap;
5126 enum pcie_link_width platform_link_width;
5127
5128 if (amdgpu_pcie_gen_cap)
5129 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5130
5131 if (amdgpu_pcie_lane_cap)
5132 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5133
5134 /* covers APUs as well */
5135 if (pci_is_root_bus(adev->pdev->bus)) {
5136 if (adev->pm.pcie_gen_mask == 0)
5137 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5138 if (adev->pm.pcie_mlw_mask == 0)
5139 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5140 return;
5141 }
5142
5143 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5144 return;
5145
5146 pcie_bandwidth_available(adev->pdev, NULL,
5147 &platform_speed_cap, &platform_link_width);
5148
5149 if (adev->pm.pcie_gen_mask == 0) {
5150 /* asic caps */
5151 pdev = adev->pdev;
5152 speed_cap = pcie_get_speed_cap(pdev);
5153 if (speed_cap == PCI_SPEED_UNKNOWN) {
5154 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5155 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5156 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5157 } else {
5158 if (speed_cap == PCIE_SPEED_32_0GT)
5159 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5160 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5161 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5162 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5163 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5164 else if (speed_cap == PCIE_SPEED_16_0GT)
5165 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5166 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5167 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5168 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5169 else if (speed_cap == PCIE_SPEED_8_0GT)
5170 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5171 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5172 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5173 else if (speed_cap == PCIE_SPEED_5_0GT)
5174 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5175 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5176 else
5177 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5178 }
5179 /* platform caps */
5180 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5181 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5182 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5183 } else {
5184 if (platform_speed_cap == PCIE_SPEED_32_0GT)
5185 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5186 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5187 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5188 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5189 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5190 else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5191 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5192 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5193 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5194 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5195 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5196 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5197 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5198 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5199 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5200 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5201 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5202 else
5203 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5204
5205 }
5206 }
5207 if (adev->pm.pcie_mlw_mask == 0) {
5208 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5209 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5210 } else {
5211 switch (platform_link_width) {
5212 case PCIE_LNK_X32:
5213 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5214 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5215 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5216 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5217 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5218 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5219 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5220 break;
5221 case PCIE_LNK_X16:
5222 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5223 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5224 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5225 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5226 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5227 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5228 break;
5229 case PCIE_LNK_X12:
5230 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5231 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5232 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5233 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5234 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5235 break;
5236 case PCIE_LNK_X8:
5237 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5238 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5239 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5240 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5241 break;
5242 case PCIE_LNK_X4:
5243 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5244 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5245 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5246 break;
5247 case PCIE_LNK_X2:
5248 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5249 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5250 break;
5251 case PCIE_LNK_X1:
5252 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5253 break;
5254 default:
5255 break;
5256 }
5257 }
5258 }
5259}
5260
5261int amdgpu_device_baco_enter(struct drm_device *dev)
5262{
5263 struct amdgpu_device *adev = drm_to_adev(dev);
5264 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5265
5266 if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5267 return -ENOTSUPP;
5268
5269 if (ras && adev->ras_enabled &&
5270 adev->nbio.funcs->enable_doorbell_interrupt)
5271 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5272
5273 return amdgpu_dpm_baco_enter(adev);
5274}
5275
5276int amdgpu_device_baco_exit(struct drm_device *dev)
5277{
5278 struct amdgpu_device *adev = drm_to_adev(dev);
5279 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5280 int ret = 0;
5281
5282 if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5283 return -ENOTSUPP;
5284
5285 ret = amdgpu_dpm_baco_exit(adev);
5286 if (ret)
5287 return ret;
5288
5289 if (ras && adev->ras_enabled &&
5290 adev->nbio.funcs->enable_doorbell_interrupt)
5291 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5292
5293 return 0;
5294}
5295
5296static void amdgpu_cancel_all_tdr(struct amdgpu_device *adev)
5297{
5298 int i;
5299
5300 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5301 struct amdgpu_ring *ring = adev->rings[i];
5302
5303 if (!ring || !ring->sched.thread)
5304 continue;
5305
5306 cancel_delayed_work_sync(&ring->sched.work_tdr);
5307 }
5308}
5309
5310/**
5311 * amdgpu_pci_error_detected - Called when a PCI error is detected.
5312 * @pdev: PCI device struct
5313 * @state: PCI channel state
5314 *
5315 * Description: Called when a PCI error is detected.
5316 *
5317 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5318 */
5319pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5320{
5321 struct drm_device *dev = pci_get_drvdata(pdev);
5322 struct amdgpu_device *adev = drm_to_adev(dev);
5323 int i;
5324
5325 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5326
5327 if (adev->gmc.xgmi.num_physical_nodes > 1) {
5328 DRM_WARN("No support for XGMI hive yet...");
5329 return PCI_ERS_RESULT_DISCONNECT;
5330 }
5331
5332 adev->pci_channel_state = state;
5333
5334 switch (state) {
5335 case pci_channel_io_normal:
5336 return PCI_ERS_RESULT_CAN_RECOVER;
5337 /* Fatal error, prepare for slot reset */
5338 case pci_channel_io_frozen:
5339 /*
5340 * Cancel and wait for all TDRs in progress if failing to
5341 * set adev->in_gpu_reset in amdgpu_device_lock_adev
5342 *
5343 * Locking adev->reset_sem will prevent any external access
5344 * to GPU during PCI error recovery
5345 */
5346 while (!amdgpu_device_lock_adev(adev, NULL))
5347 amdgpu_cancel_all_tdr(adev);
5348
5349 /*
5350 * Block any work scheduling as we do for regular GPU reset
5351 * for the duration of the recovery
5352 */
5353 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5354 struct amdgpu_ring *ring = adev->rings[i];
5355
5356 if (!ring || !ring->sched.thread)
5357 continue;
5358
5359 drm_sched_stop(&ring->sched, NULL);
5360 }
5361 atomic_inc(&adev->gpu_reset_counter);
5362 return PCI_ERS_RESULT_NEED_RESET;
5363 case pci_channel_io_perm_failure:
5364 /* Permanent error, prepare for device removal */
5365 return PCI_ERS_RESULT_DISCONNECT;
5366 }
5367
5368 return PCI_ERS_RESULT_NEED_RESET;
5369}
5370
5371/**
5372 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5373 * @pdev: pointer to PCI device
5374 */
5375pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5376{
5377
5378 DRM_INFO("PCI error: mmio enabled callback!!\n");
5379
5380 /* TODO - dump whatever for debugging purposes */
5381
5382 /* This called only if amdgpu_pci_error_detected returns
5383 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5384 * works, no need to reset slot.
5385 */
5386
5387 return PCI_ERS_RESULT_RECOVERED;
5388}
5389
5390/**
5391 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5392 * @pdev: PCI device struct
5393 *
5394 * Description: This routine is called by the pci error recovery
5395 * code after the PCI slot has been reset, just before we
5396 * should resume normal operations.
5397 */
5398pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5399{
5400 struct drm_device *dev = pci_get_drvdata(pdev);
5401 struct amdgpu_device *adev = drm_to_adev(dev);
5402 int r, i;
5403 struct amdgpu_reset_context reset_context;
5404 u32 memsize;
5405 struct list_head device_list;
5406
5407 DRM_INFO("PCI error: slot reset callback!!\n");
5408
5409 memset(&reset_context, 0, sizeof(reset_context));
5410
5411 INIT_LIST_HEAD(&device_list);
5412 list_add_tail(&adev->reset_list, &device_list);
5413
5414 /* wait for asic to come out of reset */
5415 msleep(500);
5416
5417 /* Restore PCI confspace */
5418 amdgpu_device_load_pci_state(pdev);
5419
5420 /* confirm ASIC came out of reset */
5421 for (i = 0; i < adev->usec_timeout; i++) {
5422 memsize = amdgpu_asic_get_config_memsize(adev);
5423
5424 if (memsize != 0xffffffff)
5425 break;
5426 udelay(1);
5427 }
5428 if (memsize == 0xffffffff) {
5429 r = -ETIME;
5430 goto out;
5431 }
5432
5433 reset_context.method = AMD_RESET_METHOD_NONE;
5434 reset_context.reset_req_dev = adev;
5435 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5436 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5437
5438 adev->no_hw_access = true;
5439 r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5440 adev->no_hw_access = false;
5441 if (r)
5442 goto out;
5443
5444 r = amdgpu_do_asic_reset(&device_list, &reset_context);
5445
5446out:
5447 if (!r) {
5448 if (amdgpu_device_cache_pci_state(adev->pdev))
5449 pci_restore_state(adev->pdev);
5450
5451 DRM_INFO("PCIe error recovery succeeded\n");
5452 } else {
5453 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5454 amdgpu_device_unlock_adev(adev);
5455 }
5456
5457 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5458}
5459
5460/**
5461 * amdgpu_pci_resume() - resume normal ops after PCI reset
5462 * @pdev: pointer to PCI device
5463 *
5464 * Called when the error recovery driver tells us that its
5465 * OK to resume normal operation.
5466 */
5467void amdgpu_pci_resume(struct pci_dev *pdev)
5468{
5469 struct drm_device *dev = pci_get_drvdata(pdev);
5470 struct amdgpu_device *adev = drm_to_adev(dev);
5471 int i;
5472
5473
5474 DRM_INFO("PCI error: resume callback!!\n");
5475
5476 /* Only continue execution for the case of pci_channel_io_frozen */
5477 if (adev->pci_channel_state != pci_channel_io_frozen)
5478 return;
5479
5480 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5481 struct amdgpu_ring *ring = adev->rings[i];
5482
5483 if (!ring || !ring->sched.thread)
5484 continue;
5485
5486
5487 drm_sched_resubmit_jobs(&ring->sched);
5488 drm_sched_start(&ring->sched, true);
5489 }
5490
5491 amdgpu_device_unlock_adev(adev);
5492}
5493
5494bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5495{
5496 struct drm_device *dev = pci_get_drvdata(pdev);
5497 struct amdgpu_device *adev = drm_to_adev(dev);
5498 int r;
5499
5500 r = pci_save_state(pdev);
5501 if (!r) {
5502 kfree(adev->pci_state);
5503
5504 adev->pci_state = pci_store_saved_state(pdev);
5505
5506 if (!adev->pci_state) {
5507 DRM_ERROR("Failed to store PCI saved state");
5508 return false;
5509 }
5510 } else {
5511 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5512 return false;
5513 }
5514
5515 return true;
5516}
5517
5518bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5519{
5520 struct drm_device *dev = pci_get_drvdata(pdev);
5521 struct amdgpu_device *adev = drm_to_adev(dev);
5522 int r;
5523
5524 if (!adev->pci_state)
5525 return false;
5526
5527 r = pci_load_saved_state(pdev, adev->pci_state);
5528
5529 if (!r) {
5530 pci_restore_state(pdev);
5531 } else {
5532 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5533 return false;
5534 }
5535
5536 return true;
5537}
5538
5539void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5540 struct amdgpu_ring *ring)
5541{
5542#ifdef CONFIG_X86_64
5543 if (adev->flags & AMD_IS_APU)
5544 return;
5545#endif
5546 if (adev->gmc.xgmi.connected_to_cpu)
5547 return;
5548
5549 if (ring && ring->funcs->emit_hdp_flush)
5550 amdgpu_ring_emit_hdp_flush(ring);
5551 else
5552 amdgpu_asic_flush_hdp(adev, ring);
5553}
5554
5555void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5556 struct amdgpu_ring *ring)
5557{
5558#ifdef CONFIG_X86_64
5559 if (adev->flags & AMD_IS_APU)
5560 return;
5561#endif
5562 if (adev->gmc.xgmi.connected_to_cpu)
5563 return;
5564
5565 amdgpu_asic_invalidate_hdp(adev, ring);
5566}
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/power_supply.h>
29#include <linux/kthread.h>
30#include <linux/module.h>
31#include <linux/console.h>
32#include <linux/slab.h>
33
34#include <drm/drm_atomic_helper.h>
35#include <drm/drm_probe_helper.h>
36#include <drm/amdgpu_drm.h>
37#include <linux/vgaarb.h>
38#include <linux/vga_switcheroo.h>
39#include <linux/efi.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_i2c.h"
43#include "atom.h"
44#include "amdgpu_atombios.h"
45#include "amdgpu_atomfirmware.h"
46#include "amd_pcie.h"
47#ifdef CONFIG_DRM_AMDGPU_SI
48#include "si.h"
49#endif
50#ifdef CONFIG_DRM_AMDGPU_CIK
51#include "cik.h"
52#endif
53#include "vi.h"
54#include "soc15.h"
55#include "nv.h"
56#include "bif/bif_4_1_d.h"
57#include <linux/pci.h>
58#include <linux/firmware.h>
59#include "amdgpu_vf_error.h"
60
61#include "amdgpu_amdkfd.h"
62#include "amdgpu_pm.h"
63
64#include "amdgpu_xgmi.h"
65#include "amdgpu_ras.h"
66#include "amdgpu_pmu.h"
67
68MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
69MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
70MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
71MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
72MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
73MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
74MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
75MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
76MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
77MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
78
79#define AMDGPU_RESUME_MS 2000
80
81static const char *amdgpu_asic_name[] = {
82 "TAHITI",
83 "PITCAIRN",
84 "VERDE",
85 "OLAND",
86 "HAINAN",
87 "BONAIRE",
88 "KAVERI",
89 "KABINI",
90 "HAWAII",
91 "MULLINS",
92 "TOPAZ",
93 "TONGA",
94 "FIJI",
95 "CARRIZO",
96 "STONEY",
97 "POLARIS10",
98 "POLARIS11",
99 "POLARIS12",
100 "VEGAM",
101 "VEGA10",
102 "VEGA12",
103 "VEGA20",
104 "RAVEN",
105 "ARCTURUS",
106 "RENOIR",
107 "NAVI10",
108 "NAVI14",
109 "NAVI12",
110 "LAST",
111};
112
113/**
114 * DOC: pcie_replay_count
115 *
116 * The amdgpu driver provides a sysfs API for reporting the total number
117 * of PCIe replays (NAKs)
118 * The file pcie_replay_count is used for this and returns the total
119 * number of replays as a sum of the NAKs generated and NAKs received
120 */
121
122static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
123 struct device_attribute *attr, char *buf)
124{
125 struct drm_device *ddev = dev_get_drvdata(dev);
126 struct amdgpu_device *adev = ddev->dev_private;
127 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
128
129 return snprintf(buf, PAGE_SIZE, "%llu\n", cnt);
130}
131
132static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
133 amdgpu_device_get_pcie_replay_count, NULL);
134
135static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
136
137/**
138 * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
139 *
140 * @dev: drm_device pointer
141 *
142 * Returns true if the device is a dGPU with HG/PX power control,
143 * otherwise return false.
144 */
145bool amdgpu_device_is_px(struct drm_device *dev)
146{
147 struct amdgpu_device *adev = dev->dev_private;
148
149 if (adev->flags & AMD_IS_PX)
150 return true;
151 return false;
152}
153
154/*
155 * MMIO register access helper functions.
156 */
157/**
158 * amdgpu_mm_rreg - read a memory mapped IO register
159 *
160 * @adev: amdgpu_device pointer
161 * @reg: dword aligned register offset
162 * @acc_flags: access flags which require special behavior
163 *
164 * Returns the 32 bit value from the offset specified.
165 */
166uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
167 uint32_t acc_flags)
168{
169 uint32_t ret;
170
171 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
172 return amdgpu_virt_kiq_rreg(adev, reg);
173
174 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
175 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
176 else {
177 unsigned long flags;
178
179 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
180 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
181 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
182 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
183 }
184 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
185 return ret;
186}
187
188/*
189 * MMIO register read with bytes helper functions
190 * @offset:bytes offset from MMIO start
191 *
192*/
193
194/**
195 * amdgpu_mm_rreg8 - read a memory mapped IO register
196 *
197 * @adev: amdgpu_device pointer
198 * @offset: byte aligned register offset
199 *
200 * Returns the 8 bit value from the offset specified.
201 */
202uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
203 if (offset < adev->rmmio_size)
204 return (readb(adev->rmmio + offset));
205 BUG();
206}
207
208/*
209 * MMIO register write with bytes helper functions
210 * @offset:bytes offset from MMIO start
211 * @value: the value want to be written to the register
212 *
213*/
214/**
215 * amdgpu_mm_wreg8 - read a memory mapped IO register
216 *
217 * @adev: amdgpu_device pointer
218 * @offset: byte aligned register offset
219 * @value: 8 bit value to write
220 *
221 * Writes the value specified to the offset specified.
222 */
223void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
224 if (offset < adev->rmmio_size)
225 writeb(value, adev->rmmio + offset);
226 else
227 BUG();
228}
229
230/**
231 * amdgpu_mm_wreg - write to a memory mapped IO register
232 *
233 * @adev: amdgpu_device pointer
234 * @reg: dword aligned register offset
235 * @v: 32 bit value to write to the register
236 * @acc_flags: access flags which require special behavior
237 *
238 * Writes the value specified to the offset specified.
239 */
240void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
241 uint32_t acc_flags)
242{
243 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
244
245 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
246 adev->last_mm_index = v;
247 }
248
249 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
250 return amdgpu_virt_kiq_wreg(adev, reg, v);
251
252 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
253 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
254 else {
255 unsigned long flags;
256
257 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
258 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
259 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
260 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
261 }
262
263 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
264 udelay(500);
265 }
266}
267
268/**
269 * amdgpu_io_rreg - read an IO register
270 *
271 * @adev: amdgpu_device pointer
272 * @reg: dword aligned register offset
273 *
274 * Returns the 32 bit value from the offset specified.
275 */
276u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
277{
278 if ((reg * 4) < adev->rio_mem_size)
279 return ioread32(adev->rio_mem + (reg * 4));
280 else {
281 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
282 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
283 }
284}
285
286/**
287 * amdgpu_io_wreg - write to an IO register
288 *
289 * @adev: amdgpu_device pointer
290 * @reg: dword aligned register offset
291 * @v: 32 bit value to write to the register
292 *
293 * Writes the value specified to the offset specified.
294 */
295void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
296{
297 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
298 adev->last_mm_index = v;
299 }
300
301 if ((reg * 4) < adev->rio_mem_size)
302 iowrite32(v, adev->rio_mem + (reg * 4));
303 else {
304 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
305 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
306 }
307
308 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
309 udelay(500);
310 }
311}
312
313/**
314 * amdgpu_mm_rdoorbell - read a doorbell dword
315 *
316 * @adev: amdgpu_device pointer
317 * @index: doorbell index
318 *
319 * Returns the value in the doorbell aperture at the
320 * requested doorbell index (CIK).
321 */
322u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
323{
324 if (index < adev->doorbell.num_doorbells) {
325 return readl(adev->doorbell.ptr + index);
326 } else {
327 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
328 return 0;
329 }
330}
331
332/**
333 * amdgpu_mm_wdoorbell - write a doorbell dword
334 *
335 * @adev: amdgpu_device pointer
336 * @index: doorbell index
337 * @v: value to write
338 *
339 * Writes @v to the doorbell aperture at the
340 * requested doorbell index (CIK).
341 */
342void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
343{
344 if (index < adev->doorbell.num_doorbells) {
345 writel(v, adev->doorbell.ptr + index);
346 } else {
347 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
348 }
349}
350
351/**
352 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
353 *
354 * @adev: amdgpu_device pointer
355 * @index: doorbell index
356 *
357 * Returns the value in the doorbell aperture at the
358 * requested doorbell index (VEGA10+).
359 */
360u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
361{
362 if (index < adev->doorbell.num_doorbells) {
363 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
364 } else {
365 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
366 return 0;
367 }
368}
369
370/**
371 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
372 *
373 * @adev: amdgpu_device pointer
374 * @index: doorbell index
375 * @v: value to write
376 *
377 * Writes @v to the doorbell aperture at the
378 * requested doorbell index (VEGA10+).
379 */
380void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
381{
382 if (index < adev->doorbell.num_doorbells) {
383 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
384 } else {
385 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
386 }
387}
388
389/**
390 * amdgpu_invalid_rreg - dummy reg read function
391 *
392 * @adev: amdgpu device pointer
393 * @reg: offset of register
394 *
395 * Dummy register read function. Used for register blocks
396 * that certain asics don't have (all asics).
397 * Returns the value in the register.
398 */
399static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
400{
401 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
402 BUG();
403 return 0;
404}
405
406/**
407 * amdgpu_invalid_wreg - dummy reg write function
408 *
409 * @adev: amdgpu device pointer
410 * @reg: offset of register
411 * @v: value to write to the register
412 *
413 * Dummy register read function. Used for register blocks
414 * that certain asics don't have (all asics).
415 */
416static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
417{
418 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
419 reg, v);
420 BUG();
421}
422
423/**
424 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
425 *
426 * @adev: amdgpu device pointer
427 * @reg: offset of register
428 *
429 * Dummy register read function. Used for register blocks
430 * that certain asics don't have (all asics).
431 * Returns the value in the register.
432 */
433static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
434{
435 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
436 BUG();
437 return 0;
438}
439
440/**
441 * amdgpu_invalid_wreg64 - dummy reg write function
442 *
443 * @adev: amdgpu device pointer
444 * @reg: offset of register
445 * @v: value to write to the register
446 *
447 * Dummy register read function. Used for register blocks
448 * that certain asics don't have (all asics).
449 */
450static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
451{
452 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
453 reg, v);
454 BUG();
455}
456
457/**
458 * amdgpu_block_invalid_rreg - dummy reg read function
459 *
460 * @adev: amdgpu device pointer
461 * @block: offset of instance
462 * @reg: offset of register
463 *
464 * Dummy register read function. Used for register blocks
465 * that certain asics don't have (all asics).
466 * Returns the value in the register.
467 */
468static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
469 uint32_t block, uint32_t reg)
470{
471 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
472 reg, block);
473 BUG();
474 return 0;
475}
476
477/**
478 * amdgpu_block_invalid_wreg - dummy reg write function
479 *
480 * @adev: amdgpu device pointer
481 * @block: offset of instance
482 * @reg: offset of register
483 * @v: value to write to the register
484 *
485 * Dummy register read function. Used for register blocks
486 * that certain asics don't have (all asics).
487 */
488static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
489 uint32_t block,
490 uint32_t reg, uint32_t v)
491{
492 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
493 reg, block, v);
494 BUG();
495}
496
497/**
498 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
499 *
500 * @adev: amdgpu device pointer
501 *
502 * Allocates a scratch page of VRAM for use by various things in the
503 * driver.
504 */
505static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
506{
507 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
508 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
509 &adev->vram_scratch.robj,
510 &adev->vram_scratch.gpu_addr,
511 (void **)&adev->vram_scratch.ptr);
512}
513
514/**
515 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
516 *
517 * @adev: amdgpu device pointer
518 *
519 * Frees the VRAM scratch page.
520 */
521static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
522{
523 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
524}
525
526/**
527 * amdgpu_device_program_register_sequence - program an array of registers.
528 *
529 * @adev: amdgpu_device pointer
530 * @registers: pointer to the register array
531 * @array_size: size of the register array
532 *
533 * Programs an array or registers with and and or masks.
534 * This is a helper for setting golden registers.
535 */
536void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
537 const u32 *registers,
538 const u32 array_size)
539{
540 u32 tmp, reg, and_mask, or_mask;
541 int i;
542
543 if (array_size % 3)
544 return;
545
546 for (i = 0; i < array_size; i +=3) {
547 reg = registers[i + 0];
548 and_mask = registers[i + 1];
549 or_mask = registers[i + 2];
550
551 if (and_mask == 0xffffffff) {
552 tmp = or_mask;
553 } else {
554 tmp = RREG32(reg);
555 tmp &= ~and_mask;
556 if (adev->family >= AMDGPU_FAMILY_AI)
557 tmp |= (or_mask & and_mask);
558 else
559 tmp |= or_mask;
560 }
561 WREG32(reg, tmp);
562 }
563}
564
565/**
566 * amdgpu_device_pci_config_reset - reset the GPU
567 *
568 * @adev: amdgpu_device pointer
569 *
570 * Resets the GPU using the pci config reset sequence.
571 * Only applicable to asics prior to vega10.
572 */
573void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
574{
575 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
576}
577
578/*
579 * GPU doorbell aperture helpers function.
580 */
581/**
582 * amdgpu_device_doorbell_init - Init doorbell driver information.
583 *
584 * @adev: amdgpu_device pointer
585 *
586 * Init doorbell driver information (CIK)
587 * Returns 0 on success, error on failure.
588 */
589static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
590{
591
592 /* No doorbell on SI hardware generation */
593 if (adev->asic_type < CHIP_BONAIRE) {
594 adev->doorbell.base = 0;
595 adev->doorbell.size = 0;
596 adev->doorbell.num_doorbells = 0;
597 adev->doorbell.ptr = NULL;
598 return 0;
599 }
600
601 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
602 return -EINVAL;
603
604 amdgpu_asic_init_doorbell_index(adev);
605
606 /* doorbell bar mapping */
607 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
608 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
609
610 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
611 adev->doorbell_index.max_assignment+1);
612 if (adev->doorbell.num_doorbells == 0)
613 return -EINVAL;
614
615 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
616 * paging queue doorbell use the second page. The
617 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
618 * doorbells are in the first page. So with paging queue enabled,
619 * the max num_doorbells should + 1 page (0x400 in dword)
620 */
621 if (adev->asic_type >= CHIP_VEGA10)
622 adev->doorbell.num_doorbells += 0x400;
623
624 adev->doorbell.ptr = ioremap(adev->doorbell.base,
625 adev->doorbell.num_doorbells *
626 sizeof(u32));
627 if (adev->doorbell.ptr == NULL)
628 return -ENOMEM;
629
630 return 0;
631}
632
633/**
634 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
635 *
636 * @adev: amdgpu_device pointer
637 *
638 * Tear down doorbell driver information (CIK)
639 */
640static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
641{
642 iounmap(adev->doorbell.ptr);
643 adev->doorbell.ptr = NULL;
644}
645
646
647
648/*
649 * amdgpu_device_wb_*()
650 * Writeback is the method by which the GPU updates special pages in memory
651 * with the status of certain GPU events (fences, ring pointers,etc.).
652 */
653
654/**
655 * amdgpu_device_wb_fini - Disable Writeback and free memory
656 *
657 * @adev: amdgpu_device pointer
658 *
659 * Disables Writeback and frees the Writeback memory (all asics).
660 * Used at driver shutdown.
661 */
662static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
663{
664 if (adev->wb.wb_obj) {
665 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
666 &adev->wb.gpu_addr,
667 (void **)&adev->wb.wb);
668 adev->wb.wb_obj = NULL;
669 }
670}
671
672/**
673 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
674 *
675 * @adev: amdgpu_device pointer
676 *
677 * Initializes writeback and allocates writeback memory (all asics).
678 * Used at driver startup.
679 * Returns 0 on success or an -error on failure.
680 */
681static int amdgpu_device_wb_init(struct amdgpu_device *adev)
682{
683 int r;
684
685 if (adev->wb.wb_obj == NULL) {
686 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
687 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
688 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
689 &adev->wb.wb_obj, &adev->wb.gpu_addr,
690 (void **)&adev->wb.wb);
691 if (r) {
692 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
693 return r;
694 }
695
696 adev->wb.num_wb = AMDGPU_MAX_WB;
697 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
698
699 /* clear wb memory */
700 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
701 }
702
703 return 0;
704}
705
706/**
707 * amdgpu_device_wb_get - Allocate a wb entry
708 *
709 * @adev: amdgpu_device pointer
710 * @wb: wb index
711 *
712 * Allocate a wb slot for use by the driver (all asics).
713 * Returns 0 on success or -EINVAL on failure.
714 */
715int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
716{
717 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
718
719 if (offset < adev->wb.num_wb) {
720 __set_bit(offset, adev->wb.used);
721 *wb = offset << 3; /* convert to dw offset */
722 return 0;
723 } else {
724 return -EINVAL;
725 }
726}
727
728/**
729 * amdgpu_device_wb_free - Free a wb entry
730 *
731 * @adev: amdgpu_device pointer
732 * @wb: wb index
733 *
734 * Free a wb slot allocated for use by the driver (all asics)
735 */
736void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
737{
738 wb >>= 3;
739 if (wb < adev->wb.num_wb)
740 __clear_bit(wb, adev->wb.used);
741}
742
743/**
744 * amdgpu_device_resize_fb_bar - try to resize FB BAR
745 *
746 * @adev: amdgpu_device pointer
747 *
748 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
749 * to fail, but if any of the BARs is not accessible after the size we abort
750 * driver loading by returning -ENODEV.
751 */
752int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
753{
754 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
755 u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
756 struct pci_bus *root;
757 struct resource *res;
758 unsigned i;
759 u16 cmd;
760 int r;
761
762 /* Bypass for VF */
763 if (amdgpu_sriov_vf(adev))
764 return 0;
765
766 /* Check if the root BUS has 64bit memory resources */
767 root = adev->pdev->bus;
768 while (root->parent)
769 root = root->parent;
770
771 pci_bus_for_each_resource(root, res, i) {
772 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
773 res->start > 0x100000000ull)
774 break;
775 }
776
777 /* Trying to resize is pointless without a root hub window above 4GB */
778 if (!res)
779 return 0;
780
781 /* Disable memory decoding while we change the BAR addresses and size */
782 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
783 pci_write_config_word(adev->pdev, PCI_COMMAND,
784 cmd & ~PCI_COMMAND_MEMORY);
785
786 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
787 amdgpu_device_doorbell_fini(adev);
788 if (adev->asic_type >= CHIP_BONAIRE)
789 pci_release_resource(adev->pdev, 2);
790
791 pci_release_resource(adev->pdev, 0);
792
793 r = pci_resize_resource(adev->pdev, 0, rbar_size);
794 if (r == -ENOSPC)
795 DRM_INFO("Not enough PCI address space for a large BAR.");
796 else if (r && r != -ENOTSUPP)
797 DRM_ERROR("Problem resizing BAR0 (%d).", r);
798
799 pci_assign_unassigned_bus_resources(adev->pdev->bus);
800
801 /* When the doorbell or fb BAR isn't available we have no chance of
802 * using the device.
803 */
804 r = amdgpu_device_doorbell_init(adev);
805 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
806 return -ENODEV;
807
808 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
809
810 return 0;
811}
812
813/*
814 * GPU helpers function.
815 */
816/**
817 * amdgpu_device_need_post - check if the hw need post or not
818 *
819 * @adev: amdgpu_device pointer
820 *
821 * Check if the asic has been initialized (all asics) at driver startup
822 * or post is needed if hw reset is performed.
823 * Returns true if need or false if not.
824 */
825bool amdgpu_device_need_post(struct amdgpu_device *adev)
826{
827 uint32_t reg;
828
829 if (amdgpu_sriov_vf(adev))
830 return false;
831
832 if (amdgpu_passthrough(adev)) {
833 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
834 * some old smc fw still need driver do vPost otherwise gpu hang, while
835 * those smc fw version above 22.15 doesn't have this flaw, so we force
836 * vpost executed for smc version below 22.15
837 */
838 if (adev->asic_type == CHIP_FIJI) {
839 int err;
840 uint32_t fw_ver;
841 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
842 /* force vPost if error occured */
843 if (err)
844 return true;
845
846 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
847 if (fw_ver < 0x00160e00)
848 return true;
849 }
850 }
851
852 if (adev->has_hw_reset) {
853 adev->has_hw_reset = false;
854 return true;
855 }
856
857 /* bios scratch used on CIK+ */
858 if (adev->asic_type >= CHIP_BONAIRE)
859 return amdgpu_atombios_scratch_need_asic_init(adev);
860
861 /* check MEM_SIZE for older asics */
862 reg = amdgpu_asic_get_config_memsize(adev);
863
864 if ((reg != 0) && (reg != 0xffffffff))
865 return false;
866
867 return true;
868}
869
870/* if we get transitioned to only one device, take VGA back */
871/**
872 * amdgpu_device_vga_set_decode - enable/disable vga decode
873 *
874 * @cookie: amdgpu_device pointer
875 * @state: enable/disable vga decode
876 *
877 * Enable/disable vga decode (all asics).
878 * Returns VGA resource flags.
879 */
880static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
881{
882 struct amdgpu_device *adev = cookie;
883 amdgpu_asic_set_vga_state(adev, state);
884 if (state)
885 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
886 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
887 else
888 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
889}
890
891/**
892 * amdgpu_device_check_block_size - validate the vm block size
893 *
894 * @adev: amdgpu_device pointer
895 *
896 * Validates the vm block size specified via module parameter.
897 * The vm block size defines number of bits in page table versus page directory,
898 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
899 * page table and the remaining bits are in the page directory.
900 */
901static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
902{
903 /* defines number of bits in page table versus page directory,
904 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
905 * page table and the remaining bits are in the page directory */
906 if (amdgpu_vm_block_size == -1)
907 return;
908
909 if (amdgpu_vm_block_size < 9) {
910 dev_warn(adev->dev, "VM page table size (%d) too small\n",
911 amdgpu_vm_block_size);
912 amdgpu_vm_block_size = -1;
913 }
914}
915
916/**
917 * amdgpu_device_check_vm_size - validate the vm size
918 *
919 * @adev: amdgpu_device pointer
920 *
921 * Validates the vm size in GB specified via module parameter.
922 * The VM size is the size of the GPU virtual memory space in GB.
923 */
924static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
925{
926 /* no need to check the default value */
927 if (amdgpu_vm_size == -1)
928 return;
929
930 if (amdgpu_vm_size < 1) {
931 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
932 amdgpu_vm_size);
933 amdgpu_vm_size = -1;
934 }
935}
936
937static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
938{
939 struct sysinfo si;
940 bool is_os_64 = (sizeof(void *) == 8) ? true : false;
941 uint64_t total_memory;
942 uint64_t dram_size_seven_GB = 0x1B8000000;
943 uint64_t dram_size_three_GB = 0xB8000000;
944
945 if (amdgpu_smu_memory_pool_size == 0)
946 return;
947
948 if (!is_os_64) {
949 DRM_WARN("Not 64-bit OS, feature not supported\n");
950 goto def_value;
951 }
952 si_meminfo(&si);
953 total_memory = (uint64_t)si.totalram * si.mem_unit;
954
955 if ((amdgpu_smu_memory_pool_size == 1) ||
956 (amdgpu_smu_memory_pool_size == 2)) {
957 if (total_memory < dram_size_three_GB)
958 goto def_value1;
959 } else if ((amdgpu_smu_memory_pool_size == 4) ||
960 (amdgpu_smu_memory_pool_size == 8)) {
961 if (total_memory < dram_size_seven_GB)
962 goto def_value1;
963 } else {
964 DRM_WARN("Smu memory pool size not supported\n");
965 goto def_value;
966 }
967 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
968
969 return;
970
971def_value1:
972 DRM_WARN("No enough system memory\n");
973def_value:
974 adev->pm.smu_prv_buffer_size = 0;
975}
976
977/**
978 * amdgpu_device_check_arguments - validate module params
979 *
980 * @adev: amdgpu_device pointer
981 *
982 * Validates certain module parameters and updates
983 * the associated values used by the driver (all asics).
984 */
985static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
986{
987 int ret = 0;
988
989 if (amdgpu_sched_jobs < 4) {
990 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
991 amdgpu_sched_jobs);
992 amdgpu_sched_jobs = 4;
993 } else if (!is_power_of_2(amdgpu_sched_jobs)){
994 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
995 amdgpu_sched_jobs);
996 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
997 }
998
999 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1000 /* gart size must be greater or equal to 32M */
1001 dev_warn(adev->dev, "gart size (%d) too small\n",
1002 amdgpu_gart_size);
1003 amdgpu_gart_size = -1;
1004 }
1005
1006 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1007 /* gtt size must be greater or equal to 32M */
1008 dev_warn(adev->dev, "gtt size (%d) too small\n",
1009 amdgpu_gtt_size);
1010 amdgpu_gtt_size = -1;
1011 }
1012
1013 /* valid range is between 4 and 9 inclusive */
1014 if (amdgpu_vm_fragment_size != -1 &&
1015 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1016 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1017 amdgpu_vm_fragment_size = -1;
1018 }
1019
1020 amdgpu_device_check_smu_prv_buffer_size(adev);
1021
1022 amdgpu_device_check_vm_size(adev);
1023
1024 amdgpu_device_check_block_size(adev);
1025
1026 ret = amdgpu_device_get_job_timeout_settings(adev);
1027 if (ret) {
1028 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
1029 return ret;
1030 }
1031
1032 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1033
1034 return ret;
1035}
1036
1037/**
1038 * amdgpu_switcheroo_set_state - set switcheroo state
1039 *
1040 * @pdev: pci dev pointer
1041 * @state: vga_switcheroo state
1042 *
1043 * Callback for the switcheroo driver. Suspends or resumes the
1044 * the asics before or after it is powered up using ACPI methods.
1045 */
1046static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1047{
1048 struct drm_device *dev = pci_get_drvdata(pdev);
1049
1050 if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1051 return;
1052
1053 if (state == VGA_SWITCHEROO_ON) {
1054 pr_info("amdgpu: switched on\n");
1055 /* don't suspend or resume card normally */
1056 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1057
1058 amdgpu_device_resume(dev, true, true);
1059
1060 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1061 drm_kms_helper_poll_enable(dev);
1062 } else {
1063 pr_info("amdgpu: switched off\n");
1064 drm_kms_helper_poll_disable(dev);
1065 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1066 amdgpu_device_suspend(dev, true, true);
1067 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1068 }
1069}
1070
1071/**
1072 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1073 *
1074 * @pdev: pci dev pointer
1075 *
1076 * Callback for the switcheroo driver. Check of the switcheroo
1077 * state can be changed.
1078 * Returns true if the state can be changed, false if not.
1079 */
1080static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1081{
1082 struct drm_device *dev = pci_get_drvdata(pdev);
1083
1084 /*
1085 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1086 * locking inversion with the driver load path. And the access here is
1087 * completely racy anyway. So don't bother with locking for now.
1088 */
1089 return dev->open_count == 0;
1090}
1091
1092static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1093 .set_gpu_state = amdgpu_switcheroo_set_state,
1094 .reprobe = NULL,
1095 .can_switch = amdgpu_switcheroo_can_switch,
1096};
1097
1098/**
1099 * amdgpu_device_ip_set_clockgating_state - set the CG state
1100 *
1101 * @dev: amdgpu_device pointer
1102 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1103 * @state: clockgating state (gate or ungate)
1104 *
1105 * Sets the requested clockgating state for all instances of
1106 * the hardware IP specified.
1107 * Returns the error code from the last instance.
1108 */
1109int amdgpu_device_ip_set_clockgating_state(void *dev,
1110 enum amd_ip_block_type block_type,
1111 enum amd_clockgating_state state)
1112{
1113 struct amdgpu_device *adev = dev;
1114 int i, r = 0;
1115
1116 for (i = 0; i < adev->num_ip_blocks; i++) {
1117 if (!adev->ip_blocks[i].status.valid)
1118 continue;
1119 if (adev->ip_blocks[i].version->type != block_type)
1120 continue;
1121 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1122 continue;
1123 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1124 (void *)adev, state);
1125 if (r)
1126 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1127 adev->ip_blocks[i].version->funcs->name, r);
1128 }
1129 return r;
1130}
1131
1132/**
1133 * amdgpu_device_ip_set_powergating_state - set the PG state
1134 *
1135 * @dev: amdgpu_device pointer
1136 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1137 * @state: powergating state (gate or ungate)
1138 *
1139 * Sets the requested powergating state for all instances of
1140 * the hardware IP specified.
1141 * Returns the error code from the last instance.
1142 */
1143int amdgpu_device_ip_set_powergating_state(void *dev,
1144 enum amd_ip_block_type block_type,
1145 enum amd_powergating_state state)
1146{
1147 struct amdgpu_device *adev = dev;
1148 int i, r = 0;
1149
1150 for (i = 0; i < adev->num_ip_blocks; i++) {
1151 if (!adev->ip_blocks[i].status.valid)
1152 continue;
1153 if (adev->ip_blocks[i].version->type != block_type)
1154 continue;
1155 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1156 continue;
1157 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1158 (void *)adev, state);
1159 if (r)
1160 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1161 adev->ip_blocks[i].version->funcs->name, r);
1162 }
1163 return r;
1164}
1165
1166/**
1167 * amdgpu_device_ip_get_clockgating_state - get the CG state
1168 *
1169 * @adev: amdgpu_device pointer
1170 * @flags: clockgating feature flags
1171 *
1172 * Walks the list of IPs on the device and updates the clockgating
1173 * flags for each IP.
1174 * Updates @flags with the feature flags for each hardware IP where
1175 * clockgating is enabled.
1176 */
1177void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1178 u32 *flags)
1179{
1180 int i;
1181
1182 for (i = 0; i < adev->num_ip_blocks; i++) {
1183 if (!adev->ip_blocks[i].status.valid)
1184 continue;
1185 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1186 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1187 }
1188}
1189
1190/**
1191 * amdgpu_device_ip_wait_for_idle - wait for idle
1192 *
1193 * @adev: amdgpu_device pointer
1194 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1195 *
1196 * Waits for the request hardware IP to be idle.
1197 * Returns 0 for success or a negative error code on failure.
1198 */
1199int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1200 enum amd_ip_block_type block_type)
1201{
1202 int i, r;
1203
1204 for (i = 0; i < adev->num_ip_blocks; i++) {
1205 if (!adev->ip_blocks[i].status.valid)
1206 continue;
1207 if (adev->ip_blocks[i].version->type == block_type) {
1208 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1209 if (r)
1210 return r;
1211 break;
1212 }
1213 }
1214 return 0;
1215
1216}
1217
1218/**
1219 * amdgpu_device_ip_is_idle - is the hardware IP idle
1220 *
1221 * @adev: amdgpu_device pointer
1222 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1223 *
1224 * Check if the hardware IP is idle or not.
1225 * Returns true if it the IP is idle, false if not.
1226 */
1227bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1228 enum amd_ip_block_type block_type)
1229{
1230 int i;
1231
1232 for (i = 0; i < adev->num_ip_blocks; i++) {
1233 if (!adev->ip_blocks[i].status.valid)
1234 continue;
1235 if (adev->ip_blocks[i].version->type == block_type)
1236 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1237 }
1238 return true;
1239
1240}
1241
1242/**
1243 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1244 *
1245 * @adev: amdgpu_device pointer
1246 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1247 *
1248 * Returns a pointer to the hardware IP block structure
1249 * if it exists for the asic, otherwise NULL.
1250 */
1251struct amdgpu_ip_block *
1252amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1253 enum amd_ip_block_type type)
1254{
1255 int i;
1256
1257 for (i = 0; i < adev->num_ip_blocks; i++)
1258 if (adev->ip_blocks[i].version->type == type)
1259 return &adev->ip_blocks[i];
1260
1261 return NULL;
1262}
1263
1264/**
1265 * amdgpu_device_ip_block_version_cmp
1266 *
1267 * @adev: amdgpu_device pointer
1268 * @type: enum amd_ip_block_type
1269 * @major: major version
1270 * @minor: minor version
1271 *
1272 * return 0 if equal or greater
1273 * return 1 if smaller or the ip_block doesn't exist
1274 */
1275int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1276 enum amd_ip_block_type type,
1277 u32 major, u32 minor)
1278{
1279 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1280
1281 if (ip_block && ((ip_block->version->major > major) ||
1282 ((ip_block->version->major == major) &&
1283 (ip_block->version->minor >= minor))))
1284 return 0;
1285
1286 return 1;
1287}
1288
1289/**
1290 * amdgpu_device_ip_block_add
1291 *
1292 * @adev: amdgpu_device pointer
1293 * @ip_block_version: pointer to the IP to add
1294 *
1295 * Adds the IP block driver information to the collection of IPs
1296 * on the asic.
1297 */
1298int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1299 const struct amdgpu_ip_block_version *ip_block_version)
1300{
1301 if (!ip_block_version)
1302 return -EINVAL;
1303
1304 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1305 ip_block_version->funcs->name);
1306
1307 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1308
1309 return 0;
1310}
1311
1312/**
1313 * amdgpu_device_enable_virtual_display - enable virtual display feature
1314 *
1315 * @adev: amdgpu_device pointer
1316 *
1317 * Enabled the virtual display feature if the user has enabled it via
1318 * the module parameter virtual_display. This feature provides a virtual
1319 * display hardware on headless boards or in virtualized environments.
1320 * This function parses and validates the configuration string specified by
1321 * the user and configues the virtual display configuration (number of
1322 * virtual connectors, crtcs, etc.) specified.
1323 */
1324static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1325{
1326 adev->enable_virtual_display = false;
1327
1328 if (amdgpu_virtual_display) {
1329 struct drm_device *ddev = adev->ddev;
1330 const char *pci_address_name = pci_name(ddev->pdev);
1331 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1332
1333 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1334 pciaddstr_tmp = pciaddstr;
1335 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1336 pciaddname = strsep(&pciaddname_tmp, ",");
1337 if (!strcmp("all", pciaddname)
1338 || !strcmp(pci_address_name, pciaddname)) {
1339 long num_crtc;
1340 int res = -1;
1341
1342 adev->enable_virtual_display = true;
1343
1344 if (pciaddname_tmp)
1345 res = kstrtol(pciaddname_tmp, 10,
1346 &num_crtc);
1347
1348 if (!res) {
1349 if (num_crtc < 1)
1350 num_crtc = 1;
1351 if (num_crtc > 6)
1352 num_crtc = 6;
1353 adev->mode_info.num_crtc = num_crtc;
1354 } else {
1355 adev->mode_info.num_crtc = 1;
1356 }
1357 break;
1358 }
1359 }
1360
1361 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1362 amdgpu_virtual_display, pci_address_name,
1363 adev->enable_virtual_display, adev->mode_info.num_crtc);
1364
1365 kfree(pciaddstr);
1366 }
1367}
1368
1369/**
1370 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1371 *
1372 * @adev: amdgpu_device pointer
1373 *
1374 * Parses the asic configuration parameters specified in the gpu info
1375 * firmware and makes them availale to the driver for use in configuring
1376 * the asic.
1377 * Returns 0 on success, -EINVAL on failure.
1378 */
1379static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1380{
1381 const char *chip_name;
1382 char fw_name[30];
1383 int err;
1384 const struct gpu_info_firmware_header_v1_0 *hdr;
1385
1386 adev->firmware.gpu_info_fw = NULL;
1387
1388 switch (adev->asic_type) {
1389 case CHIP_TOPAZ:
1390 case CHIP_TONGA:
1391 case CHIP_FIJI:
1392 case CHIP_POLARIS10:
1393 case CHIP_POLARIS11:
1394 case CHIP_POLARIS12:
1395 case CHIP_VEGAM:
1396 case CHIP_CARRIZO:
1397 case CHIP_STONEY:
1398#ifdef CONFIG_DRM_AMDGPU_SI
1399 case CHIP_VERDE:
1400 case CHIP_TAHITI:
1401 case CHIP_PITCAIRN:
1402 case CHIP_OLAND:
1403 case CHIP_HAINAN:
1404#endif
1405#ifdef CONFIG_DRM_AMDGPU_CIK
1406 case CHIP_BONAIRE:
1407 case CHIP_HAWAII:
1408 case CHIP_KAVERI:
1409 case CHIP_KABINI:
1410 case CHIP_MULLINS:
1411#endif
1412 case CHIP_VEGA20:
1413 default:
1414 return 0;
1415 case CHIP_VEGA10:
1416 chip_name = "vega10";
1417 break;
1418 case CHIP_VEGA12:
1419 chip_name = "vega12";
1420 break;
1421 case CHIP_RAVEN:
1422 if (adev->rev_id >= 8)
1423 chip_name = "raven2";
1424 else if (adev->pdev->device == 0x15d8)
1425 chip_name = "picasso";
1426 else
1427 chip_name = "raven";
1428 break;
1429 case CHIP_ARCTURUS:
1430 chip_name = "arcturus";
1431 break;
1432 case CHIP_RENOIR:
1433 chip_name = "renoir";
1434 break;
1435 case CHIP_NAVI10:
1436 chip_name = "navi10";
1437 break;
1438 case CHIP_NAVI14:
1439 chip_name = "navi14";
1440 break;
1441 case CHIP_NAVI12:
1442 chip_name = "navi12";
1443 break;
1444 }
1445
1446 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1447 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1448 if (err) {
1449 dev_err(adev->dev,
1450 "Failed to load gpu_info firmware \"%s\"\n",
1451 fw_name);
1452 goto out;
1453 }
1454 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1455 if (err) {
1456 dev_err(adev->dev,
1457 "Failed to validate gpu_info firmware \"%s\"\n",
1458 fw_name);
1459 goto out;
1460 }
1461
1462 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1463 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1464
1465 switch (hdr->version_major) {
1466 case 1:
1467 {
1468 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1469 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1470 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1471
1472 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1473 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1474 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1475 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1476 adev->gfx.config.max_texture_channel_caches =
1477 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1478 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1479 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1480 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1481 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1482 adev->gfx.config.double_offchip_lds_buf =
1483 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1484 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1485 adev->gfx.cu_info.max_waves_per_simd =
1486 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1487 adev->gfx.cu_info.max_scratch_slots_per_cu =
1488 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1489 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1490 if (hdr->version_minor >= 1) {
1491 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1492 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1493 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1494 adev->gfx.config.num_sc_per_sh =
1495 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1496 adev->gfx.config.num_packer_per_sc =
1497 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1498 }
1499#ifdef CONFIG_DRM_AMD_DC_DCN2_0
1500 if (hdr->version_minor == 2) {
1501 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1502 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1503 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1504 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1505 }
1506#endif
1507 break;
1508 }
1509 default:
1510 dev_err(adev->dev,
1511 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1512 err = -EINVAL;
1513 goto out;
1514 }
1515out:
1516 return err;
1517}
1518
1519/**
1520 * amdgpu_device_ip_early_init - run early init for hardware IPs
1521 *
1522 * @adev: amdgpu_device pointer
1523 *
1524 * Early initialization pass for hardware IPs. The hardware IPs that make
1525 * up each asic are discovered each IP's early_init callback is run. This
1526 * is the first stage in initializing the asic.
1527 * Returns 0 on success, negative error code on failure.
1528 */
1529static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1530{
1531 int i, r;
1532
1533 amdgpu_device_enable_virtual_display(adev);
1534
1535 switch (adev->asic_type) {
1536 case CHIP_TOPAZ:
1537 case CHIP_TONGA:
1538 case CHIP_FIJI:
1539 case CHIP_POLARIS10:
1540 case CHIP_POLARIS11:
1541 case CHIP_POLARIS12:
1542 case CHIP_VEGAM:
1543 case CHIP_CARRIZO:
1544 case CHIP_STONEY:
1545 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1546 adev->family = AMDGPU_FAMILY_CZ;
1547 else
1548 adev->family = AMDGPU_FAMILY_VI;
1549
1550 r = vi_set_ip_blocks(adev);
1551 if (r)
1552 return r;
1553 break;
1554#ifdef CONFIG_DRM_AMDGPU_SI
1555 case CHIP_VERDE:
1556 case CHIP_TAHITI:
1557 case CHIP_PITCAIRN:
1558 case CHIP_OLAND:
1559 case CHIP_HAINAN:
1560 adev->family = AMDGPU_FAMILY_SI;
1561 r = si_set_ip_blocks(adev);
1562 if (r)
1563 return r;
1564 break;
1565#endif
1566#ifdef CONFIG_DRM_AMDGPU_CIK
1567 case CHIP_BONAIRE:
1568 case CHIP_HAWAII:
1569 case CHIP_KAVERI:
1570 case CHIP_KABINI:
1571 case CHIP_MULLINS:
1572 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1573 adev->family = AMDGPU_FAMILY_CI;
1574 else
1575 adev->family = AMDGPU_FAMILY_KV;
1576
1577 r = cik_set_ip_blocks(adev);
1578 if (r)
1579 return r;
1580 break;
1581#endif
1582 case CHIP_VEGA10:
1583 case CHIP_VEGA12:
1584 case CHIP_VEGA20:
1585 case CHIP_RAVEN:
1586 case CHIP_ARCTURUS:
1587 case CHIP_RENOIR:
1588 if (adev->asic_type == CHIP_RAVEN ||
1589 adev->asic_type == CHIP_RENOIR)
1590 adev->family = AMDGPU_FAMILY_RV;
1591 else
1592 adev->family = AMDGPU_FAMILY_AI;
1593
1594 r = soc15_set_ip_blocks(adev);
1595 if (r)
1596 return r;
1597 break;
1598 case CHIP_NAVI10:
1599 case CHIP_NAVI14:
1600 case CHIP_NAVI12:
1601 adev->family = AMDGPU_FAMILY_NV;
1602
1603 r = nv_set_ip_blocks(adev);
1604 if (r)
1605 return r;
1606 break;
1607 default:
1608 /* FIXME: not supported yet */
1609 return -EINVAL;
1610 }
1611
1612 r = amdgpu_device_parse_gpu_info_fw(adev);
1613 if (r)
1614 return r;
1615
1616 amdgpu_amdkfd_device_probe(adev);
1617
1618 if (amdgpu_sriov_vf(adev)) {
1619 r = amdgpu_virt_request_full_gpu(adev, true);
1620 if (r)
1621 return -EAGAIN;
1622 }
1623
1624 adev->pm.pp_feature = amdgpu_pp_feature_mask;
1625 if (amdgpu_sriov_vf(adev))
1626 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1627
1628 for (i = 0; i < adev->num_ip_blocks; i++) {
1629 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1630 DRM_ERROR("disabled ip block: %d <%s>\n",
1631 i, adev->ip_blocks[i].version->funcs->name);
1632 adev->ip_blocks[i].status.valid = false;
1633 } else {
1634 if (adev->ip_blocks[i].version->funcs->early_init) {
1635 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1636 if (r == -ENOENT) {
1637 adev->ip_blocks[i].status.valid = false;
1638 } else if (r) {
1639 DRM_ERROR("early_init of IP block <%s> failed %d\n",
1640 adev->ip_blocks[i].version->funcs->name, r);
1641 return r;
1642 } else {
1643 adev->ip_blocks[i].status.valid = true;
1644 }
1645 } else {
1646 adev->ip_blocks[i].status.valid = true;
1647 }
1648 }
1649 /* get the vbios after the asic_funcs are set up */
1650 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
1651 /* Read BIOS */
1652 if (!amdgpu_get_bios(adev))
1653 return -EINVAL;
1654
1655 r = amdgpu_atombios_init(adev);
1656 if (r) {
1657 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
1658 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
1659 return r;
1660 }
1661 }
1662 }
1663
1664 adev->cg_flags &= amdgpu_cg_mask;
1665 adev->pg_flags &= amdgpu_pg_mask;
1666
1667 return 0;
1668}
1669
1670static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
1671{
1672 int i, r;
1673
1674 for (i = 0; i < adev->num_ip_blocks; i++) {
1675 if (!adev->ip_blocks[i].status.sw)
1676 continue;
1677 if (adev->ip_blocks[i].status.hw)
1678 continue;
1679 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1680 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
1681 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
1682 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1683 if (r) {
1684 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1685 adev->ip_blocks[i].version->funcs->name, r);
1686 return r;
1687 }
1688 adev->ip_blocks[i].status.hw = true;
1689 }
1690 }
1691
1692 return 0;
1693}
1694
1695static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
1696{
1697 int i, r;
1698
1699 for (i = 0; i < adev->num_ip_blocks; i++) {
1700 if (!adev->ip_blocks[i].status.sw)
1701 continue;
1702 if (adev->ip_blocks[i].status.hw)
1703 continue;
1704 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1705 if (r) {
1706 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1707 adev->ip_blocks[i].version->funcs->name, r);
1708 return r;
1709 }
1710 adev->ip_blocks[i].status.hw = true;
1711 }
1712
1713 return 0;
1714}
1715
1716static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
1717{
1718 int r = 0;
1719 int i;
1720 uint32_t smu_version;
1721
1722 if (adev->asic_type >= CHIP_VEGA10) {
1723 for (i = 0; i < adev->num_ip_blocks; i++) {
1724 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
1725 continue;
1726
1727 /* no need to do the fw loading again if already done*/
1728 if (adev->ip_blocks[i].status.hw == true)
1729 break;
1730
1731 if (adev->in_gpu_reset || adev->in_suspend) {
1732 r = adev->ip_blocks[i].version->funcs->resume(adev);
1733 if (r) {
1734 DRM_ERROR("resume of IP block <%s> failed %d\n",
1735 adev->ip_blocks[i].version->funcs->name, r);
1736 return r;
1737 }
1738 } else {
1739 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1740 if (r) {
1741 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1742 adev->ip_blocks[i].version->funcs->name, r);
1743 return r;
1744 }
1745 }
1746
1747 adev->ip_blocks[i].status.hw = true;
1748 break;
1749 }
1750 }
1751
1752 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
1753
1754 return r;
1755}
1756
1757/**
1758 * amdgpu_device_ip_init - run init for hardware IPs
1759 *
1760 * @adev: amdgpu_device pointer
1761 *
1762 * Main initialization pass for hardware IPs. The list of all the hardware
1763 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
1764 * are run. sw_init initializes the software state associated with each IP
1765 * and hw_init initializes the hardware associated with each IP.
1766 * Returns 0 on success, negative error code on failure.
1767 */
1768static int amdgpu_device_ip_init(struct amdgpu_device *adev)
1769{
1770 int i, r;
1771
1772 r = amdgpu_ras_init(adev);
1773 if (r)
1774 return r;
1775
1776 for (i = 0; i < adev->num_ip_blocks; i++) {
1777 if (!adev->ip_blocks[i].status.valid)
1778 continue;
1779 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1780 if (r) {
1781 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1782 adev->ip_blocks[i].version->funcs->name, r);
1783 goto init_failed;
1784 }
1785 adev->ip_blocks[i].status.sw = true;
1786
1787 /* need to do gmc hw init early so we can allocate gpu mem */
1788 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1789 r = amdgpu_device_vram_scratch_init(adev);
1790 if (r) {
1791 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1792 goto init_failed;
1793 }
1794 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1795 if (r) {
1796 DRM_ERROR("hw_init %d failed %d\n", i, r);
1797 goto init_failed;
1798 }
1799 r = amdgpu_device_wb_init(adev);
1800 if (r) {
1801 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
1802 goto init_failed;
1803 }
1804 adev->ip_blocks[i].status.hw = true;
1805
1806 /* right after GMC hw init, we create CSA */
1807 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
1808 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
1809 AMDGPU_GEM_DOMAIN_VRAM,
1810 AMDGPU_CSA_SIZE);
1811 if (r) {
1812 DRM_ERROR("allocate CSA failed %d\n", r);
1813 goto init_failed;
1814 }
1815 }
1816 }
1817 }
1818
1819 r = amdgpu_ib_pool_init(adev);
1820 if (r) {
1821 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
1822 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
1823 goto init_failed;
1824 }
1825
1826 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
1827 if (r)
1828 goto init_failed;
1829
1830 r = amdgpu_device_ip_hw_init_phase1(adev);
1831 if (r)
1832 goto init_failed;
1833
1834 r = amdgpu_device_fw_loading(adev);
1835 if (r)
1836 goto init_failed;
1837
1838 r = amdgpu_device_ip_hw_init_phase2(adev);
1839 if (r)
1840 goto init_failed;
1841
1842 if (adev->gmc.xgmi.num_physical_nodes > 1)
1843 amdgpu_xgmi_add_device(adev);
1844 amdgpu_amdkfd_device_init(adev);
1845
1846init_failed:
1847 if (amdgpu_sriov_vf(adev)) {
1848 if (!r)
1849 amdgpu_virt_init_data_exchange(adev);
1850 amdgpu_virt_release_full_gpu(adev, true);
1851 }
1852
1853 return r;
1854}
1855
1856/**
1857 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
1858 *
1859 * @adev: amdgpu_device pointer
1860 *
1861 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
1862 * this function before a GPU reset. If the value is retained after a
1863 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
1864 */
1865static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1866{
1867 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1868}
1869
1870/**
1871 * amdgpu_device_check_vram_lost - check if vram is valid
1872 *
1873 * @adev: amdgpu_device pointer
1874 *
1875 * Checks the reset magic value written to the gart pointer in VRAM.
1876 * The driver calls this after a GPU reset to see if the contents of
1877 * VRAM is lost or now.
1878 * returns true if vram is lost, false if not.
1879 */
1880static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1881{
1882 return !!memcmp(adev->gart.ptr, adev->reset_magic,
1883 AMDGPU_RESET_MAGIC_NUM);
1884}
1885
1886/**
1887 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
1888 *
1889 * @adev: amdgpu_device pointer
1890 *
1891 * The list of all the hardware IPs that make up the asic is walked and the
1892 * set_clockgating_state callbacks are run.
1893 * Late initialization pass enabling clockgating for hardware IPs.
1894 * Fini or suspend, pass disabling clockgating for hardware IPs.
1895 * Returns 0 on success, negative error code on failure.
1896 */
1897
1898static int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
1899 enum amd_clockgating_state state)
1900{
1901 int i, j, r;
1902
1903 if (amdgpu_emu_mode == 1)
1904 return 0;
1905
1906 for (j = 0; j < adev->num_ip_blocks; j++) {
1907 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
1908 if (!adev->ip_blocks[i].status.late_initialized)
1909 continue;
1910 /* skip CG for VCE/UVD, it's handled specially */
1911 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1912 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1913 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1914 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1915 /* enable clockgating to save power */
1916 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1917 state);
1918 if (r) {
1919 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1920 adev->ip_blocks[i].version->funcs->name, r);
1921 return r;
1922 }
1923 }
1924 }
1925
1926 return 0;
1927}
1928
1929static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state)
1930{
1931 int i, j, r;
1932
1933 if (amdgpu_emu_mode == 1)
1934 return 0;
1935
1936 for (j = 0; j < adev->num_ip_blocks; j++) {
1937 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
1938 if (!adev->ip_blocks[i].status.late_initialized)
1939 continue;
1940 /* skip CG for VCE/UVD, it's handled specially */
1941 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1942 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1943 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1944 adev->ip_blocks[i].version->funcs->set_powergating_state) {
1945 /* enable powergating to save power */
1946 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
1947 state);
1948 if (r) {
1949 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
1950 adev->ip_blocks[i].version->funcs->name, r);
1951 return r;
1952 }
1953 }
1954 }
1955 return 0;
1956}
1957
1958static int amdgpu_device_enable_mgpu_fan_boost(void)
1959{
1960 struct amdgpu_gpu_instance *gpu_ins;
1961 struct amdgpu_device *adev;
1962 int i, ret = 0;
1963
1964 mutex_lock(&mgpu_info.mutex);
1965
1966 /*
1967 * MGPU fan boost feature should be enabled
1968 * only when there are two or more dGPUs in
1969 * the system
1970 */
1971 if (mgpu_info.num_dgpu < 2)
1972 goto out;
1973
1974 for (i = 0; i < mgpu_info.num_dgpu; i++) {
1975 gpu_ins = &(mgpu_info.gpu_ins[i]);
1976 adev = gpu_ins->adev;
1977 if (!(adev->flags & AMD_IS_APU) &&
1978 !gpu_ins->mgpu_fan_enabled &&
1979 adev->powerplay.pp_funcs &&
1980 adev->powerplay.pp_funcs->enable_mgpu_fan_boost) {
1981 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
1982 if (ret)
1983 break;
1984
1985 gpu_ins->mgpu_fan_enabled = 1;
1986 }
1987 }
1988
1989out:
1990 mutex_unlock(&mgpu_info.mutex);
1991
1992 return ret;
1993}
1994
1995/**
1996 * amdgpu_device_ip_late_init - run late init for hardware IPs
1997 *
1998 * @adev: amdgpu_device pointer
1999 *
2000 * Late initialization pass for hardware IPs. The list of all the hardware
2001 * IPs that make up the asic is walked and the late_init callbacks are run.
2002 * late_init covers any special initialization that an IP requires
2003 * after all of the have been initialized or something that needs to happen
2004 * late in the init process.
2005 * Returns 0 on success, negative error code on failure.
2006 */
2007static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2008{
2009 int i = 0, r;
2010
2011 for (i = 0; i < adev->num_ip_blocks; i++) {
2012 if (!adev->ip_blocks[i].status.hw)
2013 continue;
2014 if (adev->ip_blocks[i].version->funcs->late_init) {
2015 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2016 if (r) {
2017 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2018 adev->ip_blocks[i].version->funcs->name, r);
2019 return r;
2020 }
2021 }
2022 adev->ip_blocks[i].status.late_initialized = true;
2023 }
2024
2025 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2026 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2027
2028 amdgpu_device_fill_reset_magic(adev);
2029
2030 r = amdgpu_device_enable_mgpu_fan_boost();
2031 if (r)
2032 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2033
2034 /* set to low pstate by default */
2035 amdgpu_xgmi_set_pstate(adev, 0);
2036
2037 return 0;
2038}
2039
2040/**
2041 * amdgpu_device_ip_fini - run fini for hardware IPs
2042 *
2043 * @adev: amdgpu_device pointer
2044 *
2045 * Main teardown pass for hardware IPs. The list of all the hardware
2046 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2047 * are run. hw_fini tears down the hardware associated with each IP
2048 * and sw_fini tears down any software state associated with each IP.
2049 * Returns 0 on success, negative error code on failure.
2050 */
2051static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2052{
2053 int i, r;
2054
2055 amdgpu_ras_pre_fini(adev);
2056
2057 if (adev->gmc.xgmi.num_physical_nodes > 1)
2058 amdgpu_xgmi_remove_device(adev);
2059
2060 amdgpu_amdkfd_device_fini(adev);
2061
2062 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2063 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2064
2065 /* need to disable SMC first */
2066 for (i = 0; i < adev->num_ip_blocks; i++) {
2067 if (!adev->ip_blocks[i].status.hw)
2068 continue;
2069 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2070 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2071 /* XXX handle errors */
2072 if (r) {
2073 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2074 adev->ip_blocks[i].version->funcs->name, r);
2075 }
2076 adev->ip_blocks[i].status.hw = false;
2077 break;
2078 }
2079 }
2080
2081 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2082 if (!adev->ip_blocks[i].status.hw)
2083 continue;
2084
2085 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2086 /* XXX handle errors */
2087 if (r) {
2088 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2089 adev->ip_blocks[i].version->funcs->name, r);
2090 }
2091
2092 adev->ip_blocks[i].status.hw = false;
2093 }
2094
2095
2096 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2097 if (!adev->ip_blocks[i].status.sw)
2098 continue;
2099
2100 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2101 amdgpu_ucode_free_bo(adev);
2102 amdgpu_free_static_csa(&adev->virt.csa_obj);
2103 amdgpu_device_wb_fini(adev);
2104 amdgpu_device_vram_scratch_fini(adev);
2105 amdgpu_ib_pool_fini(adev);
2106 }
2107
2108 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2109 /* XXX handle errors */
2110 if (r) {
2111 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2112 adev->ip_blocks[i].version->funcs->name, r);
2113 }
2114 adev->ip_blocks[i].status.sw = false;
2115 adev->ip_blocks[i].status.valid = false;
2116 }
2117
2118 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2119 if (!adev->ip_blocks[i].status.late_initialized)
2120 continue;
2121 if (adev->ip_blocks[i].version->funcs->late_fini)
2122 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2123 adev->ip_blocks[i].status.late_initialized = false;
2124 }
2125
2126 amdgpu_ras_fini(adev);
2127
2128 if (amdgpu_sriov_vf(adev))
2129 if (amdgpu_virt_release_full_gpu(adev, false))
2130 DRM_ERROR("failed to release exclusive mode on fini\n");
2131
2132 return 0;
2133}
2134
2135/**
2136 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2137 *
2138 * @work: work_struct.
2139 */
2140static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2141{
2142 struct amdgpu_device *adev =
2143 container_of(work, struct amdgpu_device, delayed_init_work.work);
2144 int r;
2145
2146 r = amdgpu_ib_ring_tests(adev);
2147 if (r)
2148 DRM_ERROR("ib ring test failed (%d).\n", r);
2149}
2150
2151static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2152{
2153 struct amdgpu_device *adev =
2154 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2155
2156 mutex_lock(&adev->gfx.gfx_off_mutex);
2157 if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
2158 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2159 adev->gfx.gfx_off_state = true;
2160 }
2161 mutex_unlock(&adev->gfx.gfx_off_mutex);
2162}
2163
2164/**
2165 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2166 *
2167 * @adev: amdgpu_device pointer
2168 *
2169 * Main suspend function for hardware IPs. The list of all the hardware
2170 * IPs that make up the asic is walked, clockgating is disabled and the
2171 * suspend callbacks are run. suspend puts the hardware and software state
2172 * in each IP into a state suitable for suspend.
2173 * Returns 0 on success, negative error code on failure.
2174 */
2175static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2176{
2177 int i, r;
2178
2179 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2180 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2181
2182 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2183 if (!adev->ip_blocks[i].status.valid)
2184 continue;
2185 /* displays are handled separately */
2186 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
2187 /* XXX handle errors */
2188 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2189 /* XXX handle errors */
2190 if (r) {
2191 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2192 adev->ip_blocks[i].version->funcs->name, r);
2193 return r;
2194 }
2195 adev->ip_blocks[i].status.hw = false;
2196 }
2197 }
2198
2199 return 0;
2200}
2201
2202/**
2203 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2204 *
2205 * @adev: amdgpu_device pointer
2206 *
2207 * Main suspend function for hardware IPs. The list of all the hardware
2208 * IPs that make up the asic is walked, clockgating is disabled and the
2209 * suspend callbacks are run. suspend puts the hardware and software state
2210 * in each IP into a state suitable for suspend.
2211 * Returns 0 on success, negative error code on failure.
2212 */
2213static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2214{
2215 int i, r;
2216
2217 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2218 if (!adev->ip_blocks[i].status.valid)
2219 continue;
2220 /* displays are handled in phase1 */
2221 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2222 continue;
2223 /* XXX handle errors */
2224 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2225 /* XXX handle errors */
2226 if (r) {
2227 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2228 adev->ip_blocks[i].version->funcs->name, r);
2229 }
2230 adev->ip_blocks[i].status.hw = false;
2231 /* handle putting the SMC in the appropriate state */
2232 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2233 if (is_support_sw_smu(adev)) {
2234 /* todo */
2235 } else if (adev->powerplay.pp_funcs &&
2236 adev->powerplay.pp_funcs->set_mp1_state) {
2237 r = adev->powerplay.pp_funcs->set_mp1_state(
2238 adev->powerplay.pp_handle,
2239 adev->mp1_state);
2240 if (r) {
2241 DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2242 adev->mp1_state, r);
2243 return r;
2244 }
2245 }
2246 }
2247
2248 adev->ip_blocks[i].status.hw = false;
2249 }
2250
2251 return 0;
2252}
2253
2254/**
2255 * amdgpu_device_ip_suspend - run suspend for hardware IPs
2256 *
2257 * @adev: amdgpu_device pointer
2258 *
2259 * Main suspend function for hardware IPs. The list of all the hardware
2260 * IPs that make up the asic is walked, clockgating is disabled and the
2261 * suspend callbacks are run. suspend puts the hardware and software state
2262 * in each IP into a state suitable for suspend.
2263 * Returns 0 on success, negative error code on failure.
2264 */
2265int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2266{
2267 int r;
2268
2269 if (amdgpu_sriov_vf(adev))
2270 amdgpu_virt_request_full_gpu(adev, false);
2271
2272 r = amdgpu_device_ip_suspend_phase1(adev);
2273 if (r)
2274 return r;
2275 r = amdgpu_device_ip_suspend_phase2(adev);
2276
2277 if (amdgpu_sriov_vf(adev))
2278 amdgpu_virt_release_full_gpu(adev, false);
2279
2280 return r;
2281}
2282
2283static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2284{
2285 int i, r;
2286
2287 static enum amd_ip_block_type ip_order[] = {
2288 AMD_IP_BLOCK_TYPE_GMC,
2289 AMD_IP_BLOCK_TYPE_COMMON,
2290 AMD_IP_BLOCK_TYPE_PSP,
2291 AMD_IP_BLOCK_TYPE_IH,
2292 };
2293
2294 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2295 int j;
2296 struct amdgpu_ip_block *block;
2297
2298 for (j = 0; j < adev->num_ip_blocks; j++) {
2299 block = &adev->ip_blocks[j];
2300
2301 block->status.hw = false;
2302 if (block->version->type != ip_order[i] ||
2303 !block->status.valid)
2304 continue;
2305
2306 r = block->version->funcs->hw_init(adev);
2307 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2308 if (r)
2309 return r;
2310 block->status.hw = true;
2311 }
2312 }
2313
2314 return 0;
2315}
2316
2317static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2318{
2319 int i, r;
2320
2321 static enum amd_ip_block_type ip_order[] = {
2322 AMD_IP_BLOCK_TYPE_SMC,
2323 AMD_IP_BLOCK_TYPE_DCE,
2324 AMD_IP_BLOCK_TYPE_GFX,
2325 AMD_IP_BLOCK_TYPE_SDMA,
2326 AMD_IP_BLOCK_TYPE_UVD,
2327 AMD_IP_BLOCK_TYPE_VCE
2328 };
2329
2330 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2331 int j;
2332 struct amdgpu_ip_block *block;
2333
2334 for (j = 0; j < adev->num_ip_blocks; j++) {
2335 block = &adev->ip_blocks[j];
2336
2337 if (block->version->type != ip_order[i] ||
2338 !block->status.valid ||
2339 block->status.hw)
2340 continue;
2341
2342 r = block->version->funcs->hw_init(adev);
2343 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2344 if (r)
2345 return r;
2346 block->status.hw = true;
2347 }
2348 }
2349
2350 return 0;
2351}
2352
2353/**
2354 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2355 *
2356 * @adev: amdgpu_device pointer
2357 *
2358 * First resume function for hardware IPs. The list of all the hardware
2359 * IPs that make up the asic is walked and the resume callbacks are run for
2360 * COMMON, GMC, and IH. resume puts the hardware into a functional state
2361 * after a suspend and updates the software state as necessary. This
2362 * function is also used for restoring the GPU after a GPU reset.
2363 * Returns 0 on success, negative error code on failure.
2364 */
2365static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
2366{
2367 int i, r;
2368
2369 for (i = 0; i < adev->num_ip_blocks; i++) {
2370 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2371 continue;
2372 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2373 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2374 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2375
2376 r = adev->ip_blocks[i].version->funcs->resume(adev);
2377 if (r) {
2378 DRM_ERROR("resume of IP block <%s> failed %d\n",
2379 adev->ip_blocks[i].version->funcs->name, r);
2380 return r;
2381 }
2382 adev->ip_blocks[i].status.hw = true;
2383 }
2384 }
2385
2386 return 0;
2387}
2388
2389/**
2390 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2391 *
2392 * @adev: amdgpu_device pointer
2393 *
2394 * First resume function for hardware IPs. The list of all the hardware
2395 * IPs that make up the asic is walked and the resume callbacks are run for
2396 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
2397 * functional state after a suspend and updates the software state as
2398 * necessary. This function is also used for restoring the GPU after a GPU
2399 * reset.
2400 * Returns 0 on success, negative error code on failure.
2401 */
2402static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
2403{
2404 int i, r;
2405
2406 for (i = 0; i < adev->num_ip_blocks; i++) {
2407 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2408 continue;
2409 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2410 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2411 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
2412 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
2413 continue;
2414 r = adev->ip_blocks[i].version->funcs->resume(adev);
2415 if (r) {
2416 DRM_ERROR("resume of IP block <%s> failed %d\n",
2417 adev->ip_blocks[i].version->funcs->name, r);
2418 return r;
2419 }
2420 adev->ip_blocks[i].status.hw = true;
2421 }
2422
2423 return 0;
2424}
2425
2426/**
2427 * amdgpu_device_ip_resume - run resume for hardware IPs
2428 *
2429 * @adev: amdgpu_device pointer
2430 *
2431 * Main resume function for hardware IPs. The hardware IPs
2432 * are split into two resume functions because they are
2433 * are also used in in recovering from a GPU reset and some additional
2434 * steps need to be take between them. In this case (S3/S4) they are
2435 * run sequentially.
2436 * Returns 0 on success, negative error code on failure.
2437 */
2438static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2439{
2440 int r;
2441
2442 r = amdgpu_device_ip_resume_phase1(adev);
2443 if (r)
2444 return r;
2445
2446 r = amdgpu_device_fw_loading(adev);
2447 if (r)
2448 return r;
2449
2450 r = amdgpu_device_ip_resume_phase2(adev);
2451
2452 return r;
2453}
2454
2455/**
2456 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2457 *
2458 * @adev: amdgpu_device pointer
2459 *
2460 * Query the VBIOS data tables to determine if the board supports SR-IOV.
2461 */
2462static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2463{
2464 if (amdgpu_sriov_vf(adev)) {
2465 if (adev->is_atom_fw) {
2466 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2467 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2468 } else {
2469 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2470 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2471 }
2472
2473 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2474 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2475 }
2476}
2477
2478/**
2479 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
2480 *
2481 * @asic_type: AMD asic type
2482 *
2483 * Check if there is DC (new modesetting infrastructre) support for an asic.
2484 * returns true if DC has support, false if not.
2485 */
2486bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
2487{
2488 switch (asic_type) {
2489#if defined(CONFIG_DRM_AMD_DC)
2490 case CHIP_BONAIRE:
2491 case CHIP_KAVERI:
2492 case CHIP_KABINI:
2493 case CHIP_MULLINS:
2494 /*
2495 * We have systems in the wild with these ASICs that require
2496 * LVDS and VGA support which is not supported with DC.
2497 *
2498 * Fallback to the non-DC driver here by default so as not to
2499 * cause regressions.
2500 */
2501 return amdgpu_dc > 0;
2502 case CHIP_HAWAII:
2503 case CHIP_CARRIZO:
2504 case CHIP_STONEY:
2505 case CHIP_POLARIS10:
2506 case CHIP_POLARIS11:
2507 case CHIP_POLARIS12:
2508 case CHIP_VEGAM:
2509 case CHIP_TONGA:
2510 case CHIP_FIJI:
2511 case CHIP_VEGA10:
2512 case CHIP_VEGA12:
2513 case CHIP_VEGA20:
2514#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
2515 case CHIP_RAVEN:
2516#endif
2517#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
2518 case CHIP_NAVI10:
2519 case CHIP_NAVI14:
2520 case CHIP_NAVI12:
2521#endif
2522#if defined(CONFIG_DRM_AMD_DC_DCN2_1)
2523 case CHIP_RENOIR:
2524#endif
2525 return amdgpu_dc != 0;
2526#endif
2527 default:
2528 return false;
2529 }
2530}
2531
2532/**
2533 * amdgpu_device_has_dc_support - check if dc is supported
2534 *
2535 * @adev: amdgpu_device_pointer
2536 *
2537 * Returns true for supported, false for not supported
2538 */
2539bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
2540{
2541 if (amdgpu_sriov_vf(adev))
2542 return false;
2543
2544 return amdgpu_device_asic_has_dc_support(adev->asic_type);
2545}
2546
2547
2548static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
2549{
2550 struct amdgpu_device *adev =
2551 container_of(__work, struct amdgpu_device, xgmi_reset_work);
2552
2553 adev->asic_reset_res = amdgpu_asic_reset(adev);
2554 if (adev->asic_reset_res)
2555 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
2556 adev->asic_reset_res, adev->ddev->unique);
2557}
2558
2559
2560/**
2561 * amdgpu_device_init - initialize the driver
2562 *
2563 * @adev: amdgpu_device pointer
2564 * @ddev: drm dev pointer
2565 * @pdev: pci dev pointer
2566 * @flags: driver flags
2567 *
2568 * Initializes the driver info and hw (all asics).
2569 * Returns 0 for success or an error on failure.
2570 * Called at driver startup.
2571 */
2572int amdgpu_device_init(struct amdgpu_device *adev,
2573 struct drm_device *ddev,
2574 struct pci_dev *pdev,
2575 uint32_t flags)
2576{
2577 int r, i;
2578 bool runtime = false;
2579 u32 max_MBps;
2580
2581 adev->shutdown = false;
2582 adev->dev = &pdev->dev;
2583 adev->ddev = ddev;
2584 adev->pdev = pdev;
2585 adev->flags = flags;
2586 adev->asic_type = flags & AMD_ASIC_MASK;
2587 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2588 if (amdgpu_emu_mode == 1)
2589 adev->usec_timeout *= 2;
2590 adev->gmc.gart_size = 512 * 1024 * 1024;
2591 adev->accel_working = false;
2592 adev->num_rings = 0;
2593 adev->mman.buffer_funcs = NULL;
2594 adev->mman.buffer_funcs_ring = NULL;
2595 adev->vm_manager.vm_pte_funcs = NULL;
2596 adev->vm_manager.vm_pte_num_rqs = 0;
2597 adev->gmc.gmc_funcs = NULL;
2598 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2599 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2600
2601 adev->smc_rreg = &amdgpu_invalid_rreg;
2602 adev->smc_wreg = &amdgpu_invalid_wreg;
2603 adev->pcie_rreg = &amdgpu_invalid_rreg;
2604 adev->pcie_wreg = &amdgpu_invalid_wreg;
2605 adev->pciep_rreg = &amdgpu_invalid_rreg;
2606 adev->pciep_wreg = &amdgpu_invalid_wreg;
2607 adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
2608 adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
2609 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2610 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2611 adev->didt_rreg = &amdgpu_invalid_rreg;
2612 adev->didt_wreg = &amdgpu_invalid_wreg;
2613 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2614 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2615 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2616 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2617
2618 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2619 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2620 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2621
2622 /* mutex initialization are all done here so we
2623 * can recall function without having locking issues */
2624 atomic_set(&adev->irq.ih.lock, 0);
2625 mutex_init(&adev->firmware.mutex);
2626 mutex_init(&adev->pm.mutex);
2627 mutex_init(&adev->gfx.gpu_clock_mutex);
2628 mutex_init(&adev->srbm_mutex);
2629 mutex_init(&adev->gfx.pipe_reserve_mutex);
2630 mutex_init(&adev->gfx.gfx_off_mutex);
2631 mutex_init(&adev->grbm_idx_mutex);
2632 mutex_init(&adev->mn_lock);
2633 mutex_init(&adev->virt.vf_errors.lock);
2634 hash_init(adev->mn_hash);
2635 mutex_init(&adev->lock_reset);
2636 mutex_init(&adev->virt.dpm_mutex);
2637 mutex_init(&adev->psp.mutex);
2638
2639 r = amdgpu_device_check_arguments(adev);
2640 if (r)
2641 return r;
2642
2643 spin_lock_init(&adev->mmio_idx_lock);
2644 spin_lock_init(&adev->smc_idx_lock);
2645 spin_lock_init(&adev->pcie_idx_lock);
2646 spin_lock_init(&adev->uvd_ctx_idx_lock);
2647 spin_lock_init(&adev->didt_idx_lock);
2648 spin_lock_init(&adev->gc_cac_idx_lock);
2649 spin_lock_init(&adev->se_cac_idx_lock);
2650 spin_lock_init(&adev->audio_endpt_idx_lock);
2651 spin_lock_init(&adev->mm_stats.lock);
2652
2653 INIT_LIST_HEAD(&adev->shadow_list);
2654 mutex_init(&adev->shadow_list_lock);
2655
2656 INIT_LIST_HEAD(&adev->ring_lru_list);
2657 spin_lock_init(&adev->ring_lru_list_lock);
2658
2659 INIT_DELAYED_WORK(&adev->delayed_init_work,
2660 amdgpu_device_delayed_init_work_handler);
2661 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
2662 amdgpu_device_delay_enable_gfx_off);
2663
2664 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
2665
2666 adev->gfx.gfx_off_req_count = 1;
2667 adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false;
2668
2669 /* Registers mapping */
2670 /* TODO: block userspace mapping of io register */
2671 if (adev->asic_type >= CHIP_BONAIRE) {
2672 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2673 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2674 } else {
2675 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2676 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2677 }
2678
2679 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2680 if (adev->rmmio == NULL) {
2681 return -ENOMEM;
2682 }
2683 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2684 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2685
2686 /* io port mapping */
2687 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2688 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2689 adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2690 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2691 break;
2692 }
2693 }
2694 if (adev->rio_mem == NULL)
2695 DRM_INFO("PCI I/O BAR is not found.\n");
2696
2697 /* enable PCIE atomic ops */
2698 r = pci_enable_atomic_ops_to_root(adev->pdev,
2699 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
2700 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
2701 if (r) {
2702 adev->have_atomics_support = false;
2703 DRM_INFO("PCIE atomic ops is not supported\n");
2704 } else {
2705 adev->have_atomics_support = true;
2706 }
2707
2708 amdgpu_device_get_pcie_info(adev);
2709
2710 if (amdgpu_mcbp)
2711 DRM_INFO("MCBP is enabled\n");
2712
2713 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
2714 adev->enable_mes = true;
2715
2716 if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10) {
2717 r = amdgpu_discovery_init(adev);
2718 if (r) {
2719 dev_err(adev->dev, "amdgpu_discovery_init failed\n");
2720 return r;
2721 }
2722 }
2723
2724 /* early init functions */
2725 r = amdgpu_device_ip_early_init(adev);
2726 if (r)
2727 return r;
2728
2729 /* doorbell bar mapping and doorbell index init*/
2730 amdgpu_device_doorbell_init(adev);
2731
2732 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2733 /* this will fail for cards that aren't VGA class devices, just
2734 * ignore it */
2735 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
2736
2737 if (amdgpu_device_is_px(ddev))
2738 runtime = true;
2739 if (!pci_is_thunderbolt_attached(adev->pdev))
2740 vga_switcheroo_register_client(adev->pdev,
2741 &amdgpu_switcheroo_ops, runtime);
2742 if (runtime)
2743 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2744
2745 if (amdgpu_emu_mode == 1) {
2746 /* post the asic on emulation mode */
2747 emu_soc_asic_init(adev);
2748 goto fence_driver_init;
2749 }
2750
2751 /* detect if we are with an SRIOV vbios */
2752 amdgpu_device_detect_sriov_bios(adev);
2753
2754 /* check if we need to reset the asic
2755 * E.g., driver was not cleanly unloaded previously, etc.
2756 */
2757 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
2758 r = amdgpu_asic_reset(adev);
2759 if (r) {
2760 dev_err(adev->dev, "asic reset on init failed\n");
2761 goto failed;
2762 }
2763 }
2764
2765 /* Post card if necessary */
2766 if (amdgpu_device_need_post(adev)) {
2767 if (!adev->bios) {
2768 dev_err(adev->dev, "no vBIOS found\n");
2769 r = -EINVAL;
2770 goto failed;
2771 }
2772 DRM_INFO("GPU posting now...\n");
2773 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2774 if (r) {
2775 dev_err(adev->dev, "gpu post error!\n");
2776 goto failed;
2777 }
2778 }
2779
2780 if (adev->is_atom_fw) {
2781 /* Initialize clocks */
2782 r = amdgpu_atomfirmware_get_clock_info(adev);
2783 if (r) {
2784 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2785 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2786 goto failed;
2787 }
2788 } else {
2789 /* Initialize clocks */
2790 r = amdgpu_atombios_get_clock_info(adev);
2791 if (r) {
2792 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2793 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2794 goto failed;
2795 }
2796 /* init i2c buses */
2797 if (!amdgpu_device_has_dc_support(adev))
2798 amdgpu_atombios_i2c_init(adev);
2799 }
2800
2801fence_driver_init:
2802 /* Fence driver */
2803 r = amdgpu_fence_driver_init(adev);
2804 if (r) {
2805 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2806 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2807 goto failed;
2808 }
2809
2810 /* init the mode config */
2811 drm_mode_config_init(adev->ddev);
2812
2813 r = amdgpu_device_ip_init(adev);
2814 if (r) {
2815 /* failed in exclusive mode due to timeout */
2816 if (amdgpu_sriov_vf(adev) &&
2817 !amdgpu_sriov_runtime(adev) &&
2818 amdgpu_virt_mmio_blocked(adev) &&
2819 !amdgpu_virt_wait_reset(adev)) {
2820 dev_err(adev->dev, "VF exclusive mode timeout\n");
2821 /* Don't send request since VF is inactive. */
2822 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2823 adev->virt.ops = NULL;
2824 r = -EAGAIN;
2825 goto failed;
2826 }
2827 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
2828 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2829 if (amdgpu_virt_request_full_gpu(adev, false))
2830 amdgpu_virt_release_full_gpu(adev, false);
2831 goto failed;
2832 }
2833
2834 adev->accel_working = true;
2835
2836 amdgpu_vm_check_compute_bug(adev);
2837
2838 /* Initialize the buffer migration limit. */
2839 if (amdgpu_moverate >= 0)
2840 max_MBps = amdgpu_moverate;
2841 else
2842 max_MBps = 8; /* Allow 8 MB/s. */
2843 /* Get a log2 for easy divisions. */
2844 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2845
2846 amdgpu_fbdev_init(adev);
2847
2848 if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
2849 amdgpu_pm_virt_sysfs_init(adev);
2850
2851 r = amdgpu_pm_sysfs_init(adev);
2852 if (r)
2853 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2854
2855 r = amdgpu_ucode_sysfs_init(adev);
2856 if (r)
2857 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
2858
2859 r = amdgpu_debugfs_gem_init(adev);
2860 if (r)
2861 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2862
2863 r = amdgpu_debugfs_regs_init(adev);
2864 if (r)
2865 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2866
2867 r = amdgpu_debugfs_firmware_init(adev);
2868 if (r)
2869 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2870
2871 r = amdgpu_debugfs_init(adev);
2872 if (r)
2873 DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2874
2875 if ((amdgpu_testing & 1)) {
2876 if (adev->accel_working)
2877 amdgpu_test_moves(adev);
2878 else
2879 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2880 }
2881 if (amdgpu_benchmarking) {
2882 if (adev->accel_working)
2883 amdgpu_benchmark(adev, amdgpu_benchmarking);
2884 else
2885 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2886 }
2887
2888 /*
2889 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
2890 * Otherwise the mgpu fan boost feature will be skipped due to the
2891 * gpu instance is counted less.
2892 */
2893 amdgpu_register_gpu_instance(adev);
2894
2895 /* enable clockgating, etc. after ib tests, etc. since some blocks require
2896 * explicit gating rather than handling it automatically.
2897 */
2898 r = amdgpu_device_ip_late_init(adev);
2899 if (r) {
2900 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
2901 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2902 goto failed;
2903 }
2904
2905 /* must succeed. */
2906 amdgpu_ras_resume(adev);
2907
2908 queue_delayed_work(system_wq, &adev->delayed_init_work,
2909 msecs_to_jiffies(AMDGPU_RESUME_MS));
2910
2911 r = device_create_file(adev->dev, &dev_attr_pcie_replay_count);
2912 if (r) {
2913 dev_err(adev->dev, "Could not create pcie_replay_count");
2914 return r;
2915 }
2916
2917 if (IS_ENABLED(CONFIG_PERF_EVENTS))
2918 r = amdgpu_pmu_init(adev);
2919 if (r)
2920 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
2921
2922 return 0;
2923
2924failed:
2925 amdgpu_vf_error_trans_all(adev);
2926 if (runtime)
2927 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2928
2929 return r;
2930}
2931
2932/**
2933 * amdgpu_device_fini - tear down the driver
2934 *
2935 * @adev: amdgpu_device pointer
2936 *
2937 * Tear down the driver info (all asics).
2938 * Called at driver shutdown.
2939 */
2940void amdgpu_device_fini(struct amdgpu_device *adev)
2941{
2942 int r;
2943
2944 DRM_INFO("amdgpu: finishing device.\n");
2945 adev->shutdown = true;
2946 /* disable all interrupts */
2947 amdgpu_irq_disable_all(adev);
2948 if (adev->mode_info.mode_config_initialized){
2949 if (!amdgpu_device_has_dc_support(adev))
2950 drm_helper_force_disable_all(adev->ddev);
2951 else
2952 drm_atomic_helper_shutdown(adev->ddev);
2953 }
2954 amdgpu_fence_driver_fini(adev);
2955 amdgpu_pm_sysfs_fini(adev);
2956 amdgpu_fbdev_fini(adev);
2957 r = amdgpu_device_ip_fini(adev);
2958 if (adev->firmware.gpu_info_fw) {
2959 release_firmware(adev->firmware.gpu_info_fw);
2960 adev->firmware.gpu_info_fw = NULL;
2961 }
2962 adev->accel_working = false;
2963 cancel_delayed_work_sync(&adev->delayed_init_work);
2964 /* free i2c buses */
2965 if (!amdgpu_device_has_dc_support(adev))
2966 amdgpu_i2c_fini(adev);
2967
2968 if (amdgpu_emu_mode != 1)
2969 amdgpu_atombios_fini(adev);
2970
2971 kfree(adev->bios);
2972 adev->bios = NULL;
2973 if (!pci_is_thunderbolt_attached(adev->pdev))
2974 vga_switcheroo_unregister_client(adev->pdev);
2975 if (adev->flags & AMD_IS_PX)
2976 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2977 vga_client_register(adev->pdev, NULL, NULL, NULL);
2978 if (adev->rio_mem)
2979 pci_iounmap(adev->pdev, adev->rio_mem);
2980 adev->rio_mem = NULL;
2981 iounmap(adev->rmmio);
2982 adev->rmmio = NULL;
2983 amdgpu_device_doorbell_fini(adev);
2984 if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
2985 amdgpu_pm_virt_sysfs_fini(adev);
2986
2987 amdgpu_debugfs_regs_cleanup(adev);
2988 device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
2989 amdgpu_ucode_sysfs_fini(adev);
2990 if (IS_ENABLED(CONFIG_PERF_EVENTS))
2991 amdgpu_pmu_fini(adev);
2992 amdgpu_debugfs_preempt_cleanup(adev);
2993 if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10)
2994 amdgpu_discovery_fini(adev);
2995}
2996
2997
2998/*
2999 * Suspend & resume.
3000 */
3001/**
3002 * amdgpu_device_suspend - initiate device suspend
3003 *
3004 * @dev: drm dev pointer
3005 * @suspend: suspend state
3006 * @fbcon : notify the fbdev of suspend
3007 *
3008 * Puts the hw in the suspend state (all asics).
3009 * Returns 0 for success or an error on failure.
3010 * Called at driver suspend.
3011 */
3012int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
3013{
3014 struct amdgpu_device *adev;
3015 struct drm_crtc *crtc;
3016 struct drm_connector *connector;
3017 int r;
3018
3019 if (dev == NULL || dev->dev_private == NULL) {
3020 return -ENODEV;
3021 }
3022
3023 adev = dev->dev_private;
3024
3025 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3026 return 0;
3027
3028 adev->in_suspend = true;
3029 drm_kms_helper_poll_disable(dev);
3030
3031 if (fbcon)
3032 amdgpu_fbdev_set_suspend(adev, 1);
3033
3034 cancel_delayed_work_sync(&adev->delayed_init_work);
3035
3036 if (!amdgpu_device_has_dc_support(adev)) {
3037 /* turn off display hw */
3038 drm_modeset_lock_all(dev);
3039 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
3040 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
3041 }
3042 drm_modeset_unlock_all(dev);
3043 /* unpin the front buffers and cursors */
3044 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3045 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3046 struct drm_framebuffer *fb = crtc->primary->fb;
3047 struct amdgpu_bo *robj;
3048
3049 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3050 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3051 r = amdgpu_bo_reserve(aobj, true);
3052 if (r == 0) {
3053 amdgpu_bo_unpin(aobj);
3054 amdgpu_bo_unreserve(aobj);
3055 }
3056 }
3057
3058 if (fb == NULL || fb->obj[0] == NULL) {
3059 continue;
3060 }
3061 robj = gem_to_amdgpu_bo(fb->obj[0]);
3062 /* don't unpin kernel fb objects */
3063 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
3064 r = amdgpu_bo_reserve(robj, true);
3065 if (r == 0) {
3066 amdgpu_bo_unpin(robj);
3067 amdgpu_bo_unreserve(robj);
3068 }
3069 }
3070 }
3071 }
3072
3073 amdgpu_amdkfd_suspend(adev);
3074
3075 amdgpu_ras_suspend(adev);
3076
3077 r = amdgpu_device_ip_suspend_phase1(adev);
3078
3079 /* evict vram memory */
3080 amdgpu_bo_evict_vram(adev);
3081
3082 amdgpu_fence_driver_suspend(adev);
3083
3084 r = amdgpu_device_ip_suspend_phase2(adev);
3085
3086 /* evict remaining vram memory
3087 * This second call to evict vram is to evict the gart page table
3088 * using the CPU.
3089 */
3090 amdgpu_bo_evict_vram(adev);
3091
3092 pci_save_state(dev->pdev);
3093 if (suspend) {
3094 /* Shut down the device */
3095 pci_disable_device(dev->pdev);
3096 pci_set_power_state(dev->pdev, PCI_D3hot);
3097 } else {
3098 r = amdgpu_asic_reset(adev);
3099 if (r)
3100 DRM_ERROR("amdgpu asic reset failed\n");
3101 }
3102
3103 return 0;
3104}
3105
3106/**
3107 * amdgpu_device_resume - initiate device resume
3108 *
3109 * @dev: drm dev pointer
3110 * @resume: resume state
3111 * @fbcon : notify the fbdev of resume
3112 *
3113 * Bring the hw back to operating state (all asics).
3114 * Returns 0 for success or an error on failure.
3115 * Called at driver resume.
3116 */
3117int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
3118{
3119 struct drm_connector *connector;
3120 struct amdgpu_device *adev = dev->dev_private;
3121 struct drm_crtc *crtc;
3122 int r = 0;
3123
3124 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3125 return 0;
3126
3127 if (resume) {
3128 pci_set_power_state(dev->pdev, PCI_D0);
3129 pci_restore_state(dev->pdev);
3130 r = pci_enable_device(dev->pdev);
3131 if (r)
3132 return r;
3133 }
3134
3135 /* post card */
3136 if (amdgpu_device_need_post(adev)) {
3137 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
3138 if (r)
3139 DRM_ERROR("amdgpu asic init failed\n");
3140 }
3141
3142 r = amdgpu_device_ip_resume(adev);
3143 if (r) {
3144 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
3145 return r;
3146 }
3147 amdgpu_fence_driver_resume(adev);
3148
3149
3150 r = amdgpu_device_ip_late_init(adev);
3151 if (r)
3152 return r;
3153
3154 queue_delayed_work(system_wq, &adev->delayed_init_work,
3155 msecs_to_jiffies(AMDGPU_RESUME_MS));
3156
3157 if (!amdgpu_device_has_dc_support(adev)) {
3158 /* pin cursors */
3159 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3160 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3161
3162 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3163 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3164 r = amdgpu_bo_reserve(aobj, true);
3165 if (r == 0) {
3166 r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
3167 if (r != 0)
3168 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
3169 amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
3170 amdgpu_bo_unreserve(aobj);
3171 }
3172 }
3173 }
3174 }
3175 r = amdgpu_amdkfd_resume(adev);
3176 if (r)
3177 return r;
3178
3179 /* Make sure IB tests flushed */
3180 flush_delayed_work(&adev->delayed_init_work);
3181
3182 /* blat the mode back in */
3183 if (fbcon) {
3184 if (!amdgpu_device_has_dc_support(adev)) {
3185 /* pre DCE11 */
3186 drm_helper_resume_force_mode(dev);
3187
3188 /* turn on display hw */
3189 drm_modeset_lock_all(dev);
3190 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
3191 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
3192 }
3193 drm_modeset_unlock_all(dev);
3194 }
3195 amdgpu_fbdev_set_suspend(adev, 0);
3196 }
3197
3198 drm_kms_helper_poll_enable(dev);
3199
3200 amdgpu_ras_resume(adev);
3201
3202 /*
3203 * Most of the connector probing functions try to acquire runtime pm
3204 * refs to ensure that the GPU is powered on when connector polling is
3205 * performed. Since we're calling this from a runtime PM callback,
3206 * trying to acquire rpm refs will cause us to deadlock.
3207 *
3208 * Since we're guaranteed to be holding the rpm lock, it's safe to
3209 * temporarily disable the rpm helpers so this doesn't deadlock us.
3210 */
3211#ifdef CONFIG_PM
3212 dev->dev->power.disable_depth++;
3213#endif
3214 if (!amdgpu_device_has_dc_support(adev))
3215 drm_helper_hpd_irq_event(dev);
3216 else
3217 drm_kms_helper_hotplug_event(dev);
3218#ifdef CONFIG_PM
3219 dev->dev->power.disable_depth--;
3220#endif
3221 adev->in_suspend = false;
3222
3223 return 0;
3224}
3225
3226/**
3227 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3228 *
3229 * @adev: amdgpu_device pointer
3230 *
3231 * The list of all the hardware IPs that make up the asic is walked and
3232 * the check_soft_reset callbacks are run. check_soft_reset determines
3233 * if the asic is still hung or not.
3234 * Returns true if any of the IPs are still in a hung state, false if not.
3235 */
3236static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
3237{
3238 int i;
3239 bool asic_hang = false;
3240
3241 if (amdgpu_sriov_vf(adev))
3242 return true;
3243
3244 if (amdgpu_asic_need_full_reset(adev))
3245 return true;
3246
3247 for (i = 0; i < adev->num_ip_blocks; i++) {
3248 if (!adev->ip_blocks[i].status.valid)
3249 continue;
3250 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
3251 adev->ip_blocks[i].status.hang =
3252 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
3253 if (adev->ip_blocks[i].status.hang) {
3254 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
3255 asic_hang = true;
3256 }
3257 }
3258 return asic_hang;
3259}
3260
3261/**
3262 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
3263 *
3264 * @adev: amdgpu_device pointer
3265 *
3266 * The list of all the hardware IPs that make up the asic is walked and the
3267 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
3268 * handles any IP specific hardware or software state changes that are
3269 * necessary for a soft reset to succeed.
3270 * Returns 0 on success, negative error code on failure.
3271 */
3272static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3273{
3274 int i, r = 0;
3275
3276 for (i = 0; i < adev->num_ip_blocks; i++) {
3277 if (!adev->ip_blocks[i].status.valid)
3278 continue;
3279 if (adev->ip_blocks[i].status.hang &&
3280 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
3281 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3282 if (r)
3283 return r;
3284 }
3285 }
3286
3287 return 0;
3288}
3289
3290/**
3291 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3292 *
3293 * @adev: amdgpu_device pointer
3294 *
3295 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
3296 * reset is necessary to recover.
3297 * Returns true if a full asic reset is required, false if not.
3298 */
3299static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
3300{
3301 int i;
3302
3303 if (amdgpu_asic_need_full_reset(adev))
3304 return true;
3305
3306 for (i = 0; i < adev->num_ip_blocks; i++) {
3307 if (!adev->ip_blocks[i].status.valid)
3308 continue;
3309 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
3310 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
3311 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
3312 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
3313 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3314 if (adev->ip_blocks[i].status.hang) {
3315 DRM_INFO("Some block need full reset!\n");
3316 return true;
3317 }
3318 }
3319 }
3320 return false;
3321}
3322
3323/**
3324 * amdgpu_device_ip_soft_reset - do a soft reset
3325 *
3326 * @adev: amdgpu_device pointer
3327 *
3328 * The list of all the hardware IPs that make up the asic is walked and the
3329 * soft_reset callbacks are run if the block is hung. soft_reset handles any
3330 * IP specific hardware or software state changes that are necessary to soft
3331 * reset the IP.
3332 * Returns 0 on success, negative error code on failure.
3333 */
3334static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
3335{
3336 int i, r = 0;
3337
3338 for (i = 0; i < adev->num_ip_blocks; i++) {
3339 if (!adev->ip_blocks[i].status.valid)
3340 continue;
3341 if (adev->ip_blocks[i].status.hang &&
3342 adev->ip_blocks[i].version->funcs->soft_reset) {
3343 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
3344 if (r)
3345 return r;
3346 }
3347 }
3348
3349 return 0;
3350}
3351
3352/**
3353 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
3354 *
3355 * @adev: amdgpu_device pointer
3356 *
3357 * The list of all the hardware IPs that make up the asic is walked and the
3358 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
3359 * handles any IP specific hardware or software state changes that are
3360 * necessary after the IP has been soft reset.
3361 * Returns 0 on success, negative error code on failure.
3362 */
3363static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
3364{
3365 int i, r = 0;
3366
3367 for (i = 0; i < adev->num_ip_blocks; i++) {
3368 if (!adev->ip_blocks[i].status.valid)
3369 continue;
3370 if (adev->ip_blocks[i].status.hang &&
3371 adev->ip_blocks[i].version->funcs->post_soft_reset)
3372 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
3373 if (r)
3374 return r;
3375 }
3376
3377 return 0;
3378}
3379
3380/**
3381 * amdgpu_device_recover_vram - Recover some VRAM contents
3382 *
3383 * @adev: amdgpu_device pointer
3384 *
3385 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
3386 * restore things like GPUVM page tables after a GPU reset where
3387 * the contents of VRAM might be lost.
3388 *
3389 * Returns:
3390 * 0 on success, negative error code on failure.
3391 */
3392static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
3393{
3394 struct dma_fence *fence = NULL, *next = NULL;
3395 struct amdgpu_bo *shadow;
3396 long r = 1, tmo;
3397
3398 if (amdgpu_sriov_runtime(adev))
3399 tmo = msecs_to_jiffies(8000);
3400 else
3401 tmo = msecs_to_jiffies(100);
3402
3403 DRM_INFO("recover vram bo from shadow start\n");
3404 mutex_lock(&adev->shadow_list_lock);
3405 list_for_each_entry(shadow, &adev->shadow_list, shadow_list) {
3406
3407 /* No need to recover an evicted BO */
3408 if (shadow->tbo.mem.mem_type != TTM_PL_TT ||
3409 shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET ||
3410 shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM)
3411 continue;
3412
3413 r = amdgpu_bo_restore_shadow(shadow, &next);
3414 if (r)
3415 break;
3416
3417 if (fence) {
3418 tmo = dma_fence_wait_timeout(fence, false, tmo);
3419 dma_fence_put(fence);
3420 fence = next;
3421 if (tmo == 0) {
3422 r = -ETIMEDOUT;
3423 break;
3424 } else if (tmo < 0) {
3425 r = tmo;
3426 break;
3427 }
3428 } else {
3429 fence = next;
3430 }
3431 }
3432 mutex_unlock(&adev->shadow_list_lock);
3433
3434 if (fence)
3435 tmo = dma_fence_wait_timeout(fence, false, tmo);
3436 dma_fence_put(fence);
3437
3438 if (r < 0 || tmo <= 0) {
3439 DRM_ERROR("recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
3440 return -EIO;
3441 }
3442
3443 DRM_INFO("recover vram bo from shadow done\n");
3444 return 0;
3445}
3446
3447
3448/**
3449 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
3450 *
3451 * @adev: amdgpu device pointer
3452 * @from_hypervisor: request from hypervisor
3453 *
3454 * do VF FLR and reinitialize Asic
3455 * return 0 means succeeded otherwise failed
3456 */
3457static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
3458 bool from_hypervisor)
3459{
3460 int r;
3461
3462 if (from_hypervisor)
3463 r = amdgpu_virt_request_full_gpu(adev, true);
3464 else
3465 r = amdgpu_virt_reset_gpu(adev);
3466 if (r)
3467 return r;
3468
3469 amdgpu_amdkfd_pre_reset(adev);
3470
3471 /* Resume IP prior to SMC */
3472 r = amdgpu_device_ip_reinit_early_sriov(adev);
3473 if (r)
3474 goto error;
3475
3476 /* we need recover gart prior to run SMC/CP/SDMA resume */
3477 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
3478
3479 r = amdgpu_device_fw_loading(adev);
3480 if (r)
3481 return r;
3482
3483 /* now we are okay to resume SMC/CP/SDMA */
3484 r = amdgpu_device_ip_reinit_late_sriov(adev);
3485 if (r)
3486 goto error;
3487
3488 amdgpu_irq_gpu_reset_resume_helper(adev);
3489 r = amdgpu_ib_ring_tests(adev);
3490 amdgpu_amdkfd_post_reset(adev);
3491
3492error:
3493 amdgpu_virt_init_data_exchange(adev);
3494 amdgpu_virt_release_full_gpu(adev, true);
3495 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3496 amdgpu_inc_vram_lost(adev);
3497 r = amdgpu_device_recover_vram(adev);
3498 }
3499
3500 return r;
3501}
3502
3503/**
3504 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
3505 *
3506 * @adev: amdgpu device pointer
3507 *
3508 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
3509 * a hung GPU.
3510 */
3511bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
3512{
3513 if (!amdgpu_device_ip_check_soft_reset(adev)) {
3514 DRM_INFO("Timeout, but no hardware hang detected.\n");
3515 return false;
3516 }
3517
3518 if (amdgpu_gpu_recovery == 0)
3519 goto disabled;
3520
3521 if (amdgpu_sriov_vf(adev))
3522 return true;
3523
3524 if (amdgpu_gpu_recovery == -1) {
3525 switch (adev->asic_type) {
3526 case CHIP_BONAIRE:
3527 case CHIP_HAWAII:
3528 case CHIP_TOPAZ:
3529 case CHIP_TONGA:
3530 case CHIP_FIJI:
3531 case CHIP_POLARIS10:
3532 case CHIP_POLARIS11:
3533 case CHIP_POLARIS12:
3534 case CHIP_VEGAM:
3535 case CHIP_VEGA20:
3536 case CHIP_VEGA10:
3537 case CHIP_VEGA12:
3538 case CHIP_RAVEN:
3539 break;
3540 default:
3541 goto disabled;
3542 }
3543 }
3544
3545 return true;
3546
3547disabled:
3548 DRM_INFO("GPU recovery disabled.\n");
3549 return false;
3550}
3551
3552
3553static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
3554 struct amdgpu_job *job,
3555 bool *need_full_reset_arg)
3556{
3557 int i, r = 0;
3558 bool need_full_reset = *need_full_reset_arg;
3559
3560 /* block all schedulers and reset given job's ring */
3561 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3562 struct amdgpu_ring *ring = adev->rings[i];
3563
3564 if (!ring || !ring->sched.thread)
3565 continue;
3566
3567 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3568 amdgpu_fence_driver_force_completion(ring);
3569 }
3570
3571 if(job)
3572 drm_sched_increase_karma(&job->base);
3573
3574 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
3575 if (!amdgpu_sriov_vf(adev)) {
3576
3577 if (!need_full_reset)
3578 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
3579
3580 if (!need_full_reset) {
3581 amdgpu_device_ip_pre_soft_reset(adev);
3582 r = amdgpu_device_ip_soft_reset(adev);
3583 amdgpu_device_ip_post_soft_reset(adev);
3584 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
3585 DRM_INFO("soft reset failed, will fallback to full reset!\n");
3586 need_full_reset = true;
3587 }
3588 }
3589
3590 if (need_full_reset)
3591 r = amdgpu_device_ip_suspend(adev);
3592
3593 *need_full_reset_arg = need_full_reset;
3594 }
3595
3596 return r;
3597}
3598
3599static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
3600 struct list_head *device_list_handle,
3601 bool *need_full_reset_arg)
3602{
3603 struct amdgpu_device *tmp_adev = NULL;
3604 bool need_full_reset = *need_full_reset_arg, vram_lost = false;
3605 int r = 0;
3606
3607 /*
3608 * ASIC reset has to be done on all HGMI hive nodes ASAP
3609 * to allow proper links negotiation in FW (within 1 sec)
3610 */
3611 if (need_full_reset) {
3612 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3613 /* For XGMI run all resets in parallel to speed up the process */
3614 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
3615 if (!queue_work(system_highpri_wq, &tmp_adev->xgmi_reset_work))
3616 r = -EALREADY;
3617 } else
3618 r = amdgpu_asic_reset(tmp_adev);
3619
3620 if (r) {
3621 DRM_ERROR("ASIC reset failed with error, %d for drm dev, %s",
3622 r, tmp_adev->ddev->unique);
3623 break;
3624 }
3625 }
3626
3627 /* For XGMI wait for all PSP resets to complete before proceed */
3628 if (!r) {
3629 list_for_each_entry(tmp_adev, device_list_handle,
3630 gmc.xgmi.head) {
3631 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
3632 flush_work(&tmp_adev->xgmi_reset_work);
3633 r = tmp_adev->asic_reset_res;
3634 if (r)
3635 break;
3636 }
3637 }
3638
3639 list_for_each_entry(tmp_adev, device_list_handle,
3640 gmc.xgmi.head) {
3641 amdgpu_ras_reserve_bad_pages(tmp_adev);
3642 }
3643 }
3644 }
3645
3646
3647 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3648 if (need_full_reset) {
3649 /* post card */
3650 if (amdgpu_atom_asic_init(tmp_adev->mode_info.atom_context))
3651 DRM_WARN("asic atom init failed!");
3652
3653 if (!r) {
3654 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
3655 r = amdgpu_device_ip_resume_phase1(tmp_adev);
3656 if (r)
3657 goto out;
3658
3659 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
3660 if (vram_lost) {
3661 DRM_INFO("VRAM is lost due to GPU reset!\n");
3662 amdgpu_inc_vram_lost(tmp_adev);
3663 }
3664
3665 r = amdgpu_gtt_mgr_recover(
3666 &tmp_adev->mman.bdev.man[TTM_PL_TT]);
3667 if (r)
3668 goto out;
3669
3670 r = amdgpu_device_fw_loading(tmp_adev);
3671 if (r)
3672 return r;
3673
3674 r = amdgpu_device_ip_resume_phase2(tmp_adev);
3675 if (r)
3676 goto out;
3677
3678 if (vram_lost)
3679 amdgpu_device_fill_reset_magic(tmp_adev);
3680
3681 /*
3682 * Add this ASIC as tracked as reset was already
3683 * complete successfully.
3684 */
3685 amdgpu_register_gpu_instance(tmp_adev);
3686
3687 r = amdgpu_device_ip_late_init(tmp_adev);
3688 if (r)
3689 goto out;
3690
3691 /* must succeed. */
3692 amdgpu_ras_resume(tmp_adev);
3693
3694 /* Update PSP FW topology after reset */
3695 if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
3696 r = amdgpu_xgmi_update_topology(hive, tmp_adev);
3697 }
3698 }
3699
3700
3701out:
3702 if (!r) {
3703 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
3704 r = amdgpu_ib_ring_tests(tmp_adev);
3705 if (r) {
3706 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
3707 r = amdgpu_device_ip_suspend(tmp_adev);
3708 need_full_reset = true;
3709 r = -EAGAIN;
3710 goto end;
3711 }
3712 }
3713
3714 if (!r)
3715 r = amdgpu_device_recover_vram(tmp_adev);
3716 else
3717 tmp_adev->asic_reset_res = r;
3718 }
3719
3720end:
3721 *need_full_reset_arg = need_full_reset;
3722 return r;
3723}
3724
3725static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock)
3726{
3727 if (trylock) {
3728 if (!mutex_trylock(&adev->lock_reset))
3729 return false;
3730 } else
3731 mutex_lock(&adev->lock_reset);
3732
3733 atomic_inc(&adev->gpu_reset_counter);
3734 adev->in_gpu_reset = 1;
3735 switch (amdgpu_asic_reset_method(adev)) {
3736 case AMD_RESET_METHOD_MODE1:
3737 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
3738 break;
3739 case AMD_RESET_METHOD_MODE2:
3740 adev->mp1_state = PP_MP1_STATE_RESET;
3741 break;
3742 default:
3743 adev->mp1_state = PP_MP1_STATE_NONE;
3744 break;
3745 }
3746 /* Block kfd: SRIOV would do it separately */
3747 if (!amdgpu_sriov_vf(adev))
3748 amdgpu_amdkfd_pre_reset(adev);
3749
3750 return true;
3751}
3752
3753static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
3754{
3755 /*unlock kfd: SRIOV would do it separately */
3756 if (!amdgpu_sriov_vf(adev))
3757 amdgpu_amdkfd_post_reset(adev);
3758 amdgpu_vf_error_trans_all(adev);
3759 adev->mp1_state = PP_MP1_STATE_NONE;
3760 adev->in_gpu_reset = 0;
3761 mutex_unlock(&adev->lock_reset);
3762}
3763
3764
3765/**
3766 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
3767 *
3768 * @adev: amdgpu device pointer
3769 * @job: which job trigger hang
3770 *
3771 * Attempt to reset the GPU if it has hung (all asics).
3772 * Attempt to do soft-reset or full-reset and reinitialize Asic
3773 * Returns 0 for success or an error on failure.
3774 */
3775
3776int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
3777 struct amdgpu_job *job)
3778{
3779 struct list_head device_list, *device_list_handle = NULL;
3780 bool need_full_reset, job_signaled;
3781 struct amdgpu_hive_info *hive = NULL;
3782 struct amdgpu_device *tmp_adev = NULL;
3783 int i, r = 0;
3784
3785 need_full_reset = job_signaled = false;
3786 INIT_LIST_HEAD(&device_list);
3787
3788 dev_info(adev->dev, "GPU reset begin!\n");
3789
3790 cancel_delayed_work_sync(&adev->delayed_init_work);
3791
3792 hive = amdgpu_get_xgmi_hive(adev, false);
3793
3794 /*
3795 * Here we trylock to avoid chain of resets executing from
3796 * either trigger by jobs on different adevs in XGMI hive or jobs on
3797 * different schedulers for same device while this TO handler is running.
3798 * We always reset all schedulers for device and all devices for XGMI
3799 * hive so that should take care of them too.
3800 */
3801
3802 if (hive && !mutex_trylock(&hive->reset_lock)) {
3803 DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
3804 job ? job->base.id : -1, hive->hive_id);
3805 return 0;
3806 }
3807
3808 /* Start with adev pre asic reset first for soft reset check.*/
3809 if (!amdgpu_device_lock_adev(adev, !hive)) {
3810 DRM_INFO("Bailing on TDR for s_job:%llx, as another already in progress",
3811 job ? job->base.id : -1);
3812 return 0;
3813 }
3814
3815 /* Build list of devices to reset */
3816 if (adev->gmc.xgmi.num_physical_nodes > 1) {
3817 if (!hive) {
3818 amdgpu_device_unlock_adev(adev);
3819 return -ENODEV;
3820 }
3821
3822 /*
3823 * In case we are in XGMI hive mode device reset is done for all the
3824 * nodes in the hive to retrain all XGMI links and hence the reset
3825 * sequence is executed in loop on all nodes.
3826 */
3827 device_list_handle = &hive->device_list;
3828 } else {
3829 list_add_tail(&adev->gmc.xgmi.head, &device_list);
3830 device_list_handle = &device_list;
3831 }
3832
3833 /*
3834 * Mark these ASICs to be reseted as untracked first
3835 * And add them back after reset completed
3836 */
3837 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)
3838 amdgpu_unregister_gpu_instance(tmp_adev);
3839
3840 /* block all schedulers and reset given job's ring */
3841 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3842 /* disable ras on ALL IPs */
3843 if (amdgpu_device_ip_need_full_reset(tmp_adev))
3844 amdgpu_ras_suspend(tmp_adev);
3845
3846 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3847 struct amdgpu_ring *ring = tmp_adev->rings[i];
3848
3849 if (!ring || !ring->sched.thread)
3850 continue;
3851
3852 drm_sched_stop(&ring->sched, job ? &job->base : NULL);
3853 }
3854 }
3855
3856
3857 /*
3858 * Must check guilty signal here since after this point all old
3859 * HW fences are force signaled.
3860 *
3861 * job->base holds a reference to parent fence
3862 */
3863 if (job && job->base.s_fence->parent &&
3864 dma_fence_is_signaled(job->base.s_fence->parent))
3865 job_signaled = true;
3866
3867 if (!amdgpu_device_ip_need_full_reset(adev))
3868 device_list_handle = &device_list;
3869
3870 if (job_signaled) {
3871 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
3872 goto skip_hw_reset;
3873 }
3874
3875
3876 /* Guilty job will be freed after this*/
3877 r = amdgpu_device_pre_asic_reset(adev, job, &need_full_reset);
3878 if (r) {
3879 /*TODO Should we stop ?*/
3880 DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
3881 r, adev->ddev->unique);
3882 adev->asic_reset_res = r;
3883 }
3884
3885retry: /* Rest of adevs pre asic reset from XGMI hive. */
3886 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3887
3888 if (tmp_adev == adev)
3889 continue;
3890
3891 amdgpu_device_lock_adev(tmp_adev, false);
3892 r = amdgpu_device_pre_asic_reset(tmp_adev,
3893 NULL,
3894 &need_full_reset);
3895 /*TODO Should we stop ?*/
3896 if (r) {
3897 DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
3898 r, tmp_adev->ddev->unique);
3899 tmp_adev->asic_reset_res = r;
3900 }
3901 }
3902
3903 /* Actual ASIC resets if needed.*/
3904 /* TODO Implement XGMI hive reset logic for SRIOV */
3905 if (amdgpu_sriov_vf(adev)) {
3906 r = amdgpu_device_reset_sriov(adev, job ? false : true);
3907 if (r)
3908 adev->asic_reset_res = r;
3909 } else {
3910 r = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset);
3911 if (r && r == -EAGAIN)
3912 goto retry;
3913 }
3914
3915skip_hw_reset:
3916
3917 /* Post ASIC reset for all devs .*/
3918 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3919 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3920 struct amdgpu_ring *ring = tmp_adev->rings[i];
3921
3922 if (!ring || !ring->sched.thread)
3923 continue;
3924
3925 /* No point to resubmit jobs if we didn't HW reset*/
3926 if (!tmp_adev->asic_reset_res && !job_signaled)
3927 drm_sched_resubmit_jobs(&ring->sched);
3928
3929 drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
3930 }
3931
3932 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
3933 drm_helper_resume_force_mode(tmp_adev->ddev);
3934 }
3935
3936 tmp_adev->asic_reset_res = 0;
3937
3938 if (r) {
3939 /* bad news, how to tell it to userspace ? */
3940 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
3941 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
3942 } else {
3943 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&adev->gpu_reset_counter));
3944 }
3945
3946 amdgpu_device_unlock_adev(tmp_adev);
3947 }
3948
3949 if (hive)
3950 mutex_unlock(&hive->reset_lock);
3951
3952 if (r)
3953 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
3954 return r;
3955}
3956
3957/**
3958 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
3959 *
3960 * @adev: amdgpu_device pointer
3961 *
3962 * Fetchs and stores in the driver the PCIE capabilities (gen speed
3963 * and lanes) of the slot the device is in. Handles APUs and
3964 * virtualized environments where PCIE config space may not be available.
3965 */
3966static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
3967{
3968 struct pci_dev *pdev;
3969 enum pci_bus_speed speed_cap, platform_speed_cap;
3970 enum pcie_link_width platform_link_width;
3971
3972 if (amdgpu_pcie_gen_cap)
3973 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3974
3975 if (amdgpu_pcie_lane_cap)
3976 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3977
3978 /* covers APUs as well */
3979 if (pci_is_root_bus(adev->pdev->bus)) {
3980 if (adev->pm.pcie_gen_mask == 0)
3981 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3982 if (adev->pm.pcie_mlw_mask == 0)
3983 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3984 return;
3985 }
3986
3987 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
3988 return;
3989
3990 pcie_bandwidth_available(adev->pdev, NULL,
3991 &platform_speed_cap, &platform_link_width);
3992
3993 if (adev->pm.pcie_gen_mask == 0) {
3994 /* asic caps */
3995 pdev = adev->pdev;
3996 speed_cap = pcie_get_speed_cap(pdev);
3997 if (speed_cap == PCI_SPEED_UNKNOWN) {
3998 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3999 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4000 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
4001 } else {
4002 if (speed_cap == PCIE_SPEED_16_0GT)
4003 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4004 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4005 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4006 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
4007 else if (speed_cap == PCIE_SPEED_8_0GT)
4008 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4009 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4010 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
4011 else if (speed_cap == PCIE_SPEED_5_0GT)
4012 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4013 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
4014 else
4015 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
4016 }
4017 /* platform caps */
4018 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
4019 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4020 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
4021 } else {
4022 if (platform_speed_cap == PCIE_SPEED_16_0GT)
4023 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4024 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4025 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4026 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
4027 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
4028 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4029 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4030 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
4031 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
4032 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4033 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
4034 else
4035 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
4036
4037 }
4038 }
4039 if (adev->pm.pcie_mlw_mask == 0) {
4040 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
4041 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
4042 } else {
4043 switch (platform_link_width) {
4044 case PCIE_LNK_X32:
4045 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
4046 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
4047 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4048 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4049 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4050 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4051 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4052 break;
4053 case PCIE_LNK_X16:
4054 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
4055 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4056 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4057 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4058 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4059 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4060 break;
4061 case PCIE_LNK_X12:
4062 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4063 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4064 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4065 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4066 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4067 break;
4068 case PCIE_LNK_X8:
4069 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4070 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4071 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4072 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4073 break;
4074 case PCIE_LNK_X4:
4075 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4076 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4077 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4078 break;
4079 case PCIE_LNK_X2:
4080 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4081 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4082 break;
4083 case PCIE_LNK_X1:
4084 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
4085 break;
4086 default:
4087 break;
4088 }
4089 }
4090 }
4091}
4092