Loading...
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/power_supply.h>
29#include <linux/kthread.h>
30#include <linux/module.h>
31#include <linux/console.h>
32#include <linux/slab.h>
33#include <linux/iommu.h>
34#include <linux/pci.h>
35#include <linux/pci-p2pdma.h>
36#include <linux/apple-gmux.h>
37
38#include <drm/drm_aperture.h>
39#include <drm/drm_atomic_helper.h>
40#include <drm/drm_crtc_helper.h>
41#include <drm/drm_fb_helper.h>
42#include <drm/drm_probe_helper.h>
43#include <drm/amdgpu_drm.h>
44#include <linux/device.h>
45#include <linux/vgaarb.h>
46#include <linux/vga_switcheroo.h>
47#include <linux/efi.h>
48#include "amdgpu.h"
49#include "amdgpu_trace.h"
50#include "amdgpu_i2c.h"
51#include "atom.h"
52#include "amdgpu_atombios.h"
53#include "amdgpu_atomfirmware.h"
54#include "amd_pcie.h"
55#ifdef CONFIG_DRM_AMDGPU_SI
56#include "si.h"
57#endif
58#ifdef CONFIG_DRM_AMDGPU_CIK
59#include "cik.h"
60#endif
61#include "vi.h"
62#include "soc15.h"
63#include "nv.h"
64#include "bif/bif_4_1_d.h"
65#include <linux/firmware.h>
66#include "amdgpu_vf_error.h"
67
68#include "amdgpu_amdkfd.h"
69#include "amdgpu_pm.h"
70
71#include "amdgpu_xgmi.h"
72#include "amdgpu_ras.h"
73#include "amdgpu_pmu.h"
74#include "amdgpu_fru_eeprom.h"
75#include "amdgpu_reset.h"
76#include "amdgpu_virt.h"
77
78#include <linux/suspend.h>
79#include <drm/task_barrier.h>
80#include <linux/pm_runtime.h>
81
82#include <drm/drm_drv.h>
83
84#if IS_ENABLED(CONFIG_X86)
85#include <asm/intel-family.h>
86#endif
87
88MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
89MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
90MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
91MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
92MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
93MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
94MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
95
96#define AMDGPU_RESUME_MS 2000
97#define AMDGPU_MAX_RETRY_LIMIT 2
98#define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
99#define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
100#define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
101#define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
102
103static const struct drm_driver amdgpu_kms_driver;
104
105const char *amdgpu_asic_name[] = {
106 "TAHITI",
107 "PITCAIRN",
108 "VERDE",
109 "OLAND",
110 "HAINAN",
111 "BONAIRE",
112 "KAVERI",
113 "KABINI",
114 "HAWAII",
115 "MULLINS",
116 "TOPAZ",
117 "TONGA",
118 "FIJI",
119 "CARRIZO",
120 "STONEY",
121 "POLARIS10",
122 "POLARIS11",
123 "POLARIS12",
124 "VEGAM",
125 "VEGA10",
126 "VEGA12",
127 "VEGA20",
128 "RAVEN",
129 "ARCTURUS",
130 "RENOIR",
131 "ALDEBARAN",
132 "NAVI10",
133 "CYAN_SKILLFISH",
134 "NAVI14",
135 "NAVI12",
136 "SIENNA_CICHLID",
137 "NAVY_FLOUNDER",
138 "VANGOGH",
139 "DIMGREY_CAVEFISH",
140 "BEIGE_GOBY",
141 "YELLOW_CARP",
142 "IP DISCOVERY",
143 "LAST",
144};
145
146/**
147 * DOC: pcie_replay_count
148 *
149 * The amdgpu driver provides a sysfs API for reporting the total number
150 * of PCIe replays (NAKs)
151 * The file pcie_replay_count is used for this and returns the total
152 * number of replays as a sum of the NAKs generated and NAKs received
153 */
154
155static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
156 struct device_attribute *attr, char *buf)
157{
158 struct drm_device *ddev = dev_get_drvdata(dev);
159 struct amdgpu_device *adev = drm_to_adev(ddev);
160 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
161
162 return sysfs_emit(buf, "%llu\n", cnt);
163}
164
165static DEVICE_ATTR(pcie_replay_count, 0444,
166 amdgpu_device_get_pcie_replay_count, NULL);
167
168static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
169 struct bin_attribute *attr, char *buf,
170 loff_t ppos, size_t count)
171{
172 struct device *dev = kobj_to_dev(kobj);
173 struct drm_device *ddev = dev_get_drvdata(dev);
174 struct amdgpu_device *adev = drm_to_adev(ddev);
175 ssize_t bytes_read;
176
177 switch (ppos) {
178 case AMDGPU_SYS_REG_STATE_XGMI:
179 bytes_read = amdgpu_asic_get_reg_state(
180 adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
181 break;
182 case AMDGPU_SYS_REG_STATE_WAFL:
183 bytes_read = amdgpu_asic_get_reg_state(
184 adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
185 break;
186 case AMDGPU_SYS_REG_STATE_PCIE:
187 bytes_read = amdgpu_asic_get_reg_state(
188 adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
189 break;
190 case AMDGPU_SYS_REG_STATE_USR:
191 bytes_read = amdgpu_asic_get_reg_state(
192 adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
193 break;
194 case AMDGPU_SYS_REG_STATE_USR_1:
195 bytes_read = amdgpu_asic_get_reg_state(
196 adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
197 break;
198 default:
199 return -EINVAL;
200 }
201
202 return bytes_read;
203}
204
205BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
206 AMDGPU_SYS_REG_STATE_END);
207
208int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
209{
210 int ret;
211
212 if (!amdgpu_asic_get_reg_state_supported(adev))
213 return 0;
214
215 ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
216
217 return ret;
218}
219
220void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
221{
222 if (!amdgpu_asic_get_reg_state_supported(adev))
223 return;
224 sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
225}
226
227/**
228 * DOC: board_info
229 *
230 * The amdgpu driver provides a sysfs API for giving board related information.
231 * It provides the form factor information in the format
232 *
233 * type : form factor
234 *
235 * Possible form factor values
236 *
237 * - "cem" - PCIE CEM card
238 * - "oam" - Open Compute Accelerator Module
239 * - "unknown" - Not known
240 *
241 */
242
243static ssize_t amdgpu_device_get_board_info(struct device *dev,
244 struct device_attribute *attr,
245 char *buf)
246{
247 struct drm_device *ddev = dev_get_drvdata(dev);
248 struct amdgpu_device *adev = drm_to_adev(ddev);
249 enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
250 const char *pkg;
251
252 if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
253 pkg_type = adev->smuio.funcs->get_pkg_type(adev);
254
255 switch (pkg_type) {
256 case AMDGPU_PKG_TYPE_CEM:
257 pkg = "cem";
258 break;
259 case AMDGPU_PKG_TYPE_OAM:
260 pkg = "oam";
261 break;
262 default:
263 pkg = "unknown";
264 break;
265 }
266
267 return sysfs_emit(buf, "%s : %s\n", "type", pkg);
268}
269
270static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
271
272static struct attribute *amdgpu_board_attrs[] = {
273 &dev_attr_board_info.attr,
274 NULL,
275};
276
277static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
278 struct attribute *attr, int n)
279{
280 struct device *dev = kobj_to_dev(kobj);
281 struct drm_device *ddev = dev_get_drvdata(dev);
282 struct amdgpu_device *adev = drm_to_adev(ddev);
283
284 if (adev->flags & AMD_IS_APU)
285 return 0;
286
287 return attr->mode;
288}
289
290static const struct attribute_group amdgpu_board_attrs_group = {
291 .attrs = amdgpu_board_attrs,
292 .is_visible = amdgpu_board_attrs_is_visible
293};
294
295static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
296
297
298/**
299 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
300 *
301 * @dev: drm_device pointer
302 *
303 * Returns true if the device is a dGPU with ATPX power control,
304 * otherwise return false.
305 */
306bool amdgpu_device_supports_px(struct drm_device *dev)
307{
308 struct amdgpu_device *adev = drm_to_adev(dev);
309
310 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
311 return true;
312 return false;
313}
314
315/**
316 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
317 *
318 * @dev: drm_device pointer
319 *
320 * Returns true if the device is a dGPU with ACPI power control,
321 * otherwise return false.
322 */
323bool amdgpu_device_supports_boco(struct drm_device *dev)
324{
325 struct amdgpu_device *adev = drm_to_adev(dev);
326
327 if (adev->has_pr3 ||
328 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
329 return true;
330 return false;
331}
332
333/**
334 * amdgpu_device_supports_baco - Does the device support BACO
335 *
336 * @dev: drm_device pointer
337 *
338 * Returns true if the device supporte BACO,
339 * otherwise return false.
340 */
341bool amdgpu_device_supports_baco(struct drm_device *dev)
342{
343 struct amdgpu_device *adev = drm_to_adev(dev);
344
345 return amdgpu_asic_supports_baco(adev);
346}
347
348/**
349 * amdgpu_device_supports_smart_shift - Is the device dGPU with
350 * smart shift support
351 *
352 * @dev: drm_device pointer
353 *
354 * Returns true if the device is a dGPU with Smart Shift support,
355 * otherwise returns false.
356 */
357bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
358{
359 return (amdgpu_device_supports_boco(dev) &&
360 amdgpu_acpi_is_power_shift_control_supported());
361}
362
363/*
364 * VRAM access helper functions
365 */
366
367/**
368 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
369 *
370 * @adev: amdgpu_device pointer
371 * @pos: offset of the buffer in vram
372 * @buf: virtual address of the buffer in system memory
373 * @size: read/write size, sizeof(@buf) must > @size
374 * @write: true - write to vram, otherwise - read from vram
375 */
376void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
377 void *buf, size_t size, bool write)
378{
379 unsigned long flags;
380 uint32_t hi = ~0, tmp = 0;
381 uint32_t *data = buf;
382 uint64_t last;
383 int idx;
384
385 if (!drm_dev_enter(adev_to_drm(adev), &idx))
386 return;
387
388 BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
389
390 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
391 for (last = pos + size; pos < last; pos += 4) {
392 tmp = pos >> 31;
393
394 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
395 if (tmp != hi) {
396 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
397 hi = tmp;
398 }
399 if (write)
400 WREG32_NO_KIQ(mmMM_DATA, *data++);
401 else
402 *data++ = RREG32_NO_KIQ(mmMM_DATA);
403 }
404
405 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
406 drm_dev_exit(idx);
407}
408
409/**
410 * amdgpu_device_aper_access - access vram by vram aperature
411 *
412 * @adev: amdgpu_device pointer
413 * @pos: offset of the buffer in vram
414 * @buf: virtual address of the buffer in system memory
415 * @size: read/write size, sizeof(@buf) must > @size
416 * @write: true - write to vram, otherwise - read from vram
417 *
418 * The return value means how many bytes have been transferred.
419 */
420size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
421 void *buf, size_t size, bool write)
422{
423#ifdef CONFIG_64BIT
424 void __iomem *addr;
425 size_t count = 0;
426 uint64_t last;
427
428 if (!adev->mman.aper_base_kaddr)
429 return 0;
430
431 last = min(pos + size, adev->gmc.visible_vram_size);
432 if (last > pos) {
433 addr = adev->mman.aper_base_kaddr + pos;
434 count = last - pos;
435
436 if (write) {
437 memcpy_toio(addr, buf, count);
438 /* Make sure HDP write cache flush happens without any reordering
439 * after the system memory contents are sent over PCIe device
440 */
441 mb();
442 amdgpu_device_flush_hdp(adev, NULL);
443 } else {
444 amdgpu_device_invalidate_hdp(adev, NULL);
445 /* Make sure HDP read cache is invalidated before issuing a read
446 * to the PCIe device
447 */
448 mb();
449 memcpy_fromio(buf, addr, count);
450 }
451
452 }
453
454 return count;
455#else
456 return 0;
457#endif
458}
459
460/**
461 * amdgpu_device_vram_access - read/write a buffer in vram
462 *
463 * @adev: amdgpu_device pointer
464 * @pos: offset of the buffer in vram
465 * @buf: virtual address of the buffer in system memory
466 * @size: read/write size, sizeof(@buf) must > @size
467 * @write: true - write to vram, otherwise - read from vram
468 */
469void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
470 void *buf, size_t size, bool write)
471{
472 size_t count;
473
474 /* try to using vram apreature to access vram first */
475 count = amdgpu_device_aper_access(adev, pos, buf, size, write);
476 size -= count;
477 if (size) {
478 /* using MM to access rest vram */
479 pos += count;
480 buf += count;
481 amdgpu_device_mm_access(adev, pos, buf, size, write);
482 }
483}
484
485/*
486 * register access helper functions.
487 */
488
489/* Check if hw access should be skipped because of hotplug or device error */
490bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
491{
492 if (adev->no_hw_access)
493 return true;
494
495#ifdef CONFIG_LOCKDEP
496 /*
497 * This is a bit complicated to understand, so worth a comment. What we assert
498 * here is that the GPU reset is not running on another thread in parallel.
499 *
500 * For this we trylock the read side of the reset semaphore, if that succeeds
501 * we know that the reset is not running in paralell.
502 *
503 * If the trylock fails we assert that we are either already holding the read
504 * side of the lock or are the reset thread itself and hold the write side of
505 * the lock.
506 */
507 if (in_task()) {
508 if (down_read_trylock(&adev->reset_domain->sem))
509 up_read(&adev->reset_domain->sem);
510 else
511 lockdep_assert_held(&adev->reset_domain->sem);
512 }
513#endif
514 return false;
515}
516
517/**
518 * amdgpu_device_rreg - read a memory mapped IO or indirect register
519 *
520 * @adev: amdgpu_device pointer
521 * @reg: dword aligned register offset
522 * @acc_flags: access flags which require special behavior
523 *
524 * Returns the 32 bit value from the offset specified.
525 */
526uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
527 uint32_t reg, uint32_t acc_flags)
528{
529 uint32_t ret;
530
531 if (amdgpu_device_skip_hw_access(adev))
532 return 0;
533
534 if ((reg * 4) < adev->rmmio_size) {
535 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
536 amdgpu_sriov_runtime(adev) &&
537 down_read_trylock(&adev->reset_domain->sem)) {
538 ret = amdgpu_kiq_rreg(adev, reg, 0);
539 up_read(&adev->reset_domain->sem);
540 } else {
541 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
542 }
543 } else {
544 ret = adev->pcie_rreg(adev, reg * 4);
545 }
546
547 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
548
549 return ret;
550}
551
552/*
553 * MMIO register read with bytes helper functions
554 * @offset:bytes offset from MMIO start
555 */
556
557/**
558 * amdgpu_mm_rreg8 - read a memory mapped IO register
559 *
560 * @adev: amdgpu_device pointer
561 * @offset: byte aligned register offset
562 *
563 * Returns the 8 bit value from the offset specified.
564 */
565uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
566{
567 if (amdgpu_device_skip_hw_access(adev))
568 return 0;
569
570 if (offset < adev->rmmio_size)
571 return (readb(adev->rmmio + offset));
572 BUG();
573}
574
575
576/**
577 * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
578 *
579 * @adev: amdgpu_device pointer
580 * @reg: dword aligned register offset
581 * @acc_flags: access flags which require special behavior
582 * @xcc_id: xcc accelerated compute core id
583 *
584 * Returns the 32 bit value from the offset specified.
585 */
586uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
587 uint32_t reg, uint32_t acc_flags,
588 uint32_t xcc_id)
589{
590 uint32_t ret, rlcg_flag;
591
592 if (amdgpu_device_skip_hw_access(adev))
593 return 0;
594
595 if ((reg * 4) < adev->rmmio_size) {
596 if (amdgpu_sriov_vf(adev) &&
597 !amdgpu_sriov_runtime(adev) &&
598 adev->gfx.rlc.rlcg_reg_access_supported &&
599 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
600 GC_HWIP, false,
601 &rlcg_flag)) {
602 ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, xcc_id);
603 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
604 amdgpu_sriov_runtime(adev) &&
605 down_read_trylock(&adev->reset_domain->sem)) {
606 ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
607 up_read(&adev->reset_domain->sem);
608 } else {
609 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
610 }
611 } else {
612 ret = adev->pcie_rreg(adev, reg * 4);
613 }
614
615 return ret;
616}
617
618/*
619 * MMIO register write with bytes helper functions
620 * @offset:bytes offset from MMIO start
621 * @value: the value want to be written to the register
622 */
623
624/**
625 * amdgpu_mm_wreg8 - read a memory mapped IO register
626 *
627 * @adev: amdgpu_device pointer
628 * @offset: byte aligned register offset
629 * @value: 8 bit value to write
630 *
631 * Writes the value specified to the offset specified.
632 */
633void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
634{
635 if (amdgpu_device_skip_hw_access(adev))
636 return;
637
638 if (offset < adev->rmmio_size)
639 writeb(value, adev->rmmio + offset);
640 else
641 BUG();
642}
643
644/**
645 * amdgpu_device_wreg - write to a memory mapped IO or indirect register
646 *
647 * @adev: amdgpu_device pointer
648 * @reg: dword aligned register offset
649 * @v: 32 bit value to write to the register
650 * @acc_flags: access flags which require special behavior
651 *
652 * Writes the value specified to the offset specified.
653 */
654void amdgpu_device_wreg(struct amdgpu_device *adev,
655 uint32_t reg, uint32_t v,
656 uint32_t acc_flags)
657{
658 if (amdgpu_device_skip_hw_access(adev))
659 return;
660
661 if ((reg * 4) < adev->rmmio_size) {
662 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
663 amdgpu_sriov_runtime(adev) &&
664 down_read_trylock(&adev->reset_domain->sem)) {
665 amdgpu_kiq_wreg(adev, reg, v, 0);
666 up_read(&adev->reset_domain->sem);
667 } else {
668 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
669 }
670 } else {
671 adev->pcie_wreg(adev, reg * 4, v);
672 }
673
674 trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
675}
676
677/**
678 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range
679 *
680 * @adev: amdgpu_device pointer
681 * @reg: mmio/rlc register
682 * @v: value to write
683 * @xcc_id: xcc accelerated compute core id
684 *
685 * this function is invoked only for the debugfs register access
686 */
687void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
688 uint32_t reg, uint32_t v,
689 uint32_t xcc_id)
690{
691 if (amdgpu_device_skip_hw_access(adev))
692 return;
693
694 if (amdgpu_sriov_fullaccess(adev) &&
695 adev->gfx.rlc.funcs &&
696 adev->gfx.rlc.funcs->is_rlcg_access_range) {
697 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
698 return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
699 } else if ((reg * 4) >= adev->rmmio_size) {
700 adev->pcie_wreg(adev, reg * 4, v);
701 } else {
702 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
703 }
704}
705
706/**
707 * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
708 *
709 * @adev: amdgpu_device pointer
710 * @reg: dword aligned register offset
711 * @v: 32 bit value to write to the register
712 * @acc_flags: access flags which require special behavior
713 * @xcc_id: xcc accelerated compute core id
714 *
715 * Writes the value specified to the offset specified.
716 */
717void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
718 uint32_t reg, uint32_t v,
719 uint32_t acc_flags, uint32_t xcc_id)
720{
721 uint32_t rlcg_flag;
722
723 if (amdgpu_device_skip_hw_access(adev))
724 return;
725
726 if ((reg * 4) < adev->rmmio_size) {
727 if (amdgpu_sriov_vf(adev) &&
728 !amdgpu_sriov_runtime(adev) &&
729 adev->gfx.rlc.rlcg_reg_access_supported &&
730 amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
731 GC_HWIP, true,
732 &rlcg_flag)) {
733 amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, xcc_id);
734 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
735 amdgpu_sriov_runtime(adev) &&
736 down_read_trylock(&adev->reset_domain->sem)) {
737 amdgpu_kiq_wreg(adev, reg, v, xcc_id);
738 up_read(&adev->reset_domain->sem);
739 } else {
740 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
741 }
742 } else {
743 adev->pcie_wreg(adev, reg * 4, v);
744 }
745}
746
747/**
748 * amdgpu_device_indirect_rreg - read an indirect register
749 *
750 * @adev: amdgpu_device pointer
751 * @reg_addr: indirect register address to read from
752 *
753 * Returns the value of indirect register @reg_addr
754 */
755u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
756 u32 reg_addr)
757{
758 unsigned long flags, pcie_index, pcie_data;
759 void __iomem *pcie_index_offset;
760 void __iomem *pcie_data_offset;
761 u32 r;
762
763 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
764 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
765
766 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
767 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
768 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
769
770 writel(reg_addr, pcie_index_offset);
771 readl(pcie_index_offset);
772 r = readl(pcie_data_offset);
773 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
774
775 return r;
776}
777
778u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
779 u64 reg_addr)
780{
781 unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
782 u32 r;
783 void __iomem *pcie_index_offset;
784 void __iomem *pcie_index_hi_offset;
785 void __iomem *pcie_data_offset;
786
787 if (unlikely(!adev->nbio.funcs)) {
788 pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
789 pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
790 } else {
791 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
792 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
793 }
794
795 if (reg_addr >> 32) {
796 if (unlikely(!adev->nbio.funcs))
797 pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
798 else
799 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
800 } else {
801 pcie_index_hi = 0;
802 }
803
804 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
805 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
806 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
807 if (pcie_index_hi != 0)
808 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
809 pcie_index_hi * 4;
810
811 writel(reg_addr, pcie_index_offset);
812 readl(pcie_index_offset);
813 if (pcie_index_hi != 0) {
814 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
815 readl(pcie_index_hi_offset);
816 }
817 r = readl(pcie_data_offset);
818
819 /* clear the high bits */
820 if (pcie_index_hi != 0) {
821 writel(0, pcie_index_hi_offset);
822 readl(pcie_index_hi_offset);
823 }
824
825 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
826
827 return r;
828}
829
830/**
831 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
832 *
833 * @adev: amdgpu_device pointer
834 * @reg_addr: indirect register address to read from
835 *
836 * Returns the value of indirect register @reg_addr
837 */
838u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
839 u32 reg_addr)
840{
841 unsigned long flags, pcie_index, pcie_data;
842 void __iomem *pcie_index_offset;
843 void __iomem *pcie_data_offset;
844 u64 r;
845
846 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
847 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
848
849 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
850 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
851 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
852
853 /* read low 32 bits */
854 writel(reg_addr, pcie_index_offset);
855 readl(pcie_index_offset);
856 r = readl(pcie_data_offset);
857 /* read high 32 bits */
858 writel(reg_addr + 4, pcie_index_offset);
859 readl(pcie_index_offset);
860 r |= ((u64)readl(pcie_data_offset) << 32);
861 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
862
863 return r;
864}
865
866u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
867 u64 reg_addr)
868{
869 unsigned long flags, pcie_index, pcie_data;
870 unsigned long pcie_index_hi = 0;
871 void __iomem *pcie_index_offset;
872 void __iomem *pcie_index_hi_offset;
873 void __iomem *pcie_data_offset;
874 u64 r;
875
876 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
877 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
878 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
879 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
880
881 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
882 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
883 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
884 if (pcie_index_hi != 0)
885 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
886 pcie_index_hi * 4;
887
888 /* read low 32 bits */
889 writel(reg_addr, pcie_index_offset);
890 readl(pcie_index_offset);
891 if (pcie_index_hi != 0) {
892 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
893 readl(pcie_index_hi_offset);
894 }
895 r = readl(pcie_data_offset);
896 /* read high 32 bits */
897 writel(reg_addr + 4, pcie_index_offset);
898 readl(pcie_index_offset);
899 if (pcie_index_hi != 0) {
900 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
901 readl(pcie_index_hi_offset);
902 }
903 r |= ((u64)readl(pcie_data_offset) << 32);
904
905 /* clear the high bits */
906 if (pcie_index_hi != 0) {
907 writel(0, pcie_index_hi_offset);
908 readl(pcie_index_hi_offset);
909 }
910
911 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
912
913 return r;
914}
915
916/**
917 * amdgpu_device_indirect_wreg - write an indirect register address
918 *
919 * @adev: amdgpu_device pointer
920 * @reg_addr: indirect register offset
921 * @reg_data: indirect register data
922 *
923 */
924void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
925 u32 reg_addr, u32 reg_data)
926{
927 unsigned long flags, pcie_index, pcie_data;
928 void __iomem *pcie_index_offset;
929 void __iomem *pcie_data_offset;
930
931 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
932 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
933
934 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
935 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
936 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
937
938 writel(reg_addr, pcie_index_offset);
939 readl(pcie_index_offset);
940 writel(reg_data, pcie_data_offset);
941 readl(pcie_data_offset);
942 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
943}
944
945void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
946 u64 reg_addr, u32 reg_data)
947{
948 unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
949 void __iomem *pcie_index_offset;
950 void __iomem *pcie_index_hi_offset;
951 void __iomem *pcie_data_offset;
952
953 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
954 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
955 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
956 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
957 else
958 pcie_index_hi = 0;
959
960 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
961 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
962 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
963 if (pcie_index_hi != 0)
964 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
965 pcie_index_hi * 4;
966
967 writel(reg_addr, pcie_index_offset);
968 readl(pcie_index_offset);
969 if (pcie_index_hi != 0) {
970 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
971 readl(pcie_index_hi_offset);
972 }
973 writel(reg_data, pcie_data_offset);
974 readl(pcie_data_offset);
975
976 /* clear the high bits */
977 if (pcie_index_hi != 0) {
978 writel(0, pcie_index_hi_offset);
979 readl(pcie_index_hi_offset);
980 }
981
982 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
983}
984
985/**
986 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
987 *
988 * @adev: amdgpu_device pointer
989 * @reg_addr: indirect register offset
990 * @reg_data: indirect register data
991 *
992 */
993void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
994 u32 reg_addr, u64 reg_data)
995{
996 unsigned long flags, pcie_index, pcie_data;
997 void __iomem *pcie_index_offset;
998 void __iomem *pcie_data_offset;
999
1000 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1001 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1002
1003 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1004 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1005 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1006
1007 /* write low 32 bits */
1008 writel(reg_addr, pcie_index_offset);
1009 readl(pcie_index_offset);
1010 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1011 readl(pcie_data_offset);
1012 /* write high 32 bits */
1013 writel(reg_addr + 4, pcie_index_offset);
1014 readl(pcie_index_offset);
1015 writel((u32)(reg_data >> 32), pcie_data_offset);
1016 readl(pcie_data_offset);
1017 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1018}
1019
1020void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
1021 u64 reg_addr, u64 reg_data)
1022{
1023 unsigned long flags, pcie_index, pcie_data;
1024 unsigned long pcie_index_hi = 0;
1025 void __iomem *pcie_index_offset;
1026 void __iomem *pcie_index_hi_offset;
1027 void __iomem *pcie_data_offset;
1028
1029 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1030 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1031 if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1032 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1033
1034 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1035 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1036 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1037 if (pcie_index_hi != 0)
1038 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1039 pcie_index_hi * 4;
1040
1041 /* write low 32 bits */
1042 writel(reg_addr, pcie_index_offset);
1043 readl(pcie_index_offset);
1044 if (pcie_index_hi != 0) {
1045 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1046 readl(pcie_index_hi_offset);
1047 }
1048 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1049 readl(pcie_data_offset);
1050 /* write high 32 bits */
1051 writel(reg_addr + 4, pcie_index_offset);
1052 readl(pcie_index_offset);
1053 if (pcie_index_hi != 0) {
1054 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1055 readl(pcie_index_hi_offset);
1056 }
1057 writel((u32)(reg_data >> 32), pcie_data_offset);
1058 readl(pcie_data_offset);
1059
1060 /* clear the high bits */
1061 if (pcie_index_hi != 0) {
1062 writel(0, pcie_index_hi_offset);
1063 readl(pcie_index_hi_offset);
1064 }
1065
1066 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1067}
1068
1069/**
1070 * amdgpu_device_get_rev_id - query device rev_id
1071 *
1072 * @adev: amdgpu_device pointer
1073 *
1074 * Return device rev_id
1075 */
1076u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
1077{
1078 return adev->nbio.funcs->get_rev_id(adev);
1079}
1080
1081/**
1082 * amdgpu_invalid_rreg - dummy reg read function
1083 *
1084 * @adev: amdgpu_device pointer
1085 * @reg: offset of register
1086 *
1087 * Dummy register read function. Used for register blocks
1088 * that certain asics don't have (all asics).
1089 * Returns the value in the register.
1090 */
1091static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
1092{
1093 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
1094 BUG();
1095 return 0;
1096}
1097
1098static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
1099{
1100 DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1101 BUG();
1102 return 0;
1103}
1104
1105/**
1106 * amdgpu_invalid_wreg - dummy reg write function
1107 *
1108 * @adev: amdgpu_device pointer
1109 * @reg: offset of register
1110 * @v: value to write to the register
1111 *
1112 * Dummy register read function. Used for register blocks
1113 * that certain asics don't have (all asics).
1114 */
1115static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
1116{
1117 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
1118 reg, v);
1119 BUG();
1120}
1121
1122static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
1123{
1124 DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
1125 reg, v);
1126 BUG();
1127}
1128
1129/**
1130 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1131 *
1132 * @adev: amdgpu_device pointer
1133 * @reg: offset of register
1134 *
1135 * Dummy register read function. Used for register blocks
1136 * that certain asics don't have (all asics).
1137 * Returns the value in the register.
1138 */
1139static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
1140{
1141 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
1142 BUG();
1143 return 0;
1144}
1145
1146static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
1147{
1148 DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1149 BUG();
1150 return 0;
1151}
1152
1153/**
1154 * amdgpu_invalid_wreg64 - dummy reg write function
1155 *
1156 * @adev: amdgpu_device pointer
1157 * @reg: offset of register
1158 * @v: value to write to the register
1159 *
1160 * Dummy register read function. Used for register blocks
1161 * that certain asics don't have (all asics).
1162 */
1163static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1164{
1165 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1166 reg, v);
1167 BUG();
1168}
1169
1170static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1171{
1172 DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1173 reg, v);
1174 BUG();
1175}
1176
1177/**
1178 * amdgpu_block_invalid_rreg - dummy reg read function
1179 *
1180 * @adev: amdgpu_device pointer
1181 * @block: offset of instance
1182 * @reg: offset of register
1183 *
1184 * Dummy register read function. Used for register blocks
1185 * that certain asics don't have (all asics).
1186 * Returns the value in the register.
1187 */
1188static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1189 uint32_t block, uint32_t reg)
1190{
1191 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1192 reg, block);
1193 BUG();
1194 return 0;
1195}
1196
1197/**
1198 * amdgpu_block_invalid_wreg - dummy reg write function
1199 *
1200 * @adev: amdgpu_device pointer
1201 * @block: offset of instance
1202 * @reg: offset of register
1203 * @v: value to write to the register
1204 *
1205 * Dummy register read function. Used for register blocks
1206 * that certain asics don't have (all asics).
1207 */
1208static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1209 uint32_t block,
1210 uint32_t reg, uint32_t v)
1211{
1212 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1213 reg, block, v);
1214 BUG();
1215}
1216
1217/**
1218 * amdgpu_device_asic_init - Wrapper for atom asic_init
1219 *
1220 * @adev: amdgpu_device pointer
1221 *
1222 * Does any asic specific work and then calls atom asic init.
1223 */
1224static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1225{
1226 int ret;
1227
1228 amdgpu_asic_pre_asic_init(adev);
1229
1230 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1231 amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
1232 amdgpu_psp_wait_for_bootloader(adev);
1233 ret = amdgpu_atomfirmware_asic_init(adev, true);
1234 return ret;
1235 } else {
1236 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1237 }
1238
1239 return 0;
1240}
1241
1242/**
1243 * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1244 *
1245 * @adev: amdgpu_device pointer
1246 *
1247 * Allocates a scratch page of VRAM for use by various things in the
1248 * driver.
1249 */
1250static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1251{
1252 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1253 AMDGPU_GEM_DOMAIN_VRAM |
1254 AMDGPU_GEM_DOMAIN_GTT,
1255 &adev->mem_scratch.robj,
1256 &adev->mem_scratch.gpu_addr,
1257 (void **)&adev->mem_scratch.ptr);
1258}
1259
1260/**
1261 * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1262 *
1263 * @adev: amdgpu_device pointer
1264 *
1265 * Frees the VRAM scratch page.
1266 */
1267static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1268{
1269 amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1270}
1271
1272/**
1273 * amdgpu_device_program_register_sequence - program an array of registers.
1274 *
1275 * @adev: amdgpu_device pointer
1276 * @registers: pointer to the register array
1277 * @array_size: size of the register array
1278 *
1279 * Programs an array or registers with and or masks.
1280 * This is a helper for setting golden registers.
1281 */
1282void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1283 const u32 *registers,
1284 const u32 array_size)
1285{
1286 u32 tmp, reg, and_mask, or_mask;
1287 int i;
1288
1289 if (array_size % 3)
1290 return;
1291
1292 for (i = 0; i < array_size; i += 3) {
1293 reg = registers[i + 0];
1294 and_mask = registers[i + 1];
1295 or_mask = registers[i + 2];
1296
1297 if (and_mask == 0xffffffff) {
1298 tmp = or_mask;
1299 } else {
1300 tmp = RREG32(reg);
1301 tmp &= ~and_mask;
1302 if (adev->family >= AMDGPU_FAMILY_AI)
1303 tmp |= (or_mask & and_mask);
1304 else
1305 tmp |= or_mask;
1306 }
1307 WREG32(reg, tmp);
1308 }
1309}
1310
1311/**
1312 * amdgpu_device_pci_config_reset - reset the GPU
1313 *
1314 * @adev: amdgpu_device pointer
1315 *
1316 * Resets the GPU using the pci config reset sequence.
1317 * Only applicable to asics prior to vega10.
1318 */
1319void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1320{
1321 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1322}
1323
1324/**
1325 * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1326 *
1327 * @adev: amdgpu_device pointer
1328 *
1329 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1330 */
1331int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1332{
1333 return pci_reset_function(adev->pdev);
1334}
1335
1336/*
1337 * amdgpu_device_wb_*()
1338 * Writeback is the method by which the GPU updates special pages in memory
1339 * with the status of certain GPU events (fences, ring pointers,etc.).
1340 */
1341
1342/**
1343 * amdgpu_device_wb_fini - Disable Writeback and free memory
1344 *
1345 * @adev: amdgpu_device pointer
1346 *
1347 * Disables Writeback and frees the Writeback memory (all asics).
1348 * Used at driver shutdown.
1349 */
1350static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1351{
1352 if (adev->wb.wb_obj) {
1353 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1354 &adev->wb.gpu_addr,
1355 (void **)&adev->wb.wb);
1356 adev->wb.wb_obj = NULL;
1357 }
1358}
1359
1360/**
1361 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1362 *
1363 * @adev: amdgpu_device pointer
1364 *
1365 * Initializes writeback and allocates writeback memory (all asics).
1366 * Used at driver startup.
1367 * Returns 0 on success or an -error on failure.
1368 */
1369static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1370{
1371 int r;
1372
1373 if (adev->wb.wb_obj == NULL) {
1374 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1375 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1376 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1377 &adev->wb.wb_obj, &adev->wb.gpu_addr,
1378 (void **)&adev->wb.wb);
1379 if (r) {
1380 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1381 return r;
1382 }
1383
1384 adev->wb.num_wb = AMDGPU_MAX_WB;
1385 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1386
1387 /* clear wb memory */
1388 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1389 }
1390
1391 return 0;
1392}
1393
1394/**
1395 * amdgpu_device_wb_get - Allocate a wb entry
1396 *
1397 * @adev: amdgpu_device pointer
1398 * @wb: wb index
1399 *
1400 * Allocate a wb slot for use by the driver (all asics).
1401 * Returns 0 on success or -EINVAL on failure.
1402 */
1403int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1404{
1405 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1406
1407 if (offset < adev->wb.num_wb) {
1408 __set_bit(offset, adev->wb.used);
1409 *wb = offset << 3; /* convert to dw offset */
1410 return 0;
1411 } else {
1412 return -EINVAL;
1413 }
1414}
1415
1416/**
1417 * amdgpu_device_wb_free - Free a wb entry
1418 *
1419 * @adev: amdgpu_device pointer
1420 * @wb: wb index
1421 *
1422 * Free a wb slot allocated for use by the driver (all asics)
1423 */
1424void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1425{
1426 wb >>= 3;
1427 if (wb < adev->wb.num_wb)
1428 __clear_bit(wb, adev->wb.used);
1429}
1430
1431/**
1432 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1433 *
1434 * @adev: amdgpu_device pointer
1435 *
1436 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1437 * to fail, but if any of the BARs is not accessible after the size we abort
1438 * driver loading by returning -ENODEV.
1439 */
1440int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1441{
1442 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1443 struct pci_bus *root;
1444 struct resource *res;
1445 unsigned int i;
1446 u16 cmd;
1447 int r;
1448
1449 if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1450 return 0;
1451
1452 /* Bypass for VF */
1453 if (amdgpu_sriov_vf(adev))
1454 return 0;
1455
1456 /* PCI_EXT_CAP_ID_VNDR extended capability is located at 0x100 */
1457 if (!pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_VNDR))
1458 DRM_WARN("System can't access extended configuration space,please check!!\n");
1459
1460 /* skip if the bios has already enabled large BAR */
1461 if (adev->gmc.real_vram_size &&
1462 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1463 return 0;
1464
1465 /* Check if the root BUS has 64bit memory resources */
1466 root = adev->pdev->bus;
1467 while (root->parent)
1468 root = root->parent;
1469
1470 pci_bus_for_each_resource(root, res, i) {
1471 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1472 res->start > 0x100000000ull)
1473 break;
1474 }
1475
1476 /* Trying to resize is pointless without a root hub window above 4GB */
1477 if (!res)
1478 return 0;
1479
1480 /* Limit the BAR size to what is available */
1481 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1482 rbar_size);
1483
1484 /* Disable memory decoding while we change the BAR addresses and size */
1485 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1486 pci_write_config_word(adev->pdev, PCI_COMMAND,
1487 cmd & ~PCI_COMMAND_MEMORY);
1488
1489 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1490 amdgpu_doorbell_fini(adev);
1491 if (adev->asic_type >= CHIP_BONAIRE)
1492 pci_release_resource(adev->pdev, 2);
1493
1494 pci_release_resource(adev->pdev, 0);
1495
1496 r = pci_resize_resource(adev->pdev, 0, rbar_size);
1497 if (r == -ENOSPC)
1498 DRM_INFO("Not enough PCI address space for a large BAR.");
1499 else if (r && r != -ENOTSUPP)
1500 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1501
1502 pci_assign_unassigned_bus_resources(adev->pdev->bus);
1503
1504 /* When the doorbell or fb BAR isn't available we have no chance of
1505 * using the device.
1506 */
1507 r = amdgpu_doorbell_init(adev);
1508 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1509 return -ENODEV;
1510
1511 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1512
1513 return 0;
1514}
1515
1516static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1517{
1518 if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1519 return false;
1520
1521 return true;
1522}
1523
1524/*
1525 * GPU helpers function.
1526 */
1527/**
1528 * amdgpu_device_need_post - check if the hw need post or not
1529 *
1530 * @adev: amdgpu_device pointer
1531 *
1532 * Check if the asic has been initialized (all asics) at driver startup
1533 * or post is needed if hw reset is performed.
1534 * Returns true if need or false if not.
1535 */
1536bool amdgpu_device_need_post(struct amdgpu_device *adev)
1537{
1538 uint32_t reg;
1539
1540 if (amdgpu_sriov_vf(adev))
1541 return false;
1542
1543 if (!amdgpu_device_read_bios(adev))
1544 return false;
1545
1546 if (amdgpu_passthrough(adev)) {
1547 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1548 * some old smc fw still need driver do vPost otherwise gpu hang, while
1549 * those smc fw version above 22.15 doesn't have this flaw, so we force
1550 * vpost executed for smc version below 22.15
1551 */
1552 if (adev->asic_type == CHIP_FIJI) {
1553 int err;
1554 uint32_t fw_ver;
1555
1556 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1557 /* force vPost if error occured */
1558 if (err)
1559 return true;
1560
1561 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1562 release_firmware(adev->pm.fw);
1563 if (fw_ver < 0x00160e00)
1564 return true;
1565 }
1566 }
1567
1568 /* Don't post if we need to reset whole hive on init */
1569 if (adev->gmc.xgmi.pending_reset)
1570 return false;
1571
1572 if (adev->has_hw_reset) {
1573 adev->has_hw_reset = false;
1574 return true;
1575 }
1576
1577 /* bios scratch used on CIK+ */
1578 if (adev->asic_type >= CHIP_BONAIRE)
1579 return amdgpu_atombios_scratch_need_asic_init(adev);
1580
1581 /* check MEM_SIZE for older asics */
1582 reg = amdgpu_asic_get_config_memsize(adev);
1583
1584 if ((reg != 0) && (reg != 0xffffffff))
1585 return false;
1586
1587 return true;
1588}
1589
1590/*
1591 * Check whether seamless boot is supported.
1592 *
1593 * So far we only support seamless boot on DCE 3.0 or later.
1594 * If users report that it works on older ASICS as well, we may
1595 * loosen this.
1596 */
1597bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1598{
1599 switch (amdgpu_seamless) {
1600 case -1:
1601 break;
1602 case 1:
1603 return true;
1604 case 0:
1605 return false;
1606 default:
1607 DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1608 amdgpu_seamless);
1609 return false;
1610 }
1611
1612 if (!(adev->flags & AMD_IS_APU))
1613 return false;
1614
1615 if (adev->mman.keep_stolen_vga_memory)
1616 return false;
1617
1618 return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
1619}
1620
1621/*
1622 * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1623 * don't support dynamic speed switching. Until we have confirmation from Intel
1624 * that a specific host supports it, it's safer that we keep it disabled for all.
1625 *
1626 * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1627 * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1628 */
1629static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1630{
1631#if IS_ENABLED(CONFIG_X86)
1632 struct cpuinfo_x86 *c = &cpu_data(0);
1633
1634 /* eGPU change speeds based on USB4 fabric conditions */
1635 if (dev_is_removable(adev->dev))
1636 return true;
1637
1638 if (c->x86_vendor == X86_VENDOR_INTEL)
1639 return false;
1640#endif
1641 return true;
1642}
1643
1644/**
1645 * amdgpu_device_should_use_aspm - check if the device should program ASPM
1646 *
1647 * @adev: amdgpu_device pointer
1648 *
1649 * Confirm whether the module parameter and pcie bridge agree that ASPM should
1650 * be set for this device.
1651 *
1652 * Returns true if it should be used or false if not.
1653 */
1654bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1655{
1656 switch (amdgpu_aspm) {
1657 case -1:
1658 break;
1659 case 0:
1660 return false;
1661 case 1:
1662 return true;
1663 default:
1664 return false;
1665 }
1666 if (adev->flags & AMD_IS_APU)
1667 return false;
1668 if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1669 return false;
1670 return pcie_aspm_enabled(adev->pdev);
1671}
1672
1673/* if we get transitioned to only one device, take VGA back */
1674/**
1675 * amdgpu_device_vga_set_decode - enable/disable vga decode
1676 *
1677 * @pdev: PCI device pointer
1678 * @state: enable/disable vga decode
1679 *
1680 * Enable/disable vga decode (all asics).
1681 * Returns VGA resource flags.
1682 */
1683static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1684 bool state)
1685{
1686 struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1687
1688 amdgpu_asic_set_vga_state(adev, state);
1689 if (state)
1690 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1691 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1692 else
1693 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1694}
1695
1696/**
1697 * amdgpu_device_check_block_size - validate the vm block size
1698 *
1699 * @adev: amdgpu_device pointer
1700 *
1701 * Validates the vm block size specified via module parameter.
1702 * The vm block size defines number of bits in page table versus page directory,
1703 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1704 * page table and the remaining bits are in the page directory.
1705 */
1706static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1707{
1708 /* defines number of bits in page table versus page directory,
1709 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1710 * page table and the remaining bits are in the page directory
1711 */
1712 if (amdgpu_vm_block_size == -1)
1713 return;
1714
1715 if (amdgpu_vm_block_size < 9) {
1716 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1717 amdgpu_vm_block_size);
1718 amdgpu_vm_block_size = -1;
1719 }
1720}
1721
1722/**
1723 * amdgpu_device_check_vm_size - validate the vm size
1724 *
1725 * @adev: amdgpu_device pointer
1726 *
1727 * Validates the vm size in GB specified via module parameter.
1728 * The VM size is the size of the GPU virtual memory space in GB.
1729 */
1730static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1731{
1732 /* no need to check the default value */
1733 if (amdgpu_vm_size == -1)
1734 return;
1735
1736 if (amdgpu_vm_size < 1) {
1737 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1738 amdgpu_vm_size);
1739 amdgpu_vm_size = -1;
1740 }
1741}
1742
1743static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1744{
1745 struct sysinfo si;
1746 bool is_os_64 = (sizeof(void *) == 8);
1747 uint64_t total_memory;
1748 uint64_t dram_size_seven_GB = 0x1B8000000;
1749 uint64_t dram_size_three_GB = 0xB8000000;
1750
1751 if (amdgpu_smu_memory_pool_size == 0)
1752 return;
1753
1754 if (!is_os_64) {
1755 DRM_WARN("Not 64-bit OS, feature not supported\n");
1756 goto def_value;
1757 }
1758 si_meminfo(&si);
1759 total_memory = (uint64_t)si.totalram * si.mem_unit;
1760
1761 if ((amdgpu_smu_memory_pool_size == 1) ||
1762 (amdgpu_smu_memory_pool_size == 2)) {
1763 if (total_memory < dram_size_three_GB)
1764 goto def_value1;
1765 } else if ((amdgpu_smu_memory_pool_size == 4) ||
1766 (amdgpu_smu_memory_pool_size == 8)) {
1767 if (total_memory < dram_size_seven_GB)
1768 goto def_value1;
1769 } else {
1770 DRM_WARN("Smu memory pool size not supported\n");
1771 goto def_value;
1772 }
1773 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1774
1775 return;
1776
1777def_value1:
1778 DRM_WARN("No enough system memory\n");
1779def_value:
1780 adev->pm.smu_prv_buffer_size = 0;
1781}
1782
1783static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1784{
1785 if (!(adev->flags & AMD_IS_APU) ||
1786 adev->asic_type < CHIP_RAVEN)
1787 return 0;
1788
1789 switch (adev->asic_type) {
1790 case CHIP_RAVEN:
1791 if (adev->pdev->device == 0x15dd)
1792 adev->apu_flags |= AMD_APU_IS_RAVEN;
1793 if (adev->pdev->device == 0x15d8)
1794 adev->apu_flags |= AMD_APU_IS_PICASSO;
1795 break;
1796 case CHIP_RENOIR:
1797 if ((adev->pdev->device == 0x1636) ||
1798 (adev->pdev->device == 0x164c))
1799 adev->apu_flags |= AMD_APU_IS_RENOIR;
1800 else
1801 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1802 break;
1803 case CHIP_VANGOGH:
1804 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1805 break;
1806 case CHIP_YELLOW_CARP:
1807 break;
1808 case CHIP_CYAN_SKILLFISH:
1809 if ((adev->pdev->device == 0x13FE) ||
1810 (adev->pdev->device == 0x143F))
1811 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1812 break;
1813 default:
1814 break;
1815 }
1816
1817 return 0;
1818}
1819
1820/**
1821 * amdgpu_device_check_arguments - validate module params
1822 *
1823 * @adev: amdgpu_device pointer
1824 *
1825 * Validates certain module parameters and updates
1826 * the associated values used by the driver (all asics).
1827 */
1828static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1829{
1830 if (amdgpu_sched_jobs < 4) {
1831 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1832 amdgpu_sched_jobs);
1833 amdgpu_sched_jobs = 4;
1834 } else if (!is_power_of_2(amdgpu_sched_jobs)) {
1835 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1836 amdgpu_sched_jobs);
1837 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1838 }
1839
1840 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1841 /* gart size must be greater or equal to 32M */
1842 dev_warn(adev->dev, "gart size (%d) too small\n",
1843 amdgpu_gart_size);
1844 amdgpu_gart_size = -1;
1845 }
1846
1847 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1848 /* gtt size must be greater or equal to 32M */
1849 dev_warn(adev->dev, "gtt size (%d) too small\n",
1850 amdgpu_gtt_size);
1851 amdgpu_gtt_size = -1;
1852 }
1853
1854 /* valid range is between 4 and 9 inclusive */
1855 if (amdgpu_vm_fragment_size != -1 &&
1856 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1857 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1858 amdgpu_vm_fragment_size = -1;
1859 }
1860
1861 if (amdgpu_sched_hw_submission < 2) {
1862 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1863 amdgpu_sched_hw_submission);
1864 amdgpu_sched_hw_submission = 2;
1865 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1866 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1867 amdgpu_sched_hw_submission);
1868 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1869 }
1870
1871 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1872 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1873 amdgpu_reset_method = -1;
1874 }
1875
1876 amdgpu_device_check_smu_prv_buffer_size(adev);
1877
1878 amdgpu_device_check_vm_size(adev);
1879
1880 amdgpu_device_check_block_size(adev);
1881
1882 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1883
1884 return 0;
1885}
1886
1887/**
1888 * amdgpu_switcheroo_set_state - set switcheroo state
1889 *
1890 * @pdev: pci dev pointer
1891 * @state: vga_switcheroo state
1892 *
1893 * Callback for the switcheroo driver. Suspends or resumes
1894 * the asics before or after it is powered up using ACPI methods.
1895 */
1896static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1897 enum vga_switcheroo_state state)
1898{
1899 struct drm_device *dev = pci_get_drvdata(pdev);
1900 int r;
1901
1902 if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1903 return;
1904
1905 if (state == VGA_SWITCHEROO_ON) {
1906 pr_info("switched on\n");
1907 /* don't suspend or resume card normally */
1908 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1909
1910 pci_set_power_state(pdev, PCI_D0);
1911 amdgpu_device_load_pci_state(pdev);
1912 r = pci_enable_device(pdev);
1913 if (r)
1914 DRM_WARN("pci_enable_device failed (%d)\n", r);
1915 amdgpu_device_resume(dev, true);
1916
1917 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1918 } else {
1919 pr_info("switched off\n");
1920 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1921 amdgpu_device_prepare(dev);
1922 amdgpu_device_suspend(dev, true);
1923 amdgpu_device_cache_pci_state(pdev);
1924 /* Shut down the device */
1925 pci_disable_device(pdev);
1926 pci_set_power_state(pdev, PCI_D3cold);
1927 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1928 }
1929}
1930
1931/**
1932 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1933 *
1934 * @pdev: pci dev pointer
1935 *
1936 * Callback for the switcheroo driver. Check of the switcheroo
1937 * state can be changed.
1938 * Returns true if the state can be changed, false if not.
1939 */
1940static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1941{
1942 struct drm_device *dev = pci_get_drvdata(pdev);
1943
1944 /*
1945 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1946 * locking inversion with the driver load path. And the access here is
1947 * completely racy anyway. So don't bother with locking for now.
1948 */
1949 return atomic_read(&dev->open_count) == 0;
1950}
1951
1952static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1953 .set_gpu_state = amdgpu_switcheroo_set_state,
1954 .reprobe = NULL,
1955 .can_switch = amdgpu_switcheroo_can_switch,
1956};
1957
1958/**
1959 * amdgpu_device_ip_set_clockgating_state - set the CG state
1960 *
1961 * @dev: amdgpu_device pointer
1962 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1963 * @state: clockgating state (gate or ungate)
1964 *
1965 * Sets the requested clockgating state for all instances of
1966 * the hardware IP specified.
1967 * Returns the error code from the last instance.
1968 */
1969int amdgpu_device_ip_set_clockgating_state(void *dev,
1970 enum amd_ip_block_type block_type,
1971 enum amd_clockgating_state state)
1972{
1973 struct amdgpu_device *adev = dev;
1974 int i, r = 0;
1975
1976 for (i = 0; i < adev->num_ip_blocks; i++) {
1977 if (!adev->ip_blocks[i].status.valid)
1978 continue;
1979 if (adev->ip_blocks[i].version->type != block_type)
1980 continue;
1981 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1982 continue;
1983 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1984 (void *)adev, state);
1985 if (r)
1986 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1987 adev->ip_blocks[i].version->funcs->name, r);
1988 }
1989 return r;
1990}
1991
1992/**
1993 * amdgpu_device_ip_set_powergating_state - set the PG state
1994 *
1995 * @dev: amdgpu_device pointer
1996 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1997 * @state: powergating state (gate or ungate)
1998 *
1999 * Sets the requested powergating state for all instances of
2000 * the hardware IP specified.
2001 * Returns the error code from the last instance.
2002 */
2003int amdgpu_device_ip_set_powergating_state(void *dev,
2004 enum amd_ip_block_type block_type,
2005 enum amd_powergating_state state)
2006{
2007 struct amdgpu_device *adev = dev;
2008 int i, r = 0;
2009
2010 for (i = 0; i < adev->num_ip_blocks; i++) {
2011 if (!adev->ip_blocks[i].status.valid)
2012 continue;
2013 if (adev->ip_blocks[i].version->type != block_type)
2014 continue;
2015 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
2016 continue;
2017 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
2018 (void *)adev, state);
2019 if (r)
2020 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
2021 adev->ip_blocks[i].version->funcs->name, r);
2022 }
2023 return r;
2024}
2025
2026/**
2027 * amdgpu_device_ip_get_clockgating_state - get the CG state
2028 *
2029 * @adev: amdgpu_device pointer
2030 * @flags: clockgating feature flags
2031 *
2032 * Walks the list of IPs on the device and updates the clockgating
2033 * flags for each IP.
2034 * Updates @flags with the feature flags for each hardware IP where
2035 * clockgating is enabled.
2036 */
2037void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
2038 u64 *flags)
2039{
2040 int i;
2041
2042 for (i = 0; i < adev->num_ip_blocks; i++) {
2043 if (!adev->ip_blocks[i].status.valid)
2044 continue;
2045 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
2046 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
2047 }
2048}
2049
2050/**
2051 * amdgpu_device_ip_wait_for_idle - wait for idle
2052 *
2053 * @adev: amdgpu_device pointer
2054 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2055 *
2056 * Waits for the request hardware IP to be idle.
2057 * Returns 0 for success or a negative error code on failure.
2058 */
2059int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
2060 enum amd_ip_block_type block_type)
2061{
2062 int i, r;
2063
2064 for (i = 0; i < adev->num_ip_blocks; i++) {
2065 if (!adev->ip_blocks[i].status.valid)
2066 continue;
2067 if (adev->ip_blocks[i].version->type == block_type) {
2068 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
2069 if (r)
2070 return r;
2071 break;
2072 }
2073 }
2074 return 0;
2075
2076}
2077
2078/**
2079 * amdgpu_device_ip_is_idle - is the hardware IP idle
2080 *
2081 * @adev: amdgpu_device pointer
2082 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2083 *
2084 * Check if the hardware IP is idle or not.
2085 * Returns true if it the IP is idle, false if not.
2086 */
2087bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
2088 enum amd_ip_block_type block_type)
2089{
2090 int i;
2091
2092 for (i = 0; i < adev->num_ip_blocks; i++) {
2093 if (!adev->ip_blocks[i].status.valid)
2094 continue;
2095 if (adev->ip_blocks[i].version->type == block_type)
2096 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
2097 }
2098 return true;
2099
2100}
2101
2102/**
2103 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2104 *
2105 * @adev: amdgpu_device pointer
2106 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2107 *
2108 * Returns a pointer to the hardware IP block structure
2109 * if it exists for the asic, otherwise NULL.
2110 */
2111struct amdgpu_ip_block *
2112amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
2113 enum amd_ip_block_type type)
2114{
2115 int i;
2116
2117 for (i = 0; i < adev->num_ip_blocks; i++)
2118 if (adev->ip_blocks[i].version->type == type)
2119 return &adev->ip_blocks[i];
2120
2121 return NULL;
2122}
2123
2124/**
2125 * amdgpu_device_ip_block_version_cmp
2126 *
2127 * @adev: amdgpu_device pointer
2128 * @type: enum amd_ip_block_type
2129 * @major: major version
2130 * @minor: minor version
2131 *
2132 * return 0 if equal or greater
2133 * return 1 if smaller or the ip_block doesn't exist
2134 */
2135int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
2136 enum amd_ip_block_type type,
2137 u32 major, u32 minor)
2138{
2139 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
2140
2141 if (ip_block && ((ip_block->version->major > major) ||
2142 ((ip_block->version->major == major) &&
2143 (ip_block->version->minor >= minor))))
2144 return 0;
2145
2146 return 1;
2147}
2148
2149/**
2150 * amdgpu_device_ip_block_add
2151 *
2152 * @adev: amdgpu_device pointer
2153 * @ip_block_version: pointer to the IP to add
2154 *
2155 * Adds the IP block driver information to the collection of IPs
2156 * on the asic.
2157 */
2158int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
2159 const struct amdgpu_ip_block_version *ip_block_version)
2160{
2161 if (!ip_block_version)
2162 return -EINVAL;
2163
2164 switch (ip_block_version->type) {
2165 case AMD_IP_BLOCK_TYPE_VCN:
2166 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2167 return 0;
2168 break;
2169 case AMD_IP_BLOCK_TYPE_JPEG:
2170 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2171 return 0;
2172 break;
2173 default:
2174 break;
2175 }
2176
2177 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2178 ip_block_version->funcs->name);
2179
2180 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2181
2182 return 0;
2183}
2184
2185/**
2186 * amdgpu_device_enable_virtual_display - enable virtual display feature
2187 *
2188 * @adev: amdgpu_device pointer
2189 *
2190 * Enabled the virtual display feature if the user has enabled it via
2191 * the module parameter virtual_display. This feature provides a virtual
2192 * display hardware on headless boards or in virtualized environments.
2193 * This function parses and validates the configuration string specified by
2194 * the user and configues the virtual display configuration (number of
2195 * virtual connectors, crtcs, etc.) specified.
2196 */
2197static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2198{
2199 adev->enable_virtual_display = false;
2200
2201 if (amdgpu_virtual_display) {
2202 const char *pci_address_name = pci_name(adev->pdev);
2203 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2204
2205 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2206 pciaddstr_tmp = pciaddstr;
2207 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2208 pciaddname = strsep(&pciaddname_tmp, ",");
2209 if (!strcmp("all", pciaddname)
2210 || !strcmp(pci_address_name, pciaddname)) {
2211 long num_crtc;
2212 int res = -1;
2213
2214 adev->enable_virtual_display = true;
2215
2216 if (pciaddname_tmp)
2217 res = kstrtol(pciaddname_tmp, 10,
2218 &num_crtc);
2219
2220 if (!res) {
2221 if (num_crtc < 1)
2222 num_crtc = 1;
2223 if (num_crtc > 6)
2224 num_crtc = 6;
2225 adev->mode_info.num_crtc = num_crtc;
2226 } else {
2227 adev->mode_info.num_crtc = 1;
2228 }
2229 break;
2230 }
2231 }
2232
2233 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2234 amdgpu_virtual_display, pci_address_name,
2235 adev->enable_virtual_display, adev->mode_info.num_crtc);
2236
2237 kfree(pciaddstr);
2238 }
2239}
2240
2241void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2242{
2243 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2244 adev->mode_info.num_crtc = 1;
2245 adev->enable_virtual_display = true;
2246 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2247 adev->enable_virtual_display, adev->mode_info.num_crtc);
2248 }
2249}
2250
2251/**
2252 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2253 *
2254 * @adev: amdgpu_device pointer
2255 *
2256 * Parses the asic configuration parameters specified in the gpu info
2257 * firmware and makes them availale to the driver for use in configuring
2258 * the asic.
2259 * Returns 0 on success, -EINVAL on failure.
2260 */
2261static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2262{
2263 const char *chip_name;
2264 char fw_name[40];
2265 int err;
2266 const struct gpu_info_firmware_header_v1_0 *hdr;
2267
2268 adev->firmware.gpu_info_fw = NULL;
2269
2270 if (adev->mman.discovery_bin)
2271 return 0;
2272
2273 switch (adev->asic_type) {
2274 default:
2275 return 0;
2276 case CHIP_VEGA10:
2277 chip_name = "vega10";
2278 break;
2279 case CHIP_VEGA12:
2280 chip_name = "vega12";
2281 break;
2282 case CHIP_RAVEN:
2283 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2284 chip_name = "raven2";
2285 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2286 chip_name = "picasso";
2287 else
2288 chip_name = "raven";
2289 break;
2290 case CHIP_ARCTURUS:
2291 chip_name = "arcturus";
2292 break;
2293 case CHIP_NAVI12:
2294 chip_name = "navi12";
2295 break;
2296 }
2297
2298 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2299 err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2300 if (err) {
2301 dev_err(adev->dev,
2302 "Failed to get gpu_info firmware \"%s\"\n",
2303 fw_name);
2304 goto out;
2305 }
2306
2307 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2308 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2309
2310 switch (hdr->version_major) {
2311 case 1:
2312 {
2313 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2314 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2315 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2316
2317 /*
2318 * Should be droped when DAL no longer needs it.
2319 */
2320 if (adev->asic_type == CHIP_NAVI12)
2321 goto parse_soc_bounding_box;
2322
2323 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2324 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2325 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2326 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2327 adev->gfx.config.max_texture_channel_caches =
2328 le32_to_cpu(gpu_info_fw->gc_num_tccs);
2329 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2330 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2331 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2332 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2333 adev->gfx.config.double_offchip_lds_buf =
2334 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2335 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2336 adev->gfx.cu_info.max_waves_per_simd =
2337 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2338 adev->gfx.cu_info.max_scratch_slots_per_cu =
2339 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2340 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2341 if (hdr->version_minor >= 1) {
2342 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2343 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2344 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2345 adev->gfx.config.num_sc_per_sh =
2346 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2347 adev->gfx.config.num_packer_per_sc =
2348 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2349 }
2350
2351parse_soc_bounding_box:
2352 /*
2353 * soc bounding box info is not integrated in disocovery table,
2354 * we always need to parse it from gpu info firmware if needed.
2355 */
2356 if (hdr->version_minor == 2) {
2357 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2358 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2359 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2360 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2361 }
2362 break;
2363 }
2364 default:
2365 dev_err(adev->dev,
2366 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2367 err = -EINVAL;
2368 goto out;
2369 }
2370out:
2371 return err;
2372}
2373
2374/**
2375 * amdgpu_device_ip_early_init - run early init for hardware IPs
2376 *
2377 * @adev: amdgpu_device pointer
2378 *
2379 * Early initialization pass for hardware IPs. The hardware IPs that make
2380 * up each asic are discovered each IP's early_init callback is run. This
2381 * is the first stage in initializing the asic.
2382 * Returns 0 on success, negative error code on failure.
2383 */
2384static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2385{
2386 struct pci_dev *parent;
2387 int i, r;
2388 bool total;
2389
2390 amdgpu_device_enable_virtual_display(adev);
2391
2392 if (amdgpu_sriov_vf(adev)) {
2393 r = amdgpu_virt_request_full_gpu(adev, true);
2394 if (r)
2395 return r;
2396 }
2397
2398 switch (adev->asic_type) {
2399#ifdef CONFIG_DRM_AMDGPU_SI
2400 case CHIP_VERDE:
2401 case CHIP_TAHITI:
2402 case CHIP_PITCAIRN:
2403 case CHIP_OLAND:
2404 case CHIP_HAINAN:
2405 adev->family = AMDGPU_FAMILY_SI;
2406 r = si_set_ip_blocks(adev);
2407 if (r)
2408 return r;
2409 break;
2410#endif
2411#ifdef CONFIG_DRM_AMDGPU_CIK
2412 case CHIP_BONAIRE:
2413 case CHIP_HAWAII:
2414 case CHIP_KAVERI:
2415 case CHIP_KABINI:
2416 case CHIP_MULLINS:
2417 if (adev->flags & AMD_IS_APU)
2418 adev->family = AMDGPU_FAMILY_KV;
2419 else
2420 adev->family = AMDGPU_FAMILY_CI;
2421
2422 r = cik_set_ip_blocks(adev);
2423 if (r)
2424 return r;
2425 break;
2426#endif
2427 case CHIP_TOPAZ:
2428 case CHIP_TONGA:
2429 case CHIP_FIJI:
2430 case CHIP_POLARIS10:
2431 case CHIP_POLARIS11:
2432 case CHIP_POLARIS12:
2433 case CHIP_VEGAM:
2434 case CHIP_CARRIZO:
2435 case CHIP_STONEY:
2436 if (adev->flags & AMD_IS_APU)
2437 adev->family = AMDGPU_FAMILY_CZ;
2438 else
2439 adev->family = AMDGPU_FAMILY_VI;
2440
2441 r = vi_set_ip_blocks(adev);
2442 if (r)
2443 return r;
2444 break;
2445 default:
2446 r = amdgpu_discovery_set_ip_blocks(adev);
2447 if (r)
2448 return r;
2449 break;
2450 }
2451
2452 if (amdgpu_has_atpx() &&
2453 (amdgpu_is_atpx_hybrid() ||
2454 amdgpu_has_atpx_dgpu_power_cntl()) &&
2455 ((adev->flags & AMD_IS_APU) == 0) &&
2456 !dev_is_removable(&adev->pdev->dev))
2457 adev->flags |= AMD_IS_PX;
2458
2459 if (!(adev->flags & AMD_IS_APU)) {
2460 parent = pcie_find_root_port(adev->pdev);
2461 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2462 }
2463
2464
2465 adev->pm.pp_feature = amdgpu_pp_feature_mask;
2466 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2467 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2468 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2469 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2470 if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2471 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2472
2473 total = true;
2474 for (i = 0; i < adev->num_ip_blocks; i++) {
2475 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2476 DRM_WARN("disabled ip block: %d <%s>\n",
2477 i, adev->ip_blocks[i].version->funcs->name);
2478 adev->ip_blocks[i].status.valid = false;
2479 } else {
2480 if (adev->ip_blocks[i].version->funcs->early_init) {
2481 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2482 if (r == -ENOENT) {
2483 adev->ip_blocks[i].status.valid = false;
2484 } else if (r) {
2485 DRM_ERROR("early_init of IP block <%s> failed %d\n",
2486 adev->ip_blocks[i].version->funcs->name, r);
2487 total = false;
2488 } else {
2489 adev->ip_blocks[i].status.valid = true;
2490 }
2491 } else {
2492 adev->ip_blocks[i].status.valid = true;
2493 }
2494 }
2495 /* get the vbios after the asic_funcs are set up */
2496 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2497 r = amdgpu_device_parse_gpu_info_fw(adev);
2498 if (r)
2499 return r;
2500
2501 /* Read BIOS */
2502 if (amdgpu_device_read_bios(adev)) {
2503 if (!amdgpu_get_bios(adev))
2504 return -EINVAL;
2505
2506 r = amdgpu_atombios_init(adev);
2507 if (r) {
2508 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2509 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2510 return r;
2511 }
2512 }
2513
2514 /*get pf2vf msg info at it's earliest time*/
2515 if (amdgpu_sriov_vf(adev))
2516 amdgpu_virt_init_data_exchange(adev);
2517
2518 }
2519 }
2520 if (!total)
2521 return -ENODEV;
2522
2523 amdgpu_amdkfd_device_probe(adev);
2524 adev->cg_flags &= amdgpu_cg_mask;
2525 adev->pg_flags &= amdgpu_pg_mask;
2526
2527 return 0;
2528}
2529
2530static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2531{
2532 int i, r;
2533
2534 for (i = 0; i < adev->num_ip_blocks; i++) {
2535 if (!adev->ip_blocks[i].status.sw)
2536 continue;
2537 if (adev->ip_blocks[i].status.hw)
2538 continue;
2539 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2540 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2541 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2542 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2543 if (r) {
2544 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2545 adev->ip_blocks[i].version->funcs->name, r);
2546 return r;
2547 }
2548 adev->ip_blocks[i].status.hw = true;
2549 }
2550 }
2551
2552 return 0;
2553}
2554
2555static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2556{
2557 int i, r;
2558
2559 for (i = 0; i < adev->num_ip_blocks; i++) {
2560 if (!adev->ip_blocks[i].status.sw)
2561 continue;
2562 if (adev->ip_blocks[i].status.hw)
2563 continue;
2564 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2565 if (r) {
2566 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2567 adev->ip_blocks[i].version->funcs->name, r);
2568 return r;
2569 }
2570 adev->ip_blocks[i].status.hw = true;
2571 }
2572
2573 return 0;
2574}
2575
2576static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2577{
2578 int r = 0;
2579 int i;
2580 uint32_t smu_version;
2581
2582 if (adev->asic_type >= CHIP_VEGA10) {
2583 for (i = 0; i < adev->num_ip_blocks; i++) {
2584 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2585 continue;
2586
2587 if (!adev->ip_blocks[i].status.sw)
2588 continue;
2589
2590 /* no need to do the fw loading again if already done*/
2591 if (adev->ip_blocks[i].status.hw == true)
2592 break;
2593
2594 if (amdgpu_in_reset(adev) || adev->in_suspend) {
2595 r = adev->ip_blocks[i].version->funcs->resume(adev);
2596 if (r) {
2597 DRM_ERROR("resume of IP block <%s> failed %d\n",
2598 adev->ip_blocks[i].version->funcs->name, r);
2599 return r;
2600 }
2601 } else {
2602 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2603 if (r) {
2604 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2605 adev->ip_blocks[i].version->funcs->name, r);
2606 return r;
2607 }
2608 }
2609
2610 adev->ip_blocks[i].status.hw = true;
2611 break;
2612 }
2613 }
2614
2615 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2616 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2617
2618 return r;
2619}
2620
2621static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2622{
2623 long timeout;
2624 int r, i;
2625
2626 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2627 struct amdgpu_ring *ring = adev->rings[i];
2628
2629 /* No need to setup the GPU scheduler for rings that don't need it */
2630 if (!ring || ring->no_scheduler)
2631 continue;
2632
2633 switch (ring->funcs->type) {
2634 case AMDGPU_RING_TYPE_GFX:
2635 timeout = adev->gfx_timeout;
2636 break;
2637 case AMDGPU_RING_TYPE_COMPUTE:
2638 timeout = adev->compute_timeout;
2639 break;
2640 case AMDGPU_RING_TYPE_SDMA:
2641 timeout = adev->sdma_timeout;
2642 break;
2643 default:
2644 timeout = adev->video_timeout;
2645 break;
2646 }
2647
2648 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL,
2649 DRM_SCHED_PRIORITY_COUNT,
2650 ring->num_hw_submission, 0,
2651 timeout, adev->reset_domain->wq,
2652 ring->sched_score, ring->name,
2653 adev->dev);
2654 if (r) {
2655 DRM_ERROR("Failed to create scheduler on ring %s.\n",
2656 ring->name);
2657 return r;
2658 }
2659 r = amdgpu_uvd_entity_init(adev, ring);
2660 if (r) {
2661 DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n",
2662 ring->name);
2663 return r;
2664 }
2665 r = amdgpu_vce_entity_init(adev, ring);
2666 if (r) {
2667 DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n",
2668 ring->name);
2669 return r;
2670 }
2671 }
2672
2673 amdgpu_xcp_update_partition_sched_list(adev);
2674
2675 return 0;
2676}
2677
2678
2679/**
2680 * amdgpu_device_ip_init - run init for hardware IPs
2681 *
2682 * @adev: amdgpu_device pointer
2683 *
2684 * Main initialization pass for hardware IPs. The list of all the hardware
2685 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2686 * are run. sw_init initializes the software state associated with each IP
2687 * and hw_init initializes the hardware associated with each IP.
2688 * Returns 0 on success, negative error code on failure.
2689 */
2690static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2691{
2692 int i, r;
2693
2694 r = amdgpu_ras_init(adev);
2695 if (r)
2696 return r;
2697
2698 for (i = 0; i < adev->num_ip_blocks; i++) {
2699 if (!adev->ip_blocks[i].status.valid)
2700 continue;
2701 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2702 if (r) {
2703 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2704 adev->ip_blocks[i].version->funcs->name, r);
2705 goto init_failed;
2706 }
2707 adev->ip_blocks[i].status.sw = true;
2708
2709 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2710 /* need to do common hw init early so everything is set up for gmc */
2711 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2712 if (r) {
2713 DRM_ERROR("hw_init %d failed %d\n", i, r);
2714 goto init_failed;
2715 }
2716 adev->ip_blocks[i].status.hw = true;
2717 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2718 /* need to do gmc hw init early so we can allocate gpu mem */
2719 /* Try to reserve bad pages early */
2720 if (amdgpu_sriov_vf(adev))
2721 amdgpu_virt_exchange_data(adev);
2722
2723 r = amdgpu_device_mem_scratch_init(adev);
2724 if (r) {
2725 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2726 goto init_failed;
2727 }
2728 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2729 if (r) {
2730 DRM_ERROR("hw_init %d failed %d\n", i, r);
2731 goto init_failed;
2732 }
2733 r = amdgpu_device_wb_init(adev);
2734 if (r) {
2735 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2736 goto init_failed;
2737 }
2738 adev->ip_blocks[i].status.hw = true;
2739
2740 /* right after GMC hw init, we create CSA */
2741 if (adev->gfx.mcbp) {
2742 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2743 AMDGPU_GEM_DOMAIN_VRAM |
2744 AMDGPU_GEM_DOMAIN_GTT,
2745 AMDGPU_CSA_SIZE);
2746 if (r) {
2747 DRM_ERROR("allocate CSA failed %d\n", r);
2748 goto init_failed;
2749 }
2750 }
2751
2752 r = amdgpu_seq64_init(adev);
2753 if (r) {
2754 DRM_ERROR("allocate seq64 failed %d\n", r);
2755 goto init_failed;
2756 }
2757 }
2758 }
2759
2760 if (amdgpu_sriov_vf(adev))
2761 amdgpu_virt_init_data_exchange(adev);
2762
2763 r = amdgpu_ib_pool_init(adev);
2764 if (r) {
2765 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2766 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2767 goto init_failed;
2768 }
2769
2770 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2771 if (r)
2772 goto init_failed;
2773
2774 r = amdgpu_device_ip_hw_init_phase1(adev);
2775 if (r)
2776 goto init_failed;
2777
2778 r = amdgpu_device_fw_loading(adev);
2779 if (r)
2780 goto init_failed;
2781
2782 r = amdgpu_device_ip_hw_init_phase2(adev);
2783 if (r)
2784 goto init_failed;
2785
2786 /*
2787 * retired pages will be loaded from eeprom and reserved here,
2788 * it should be called after amdgpu_device_ip_hw_init_phase2 since
2789 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2790 * for I2C communication which only true at this point.
2791 *
2792 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2793 * failure from bad gpu situation and stop amdgpu init process
2794 * accordingly. For other failed cases, it will still release all
2795 * the resource and print error message, rather than returning one
2796 * negative value to upper level.
2797 *
2798 * Note: theoretically, this should be called before all vram allocations
2799 * to protect retired page from abusing
2800 */
2801 r = amdgpu_ras_recovery_init(adev);
2802 if (r)
2803 goto init_failed;
2804
2805 /**
2806 * In case of XGMI grab extra reference for reset domain for this device
2807 */
2808 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2809 if (amdgpu_xgmi_add_device(adev) == 0) {
2810 if (!amdgpu_sriov_vf(adev)) {
2811 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2812
2813 if (WARN_ON(!hive)) {
2814 r = -ENOENT;
2815 goto init_failed;
2816 }
2817
2818 if (!hive->reset_domain ||
2819 !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2820 r = -ENOENT;
2821 amdgpu_put_xgmi_hive(hive);
2822 goto init_failed;
2823 }
2824
2825 /* Drop the early temporary reset domain we created for device */
2826 amdgpu_reset_put_reset_domain(adev->reset_domain);
2827 adev->reset_domain = hive->reset_domain;
2828 amdgpu_put_xgmi_hive(hive);
2829 }
2830 }
2831 }
2832
2833 r = amdgpu_device_init_schedulers(adev);
2834 if (r)
2835 goto init_failed;
2836
2837 if (adev->mman.buffer_funcs_ring->sched.ready)
2838 amdgpu_ttm_set_buffer_funcs_status(adev, true);
2839
2840 /* Don't init kfd if whole hive need to be reset during init */
2841 if (!adev->gmc.xgmi.pending_reset) {
2842 kgd2kfd_init_zone_device(adev);
2843 amdgpu_amdkfd_device_init(adev);
2844 }
2845
2846 amdgpu_fru_get_product_info(adev);
2847
2848init_failed:
2849
2850 return r;
2851}
2852
2853/**
2854 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2855 *
2856 * @adev: amdgpu_device pointer
2857 *
2858 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
2859 * this function before a GPU reset. If the value is retained after a
2860 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
2861 */
2862static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2863{
2864 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2865}
2866
2867/**
2868 * amdgpu_device_check_vram_lost - check if vram is valid
2869 *
2870 * @adev: amdgpu_device pointer
2871 *
2872 * Checks the reset magic value written to the gart pointer in VRAM.
2873 * The driver calls this after a GPU reset to see if the contents of
2874 * VRAM is lost or now.
2875 * returns true if vram is lost, false if not.
2876 */
2877static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2878{
2879 if (memcmp(adev->gart.ptr, adev->reset_magic,
2880 AMDGPU_RESET_MAGIC_NUM))
2881 return true;
2882
2883 if (!amdgpu_in_reset(adev))
2884 return false;
2885
2886 /*
2887 * For all ASICs with baco/mode1 reset, the VRAM is
2888 * always assumed to be lost.
2889 */
2890 switch (amdgpu_asic_reset_method(adev)) {
2891 case AMD_RESET_METHOD_BACO:
2892 case AMD_RESET_METHOD_MODE1:
2893 return true;
2894 default:
2895 return false;
2896 }
2897}
2898
2899/**
2900 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2901 *
2902 * @adev: amdgpu_device pointer
2903 * @state: clockgating state (gate or ungate)
2904 *
2905 * The list of all the hardware IPs that make up the asic is walked and the
2906 * set_clockgating_state callbacks are run.
2907 * Late initialization pass enabling clockgating for hardware IPs.
2908 * Fini or suspend, pass disabling clockgating for hardware IPs.
2909 * Returns 0 on success, negative error code on failure.
2910 */
2911
2912int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2913 enum amd_clockgating_state state)
2914{
2915 int i, j, r;
2916
2917 if (amdgpu_emu_mode == 1)
2918 return 0;
2919
2920 for (j = 0; j < adev->num_ip_blocks; j++) {
2921 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2922 if (!adev->ip_blocks[i].status.late_initialized)
2923 continue;
2924 /* skip CG for GFX, SDMA on S0ix */
2925 if (adev->in_s0ix &&
2926 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2927 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2928 continue;
2929 /* skip CG for VCE/UVD, it's handled specially */
2930 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2931 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2932 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2933 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2934 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2935 /* enable clockgating to save power */
2936 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2937 state);
2938 if (r) {
2939 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2940 adev->ip_blocks[i].version->funcs->name, r);
2941 return r;
2942 }
2943 }
2944 }
2945
2946 return 0;
2947}
2948
2949int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2950 enum amd_powergating_state state)
2951{
2952 int i, j, r;
2953
2954 if (amdgpu_emu_mode == 1)
2955 return 0;
2956
2957 for (j = 0; j < adev->num_ip_blocks; j++) {
2958 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2959 if (!adev->ip_blocks[i].status.late_initialized)
2960 continue;
2961 /* skip PG for GFX, SDMA on S0ix */
2962 if (adev->in_s0ix &&
2963 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2964 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2965 continue;
2966 /* skip CG for VCE/UVD, it's handled specially */
2967 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2968 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2969 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2970 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2971 adev->ip_blocks[i].version->funcs->set_powergating_state) {
2972 /* enable powergating to save power */
2973 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2974 state);
2975 if (r) {
2976 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2977 adev->ip_blocks[i].version->funcs->name, r);
2978 return r;
2979 }
2980 }
2981 }
2982 return 0;
2983}
2984
2985static int amdgpu_device_enable_mgpu_fan_boost(void)
2986{
2987 struct amdgpu_gpu_instance *gpu_ins;
2988 struct amdgpu_device *adev;
2989 int i, ret = 0;
2990
2991 mutex_lock(&mgpu_info.mutex);
2992
2993 /*
2994 * MGPU fan boost feature should be enabled
2995 * only when there are two or more dGPUs in
2996 * the system
2997 */
2998 if (mgpu_info.num_dgpu < 2)
2999 goto out;
3000
3001 for (i = 0; i < mgpu_info.num_dgpu; i++) {
3002 gpu_ins = &(mgpu_info.gpu_ins[i]);
3003 adev = gpu_ins->adev;
3004 if (!(adev->flags & AMD_IS_APU) &&
3005 !gpu_ins->mgpu_fan_enabled) {
3006 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
3007 if (ret)
3008 break;
3009
3010 gpu_ins->mgpu_fan_enabled = 1;
3011 }
3012 }
3013
3014out:
3015 mutex_unlock(&mgpu_info.mutex);
3016
3017 return ret;
3018}
3019
3020/**
3021 * amdgpu_device_ip_late_init - run late init for hardware IPs
3022 *
3023 * @adev: amdgpu_device pointer
3024 *
3025 * Late initialization pass for hardware IPs. The list of all the hardware
3026 * IPs that make up the asic is walked and the late_init callbacks are run.
3027 * late_init covers any special initialization that an IP requires
3028 * after all of the have been initialized or something that needs to happen
3029 * late in the init process.
3030 * Returns 0 on success, negative error code on failure.
3031 */
3032static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
3033{
3034 struct amdgpu_gpu_instance *gpu_instance;
3035 int i = 0, r;
3036
3037 for (i = 0; i < adev->num_ip_blocks; i++) {
3038 if (!adev->ip_blocks[i].status.hw)
3039 continue;
3040 if (adev->ip_blocks[i].version->funcs->late_init) {
3041 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
3042 if (r) {
3043 DRM_ERROR("late_init of IP block <%s> failed %d\n",
3044 adev->ip_blocks[i].version->funcs->name, r);
3045 return r;
3046 }
3047 }
3048 adev->ip_blocks[i].status.late_initialized = true;
3049 }
3050
3051 r = amdgpu_ras_late_init(adev);
3052 if (r) {
3053 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
3054 return r;
3055 }
3056
3057 amdgpu_ras_set_error_query_ready(adev, true);
3058
3059 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
3060 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
3061
3062 amdgpu_device_fill_reset_magic(adev);
3063
3064 r = amdgpu_device_enable_mgpu_fan_boost();
3065 if (r)
3066 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
3067
3068 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
3069 if (amdgpu_passthrough(adev) &&
3070 ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
3071 adev->asic_type == CHIP_ALDEBARAN))
3072 amdgpu_dpm_handle_passthrough_sbr(adev, true);
3073
3074 if (adev->gmc.xgmi.num_physical_nodes > 1) {
3075 mutex_lock(&mgpu_info.mutex);
3076
3077 /*
3078 * Reset device p-state to low as this was booted with high.
3079 *
3080 * This should be performed only after all devices from the same
3081 * hive get initialized.
3082 *
3083 * However, it's unknown how many device in the hive in advance.
3084 * As this is counted one by one during devices initializations.
3085 *
3086 * So, we wait for all XGMI interlinked devices initialized.
3087 * This may bring some delays as those devices may come from
3088 * different hives. But that should be OK.
3089 */
3090 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
3091 for (i = 0; i < mgpu_info.num_gpu; i++) {
3092 gpu_instance = &(mgpu_info.gpu_ins[i]);
3093 if (gpu_instance->adev->flags & AMD_IS_APU)
3094 continue;
3095
3096 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
3097 AMDGPU_XGMI_PSTATE_MIN);
3098 if (r) {
3099 DRM_ERROR("pstate setting failed (%d).\n", r);
3100 break;
3101 }
3102 }
3103 }
3104
3105 mutex_unlock(&mgpu_info.mutex);
3106 }
3107
3108 return 0;
3109}
3110
3111/**
3112 * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3113 *
3114 * @adev: amdgpu_device pointer
3115 *
3116 * For ASICs need to disable SMC first
3117 */
3118static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
3119{
3120 int i, r;
3121
3122 if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
3123 return;
3124
3125 for (i = 0; i < adev->num_ip_blocks; i++) {
3126 if (!adev->ip_blocks[i].status.hw)
3127 continue;
3128 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3129 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3130 /* XXX handle errors */
3131 if (r) {
3132 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3133 adev->ip_blocks[i].version->funcs->name, r);
3134 }
3135 adev->ip_blocks[i].status.hw = false;
3136 break;
3137 }
3138 }
3139}
3140
3141static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
3142{
3143 int i, r;
3144
3145 for (i = 0; i < adev->num_ip_blocks; i++) {
3146 if (!adev->ip_blocks[i].version->funcs->early_fini)
3147 continue;
3148
3149 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
3150 if (r) {
3151 DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
3152 adev->ip_blocks[i].version->funcs->name, r);
3153 }
3154 }
3155
3156 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3157 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3158
3159 amdgpu_amdkfd_suspend(adev, false);
3160
3161 /* Workaroud for ASICs need to disable SMC first */
3162 amdgpu_device_smu_fini_early(adev);
3163
3164 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3165 if (!adev->ip_blocks[i].status.hw)
3166 continue;
3167
3168 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3169 /* XXX handle errors */
3170 if (r) {
3171 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3172 adev->ip_blocks[i].version->funcs->name, r);
3173 }
3174
3175 adev->ip_blocks[i].status.hw = false;
3176 }
3177
3178 if (amdgpu_sriov_vf(adev)) {
3179 if (amdgpu_virt_release_full_gpu(adev, false))
3180 DRM_ERROR("failed to release exclusive mode on fini\n");
3181 }
3182
3183 return 0;
3184}
3185
3186/**
3187 * amdgpu_device_ip_fini - run fini for hardware IPs
3188 *
3189 * @adev: amdgpu_device pointer
3190 *
3191 * Main teardown pass for hardware IPs. The list of all the hardware
3192 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3193 * are run. hw_fini tears down the hardware associated with each IP
3194 * and sw_fini tears down any software state associated with each IP.
3195 * Returns 0 on success, negative error code on failure.
3196 */
3197static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3198{
3199 int i, r;
3200
3201 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3202 amdgpu_virt_release_ras_err_handler_data(adev);
3203
3204 if (adev->gmc.xgmi.num_physical_nodes > 1)
3205 amdgpu_xgmi_remove_device(adev);
3206
3207 amdgpu_amdkfd_device_fini_sw(adev);
3208
3209 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3210 if (!adev->ip_blocks[i].status.sw)
3211 continue;
3212
3213 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3214 amdgpu_ucode_free_bo(adev);
3215 amdgpu_free_static_csa(&adev->virt.csa_obj);
3216 amdgpu_device_wb_fini(adev);
3217 amdgpu_device_mem_scratch_fini(adev);
3218 amdgpu_ib_pool_fini(adev);
3219 amdgpu_seq64_fini(adev);
3220 }
3221
3222 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
3223 /* XXX handle errors */
3224 if (r) {
3225 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3226 adev->ip_blocks[i].version->funcs->name, r);
3227 }
3228 adev->ip_blocks[i].status.sw = false;
3229 adev->ip_blocks[i].status.valid = false;
3230 }
3231
3232 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3233 if (!adev->ip_blocks[i].status.late_initialized)
3234 continue;
3235 if (adev->ip_blocks[i].version->funcs->late_fini)
3236 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
3237 adev->ip_blocks[i].status.late_initialized = false;
3238 }
3239
3240 amdgpu_ras_fini(adev);
3241
3242 return 0;
3243}
3244
3245/**
3246 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3247 *
3248 * @work: work_struct.
3249 */
3250static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3251{
3252 struct amdgpu_device *adev =
3253 container_of(work, struct amdgpu_device, delayed_init_work.work);
3254 int r;
3255
3256 r = amdgpu_ib_ring_tests(adev);
3257 if (r)
3258 DRM_ERROR("ib ring test failed (%d).\n", r);
3259}
3260
3261static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3262{
3263 struct amdgpu_device *adev =
3264 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3265
3266 WARN_ON_ONCE(adev->gfx.gfx_off_state);
3267 WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3268
3269 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
3270 adev->gfx.gfx_off_state = true;
3271}
3272
3273/**
3274 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3275 *
3276 * @adev: amdgpu_device pointer
3277 *
3278 * Main suspend function for hardware IPs. The list of all the hardware
3279 * IPs that make up the asic is walked, clockgating is disabled and the
3280 * suspend callbacks are run. suspend puts the hardware and software state
3281 * in each IP into a state suitable for suspend.
3282 * Returns 0 on success, negative error code on failure.
3283 */
3284static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3285{
3286 int i, r;
3287
3288 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3289 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3290
3291 /*
3292 * Per PMFW team's suggestion, driver needs to handle gfxoff
3293 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3294 * scenario. Add the missing df cstate disablement here.
3295 */
3296 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3297 dev_warn(adev->dev, "Failed to disallow df cstate");
3298
3299 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3300 if (!adev->ip_blocks[i].status.valid)
3301 continue;
3302
3303 /* displays are handled separately */
3304 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3305 continue;
3306
3307 /* XXX handle errors */
3308 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3309 /* XXX handle errors */
3310 if (r) {
3311 DRM_ERROR("suspend of IP block <%s> failed %d\n",
3312 adev->ip_blocks[i].version->funcs->name, r);
3313 return r;
3314 }
3315
3316 adev->ip_blocks[i].status.hw = false;
3317 }
3318
3319 return 0;
3320}
3321
3322/**
3323 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3324 *
3325 * @adev: amdgpu_device pointer
3326 *
3327 * Main suspend function for hardware IPs. The list of all the hardware
3328 * IPs that make up the asic is walked, clockgating is disabled and the
3329 * suspend callbacks are run. suspend puts the hardware and software state
3330 * in each IP into a state suitable for suspend.
3331 * Returns 0 on success, negative error code on failure.
3332 */
3333static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3334{
3335 int i, r;
3336
3337 if (adev->in_s0ix)
3338 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3339
3340 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3341 if (!adev->ip_blocks[i].status.valid)
3342 continue;
3343 /* displays are handled in phase1 */
3344 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3345 continue;
3346 /* PSP lost connection when err_event_athub occurs */
3347 if (amdgpu_ras_intr_triggered() &&
3348 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3349 adev->ip_blocks[i].status.hw = false;
3350 continue;
3351 }
3352
3353 /* skip unnecessary suspend if we do not initialize them yet */
3354 if (adev->gmc.xgmi.pending_reset &&
3355 !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3356 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3357 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3358 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3359 adev->ip_blocks[i].status.hw = false;
3360 continue;
3361 }
3362
3363 /* skip suspend of gfx/mes and psp for S0ix
3364 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3365 * like at runtime. PSP is also part of the always on hardware
3366 * so no need to suspend it.
3367 */
3368 if (adev->in_s0ix &&
3369 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3370 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3371 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3372 continue;
3373
3374 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3375 if (adev->in_s0ix &&
3376 (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3377 IP_VERSION(5, 0, 0)) &&
3378 (adev->ip_blocks[i].version->type ==
3379 AMD_IP_BLOCK_TYPE_SDMA))
3380 continue;
3381
3382 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3383 * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3384 * from this location and RLC Autoload automatically also gets loaded
3385 * from here based on PMFW -> PSP message during re-init sequence.
3386 * Therefore, the psp suspend & resume should be skipped to avoid destroy
3387 * the TMR and reload FWs again for IMU enabled APU ASICs.
3388 */
3389 if (amdgpu_in_reset(adev) &&
3390 (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3391 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3392 continue;
3393
3394 /* XXX handle errors */
3395 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3396 /* XXX handle errors */
3397 if (r) {
3398 DRM_ERROR("suspend of IP block <%s> failed %d\n",
3399 adev->ip_blocks[i].version->funcs->name, r);
3400 }
3401 adev->ip_blocks[i].status.hw = false;
3402 /* handle putting the SMC in the appropriate state */
3403 if (!amdgpu_sriov_vf(adev)) {
3404 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3405 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3406 if (r) {
3407 DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3408 adev->mp1_state, r);
3409 return r;
3410 }
3411 }
3412 }
3413 }
3414
3415 return 0;
3416}
3417
3418/**
3419 * amdgpu_device_ip_suspend - run suspend for hardware IPs
3420 *
3421 * @adev: amdgpu_device pointer
3422 *
3423 * Main suspend function for hardware IPs. The list of all the hardware
3424 * IPs that make up the asic is walked, clockgating is disabled and the
3425 * suspend callbacks are run. suspend puts the hardware and software state
3426 * in each IP into a state suitable for suspend.
3427 * Returns 0 on success, negative error code on failure.
3428 */
3429int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3430{
3431 int r;
3432
3433 if (amdgpu_sriov_vf(adev)) {
3434 amdgpu_virt_fini_data_exchange(adev);
3435 amdgpu_virt_request_full_gpu(adev, false);
3436 }
3437
3438 amdgpu_ttm_set_buffer_funcs_status(adev, false);
3439
3440 r = amdgpu_device_ip_suspend_phase1(adev);
3441 if (r)
3442 return r;
3443 r = amdgpu_device_ip_suspend_phase2(adev);
3444
3445 if (amdgpu_sriov_vf(adev))
3446 amdgpu_virt_release_full_gpu(adev, false);
3447
3448 return r;
3449}
3450
3451static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3452{
3453 int i, r;
3454
3455 static enum amd_ip_block_type ip_order[] = {
3456 AMD_IP_BLOCK_TYPE_COMMON,
3457 AMD_IP_BLOCK_TYPE_GMC,
3458 AMD_IP_BLOCK_TYPE_PSP,
3459 AMD_IP_BLOCK_TYPE_IH,
3460 };
3461
3462 for (i = 0; i < adev->num_ip_blocks; i++) {
3463 int j;
3464 struct amdgpu_ip_block *block;
3465
3466 block = &adev->ip_blocks[i];
3467 block->status.hw = false;
3468
3469 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3470
3471 if (block->version->type != ip_order[j] ||
3472 !block->status.valid)
3473 continue;
3474
3475 r = block->version->funcs->hw_init(adev);
3476 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3477 if (r)
3478 return r;
3479 block->status.hw = true;
3480 }
3481 }
3482
3483 return 0;
3484}
3485
3486static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3487{
3488 int i, r;
3489
3490 static enum amd_ip_block_type ip_order[] = {
3491 AMD_IP_BLOCK_TYPE_SMC,
3492 AMD_IP_BLOCK_TYPE_DCE,
3493 AMD_IP_BLOCK_TYPE_GFX,
3494 AMD_IP_BLOCK_TYPE_SDMA,
3495 AMD_IP_BLOCK_TYPE_MES,
3496 AMD_IP_BLOCK_TYPE_UVD,
3497 AMD_IP_BLOCK_TYPE_VCE,
3498 AMD_IP_BLOCK_TYPE_VCN,
3499 AMD_IP_BLOCK_TYPE_JPEG
3500 };
3501
3502 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3503 int j;
3504 struct amdgpu_ip_block *block;
3505
3506 for (j = 0; j < adev->num_ip_blocks; j++) {
3507 block = &adev->ip_blocks[j];
3508
3509 if (block->version->type != ip_order[i] ||
3510 !block->status.valid ||
3511 block->status.hw)
3512 continue;
3513
3514 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3515 r = block->version->funcs->resume(adev);
3516 else
3517 r = block->version->funcs->hw_init(adev);
3518
3519 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3520 if (r)
3521 return r;
3522 block->status.hw = true;
3523 }
3524 }
3525
3526 return 0;
3527}
3528
3529/**
3530 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3531 *
3532 * @adev: amdgpu_device pointer
3533 *
3534 * First resume function for hardware IPs. The list of all the hardware
3535 * IPs that make up the asic is walked and the resume callbacks are run for
3536 * COMMON, GMC, and IH. resume puts the hardware into a functional state
3537 * after a suspend and updates the software state as necessary. This
3538 * function is also used for restoring the GPU after a GPU reset.
3539 * Returns 0 on success, negative error code on failure.
3540 */
3541static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3542{
3543 int i, r;
3544
3545 for (i = 0; i < adev->num_ip_blocks; i++) {
3546 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3547 continue;
3548 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3549 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3550 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3551 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3552
3553 r = adev->ip_blocks[i].version->funcs->resume(adev);
3554 if (r) {
3555 DRM_ERROR("resume of IP block <%s> failed %d\n",
3556 adev->ip_blocks[i].version->funcs->name, r);
3557 return r;
3558 }
3559 adev->ip_blocks[i].status.hw = true;
3560 }
3561 }
3562
3563 return 0;
3564}
3565
3566/**
3567 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3568 *
3569 * @adev: amdgpu_device pointer
3570 *
3571 * First resume function for hardware IPs. The list of all the hardware
3572 * IPs that make up the asic is walked and the resume callbacks are run for
3573 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
3574 * functional state after a suspend and updates the software state as
3575 * necessary. This function is also used for restoring the GPU after a GPU
3576 * reset.
3577 * Returns 0 on success, negative error code on failure.
3578 */
3579static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3580{
3581 int i, r;
3582
3583 for (i = 0; i < adev->num_ip_blocks; i++) {
3584 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3585 continue;
3586 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3587 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3588 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3589 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3590 continue;
3591 r = adev->ip_blocks[i].version->funcs->resume(adev);
3592 if (r) {
3593 DRM_ERROR("resume of IP block <%s> failed %d\n",
3594 adev->ip_blocks[i].version->funcs->name, r);
3595 return r;
3596 }
3597 adev->ip_blocks[i].status.hw = true;
3598 }
3599
3600 return 0;
3601}
3602
3603/**
3604 * amdgpu_device_ip_resume - run resume for hardware IPs
3605 *
3606 * @adev: amdgpu_device pointer
3607 *
3608 * Main resume function for hardware IPs. The hardware IPs
3609 * are split into two resume functions because they are
3610 * also used in recovering from a GPU reset and some additional
3611 * steps need to be take between them. In this case (S3/S4) they are
3612 * run sequentially.
3613 * Returns 0 on success, negative error code on failure.
3614 */
3615static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3616{
3617 int r;
3618
3619 r = amdgpu_device_ip_resume_phase1(adev);
3620 if (r)
3621 return r;
3622
3623 r = amdgpu_device_fw_loading(adev);
3624 if (r)
3625 return r;
3626
3627 r = amdgpu_device_ip_resume_phase2(adev);
3628
3629 if (adev->mman.buffer_funcs_ring->sched.ready)
3630 amdgpu_ttm_set_buffer_funcs_status(adev, true);
3631
3632 return r;
3633}
3634
3635/**
3636 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3637 *
3638 * @adev: amdgpu_device pointer
3639 *
3640 * Query the VBIOS data tables to determine if the board supports SR-IOV.
3641 */
3642static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3643{
3644 if (amdgpu_sriov_vf(adev)) {
3645 if (adev->is_atom_fw) {
3646 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3647 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3648 } else {
3649 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3650 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3651 }
3652
3653 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3654 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3655 }
3656}
3657
3658/**
3659 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3660 *
3661 * @asic_type: AMD asic type
3662 *
3663 * Check if there is DC (new modesetting infrastructre) support for an asic.
3664 * returns true if DC has support, false if not.
3665 */
3666bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3667{
3668 switch (asic_type) {
3669#ifdef CONFIG_DRM_AMDGPU_SI
3670 case CHIP_HAINAN:
3671#endif
3672 case CHIP_TOPAZ:
3673 /* chips with no display hardware */
3674 return false;
3675#if defined(CONFIG_DRM_AMD_DC)
3676 case CHIP_TAHITI:
3677 case CHIP_PITCAIRN:
3678 case CHIP_VERDE:
3679 case CHIP_OLAND:
3680 /*
3681 * We have systems in the wild with these ASICs that require
3682 * LVDS and VGA support which is not supported with DC.
3683 *
3684 * Fallback to the non-DC driver here by default so as not to
3685 * cause regressions.
3686 */
3687#if defined(CONFIG_DRM_AMD_DC_SI)
3688 return amdgpu_dc > 0;
3689#else
3690 return false;
3691#endif
3692 case CHIP_BONAIRE:
3693 case CHIP_KAVERI:
3694 case CHIP_KABINI:
3695 case CHIP_MULLINS:
3696 /*
3697 * We have systems in the wild with these ASICs that require
3698 * VGA support which is not supported with DC.
3699 *
3700 * Fallback to the non-DC driver here by default so as not to
3701 * cause regressions.
3702 */
3703 return amdgpu_dc > 0;
3704 default:
3705 return amdgpu_dc != 0;
3706#else
3707 default:
3708 if (amdgpu_dc > 0)
3709 DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3710 return false;
3711#endif
3712 }
3713}
3714
3715/**
3716 * amdgpu_device_has_dc_support - check if dc is supported
3717 *
3718 * @adev: amdgpu_device pointer
3719 *
3720 * Returns true for supported, false for not supported
3721 */
3722bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3723{
3724 if (adev->enable_virtual_display ||
3725 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3726 return false;
3727
3728 return amdgpu_device_asic_has_dc_support(adev->asic_type);
3729}
3730
3731static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3732{
3733 struct amdgpu_device *adev =
3734 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3735 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3736
3737 /* It's a bug to not have a hive within this function */
3738 if (WARN_ON(!hive))
3739 return;
3740
3741 /*
3742 * Use task barrier to synchronize all xgmi reset works across the
3743 * hive. task_barrier_enter and task_barrier_exit will block
3744 * until all the threads running the xgmi reset works reach
3745 * those points. task_barrier_full will do both blocks.
3746 */
3747 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3748
3749 task_barrier_enter(&hive->tb);
3750 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3751
3752 if (adev->asic_reset_res)
3753 goto fail;
3754
3755 task_barrier_exit(&hive->tb);
3756 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3757
3758 if (adev->asic_reset_res)
3759 goto fail;
3760
3761 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
3762 } else {
3763
3764 task_barrier_full(&hive->tb);
3765 adev->asic_reset_res = amdgpu_asic_reset(adev);
3766 }
3767
3768fail:
3769 if (adev->asic_reset_res)
3770 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3771 adev->asic_reset_res, adev_to_drm(adev)->unique);
3772 amdgpu_put_xgmi_hive(hive);
3773}
3774
3775static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3776{
3777 char *input = amdgpu_lockup_timeout;
3778 char *timeout_setting = NULL;
3779 int index = 0;
3780 long timeout;
3781 int ret = 0;
3782
3783 /*
3784 * By default timeout for non compute jobs is 10000
3785 * and 60000 for compute jobs.
3786 * In SR-IOV or passthrough mode, timeout for compute
3787 * jobs are 60000 by default.
3788 */
3789 adev->gfx_timeout = msecs_to_jiffies(10000);
3790 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3791 if (amdgpu_sriov_vf(adev))
3792 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3793 msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3794 else
3795 adev->compute_timeout = msecs_to_jiffies(60000);
3796
3797 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3798 while ((timeout_setting = strsep(&input, ",")) &&
3799 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3800 ret = kstrtol(timeout_setting, 0, &timeout);
3801 if (ret)
3802 return ret;
3803
3804 if (timeout == 0) {
3805 index++;
3806 continue;
3807 } else if (timeout < 0) {
3808 timeout = MAX_SCHEDULE_TIMEOUT;
3809 dev_warn(adev->dev, "lockup timeout disabled");
3810 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3811 } else {
3812 timeout = msecs_to_jiffies(timeout);
3813 }
3814
3815 switch (index++) {
3816 case 0:
3817 adev->gfx_timeout = timeout;
3818 break;
3819 case 1:
3820 adev->compute_timeout = timeout;
3821 break;
3822 case 2:
3823 adev->sdma_timeout = timeout;
3824 break;
3825 case 3:
3826 adev->video_timeout = timeout;
3827 break;
3828 default:
3829 break;
3830 }
3831 }
3832 /*
3833 * There is only one value specified and
3834 * it should apply to all non-compute jobs.
3835 */
3836 if (index == 1) {
3837 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3838 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3839 adev->compute_timeout = adev->gfx_timeout;
3840 }
3841 }
3842
3843 return ret;
3844}
3845
3846/**
3847 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3848 *
3849 * @adev: amdgpu_device pointer
3850 *
3851 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3852 */
3853static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3854{
3855 struct iommu_domain *domain;
3856
3857 domain = iommu_get_domain_for_dev(adev->dev);
3858 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3859 adev->ram_is_direct_mapped = true;
3860}
3861
3862static const struct attribute *amdgpu_dev_attributes[] = {
3863 &dev_attr_pcie_replay_count.attr,
3864 NULL
3865};
3866
3867static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3868{
3869 if (amdgpu_mcbp == 1)
3870 adev->gfx.mcbp = true;
3871 else if (amdgpu_mcbp == 0)
3872 adev->gfx.mcbp = false;
3873
3874 if (amdgpu_sriov_vf(adev))
3875 adev->gfx.mcbp = true;
3876
3877 if (adev->gfx.mcbp)
3878 DRM_INFO("MCBP is enabled\n");
3879}
3880
3881/**
3882 * amdgpu_device_init - initialize the driver
3883 *
3884 * @adev: amdgpu_device pointer
3885 * @flags: driver flags
3886 *
3887 * Initializes the driver info and hw (all asics).
3888 * Returns 0 for success or an error on failure.
3889 * Called at driver startup.
3890 */
3891int amdgpu_device_init(struct amdgpu_device *adev,
3892 uint32_t flags)
3893{
3894 struct drm_device *ddev = adev_to_drm(adev);
3895 struct pci_dev *pdev = adev->pdev;
3896 int r, i;
3897 bool px = false;
3898 u32 max_MBps;
3899 int tmp;
3900
3901 adev->shutdown = false;
3902 adev->flags = flags;
3903
3904 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3905 adev->asic_type = amdgpu_force_asic_type;
3906 else
3907 adev->asic_type = flags & AMD_ASIC_MASK;
3908
3909 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3910 if (amdgpu_emu_mode == 1)
3911 adev->usec_timeout *= 10;
3912 adev->gmc.gart_size = 512 * 1024 * 1024;
3913 adev->accel_working = false;
3914 adev->num_rings = 0;
3915 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3916 adev->mman.buffer_funcs = NULL;
3917 adev->mman.buffer_funcs_ring = NULL;
3918 adev->vm_manager.vm_pte_funcs = NULL;
3919 adev->vm_manager.vm_pte_num_scheds = 0;
3920 adev->gmc.gmc_funcs = NULL;
3921 adev->harvest_ip_mask = 0x0;
3922 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3923 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3924
3925 adev->smc_rreg = &amdgpu_invalid_rreg;
3926 adev->smc_wreg = &amdgpu_invalid_wreg;
3927 adev->pcie_rreg = &amdgpu_invalid_rreg;
3928 adev->pcie_wreg = &amdgpu_invalid_wreg;
3929 adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
3930 adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
3931 adev->pciep_rreg = &amdgpu_invalid_rreg;
3932 adev->pciep_wreg = &amdgpu_invalid_wreg;
3933 adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3934 adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3935 adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
3936 adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
3937 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3938 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3939 adev->didt_rreg = &amdgpu_invalid_rreg;
3940 adev->didt_wreg = &amdgpu_invalid_wreg;
3941 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3942 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3943 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3944 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3945
3946 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3947 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3948 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3949
3950 /* mutex initialization are all done here so we
3951 * can recall function without having locking issues
3952 */
3953 mutex_init(&adev->firmware.mutex);
3954 mutex_init(&adev->pm.mutex);
3955 mutex_init(&adev->gfx.gpu_clock_mutex);
3956 mutex_init(&adev->srbm_mutex);
3957 mutex_init(&adev->gfx.pipe_reserve_mutex);
3958 mutex_init(&adev->gfx.gfx_off_mutex);
3959 mutex_init(&adev->gfx.partition_mutex);
3960 mutex_init(&adev->grbm_idx_mutex);
3961 mutex_init(&adev->mn_lock);
3962 mutex_init(&adev->virt.vf_errors.lock);
3963 hash_init(adev->mn_hash);
3964 mutex_init(&adev->psp.mutex);
3965 mutex_init(&adev->notifier_lock);
3966 mutex_init(&adev->pm.stable_pstate_ctx_lock);
3967 mutex_init(&adev->benchmark_mutex);
3968
3969 amdgpu_device_init_apu_flags(adev);
3970
3971 r = amdgpu_device_check_arguments(adev);
3972 if (r)
3973 return r;
3974
3975 spin_lock_init(&adev->mmio_idx_lock);
3976 spin_lock_init(&adev->smc_idx_lock);
3977 spin_lock_init(&adev->pcie_idx_lock);
3978 spin_lock_init(&adev->uvd_ctx_idx_lock);
3979 spin_lock_init(&adev->didt_idx_lock);
3980 spin_lock_init(&adev->gc_cac_idx_lock);
3981 spin_lock_init(&adev->se_cac_idx_lock);
3982 spin_lock_init(&adev->audio_endpt_idx_lock);
3983 spin_lock_init(&adev->mm_stats.lock);
3984
3985 INIT_LIST_HEAD(&adev->shadow_list);
3986 mutex_init(&adev->shadow_list_lock);
3987
3988 INIT_LIST_HEAD(&adev->reset_list);
3989
3990 INIT_LIST_HEAD(&adev->ras_list);
3991
3992 INIT_LIST_HEAD(&adev->pm.od_kobj_list);
3993
3994 INIT_DELAYED_WORK(&adev->delayed_init_work,
3995 amdgpu_device_delayed_init_work_handler);
3996 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3997 amdgpu_device_delay_enable_gfx_off);
3998
3999 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
4000
4001 adev->gfx.gfx_off_req_count = 1;
4002 adev->gfx.gfx_off_residency = 0;
4003 adev->gfx.gfx_off_entrycount = 0;
4004 adev->pm.ac_power = power_supply_is_system_supplied() > 0;
4005
4006 atomic_set(&adev->throttling_logging_enabled, 1);
4007 /*
4008 * If throttling continues, logging will be performed every minute
4009 * to avoid log flooding. "-1" is subtracted since the thermal
4010 * throttling interrupt comes every second. Thus, the total logging
4011 * interval is 59 seconds(retelimited printk interval) + 1(waiting
4012 * for throttling interrupt) = 60 seconds.
4013 */
4014 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
4015 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
4016
4017 /* Registers mapping */
4018 /* TODO: block userspace mapping of io register */
4019 if (adev->asic_type >= CHIP_BONAIRE) {
4020 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
4021 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
4022 } else {
4023 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
4024 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
4025 }
4026
4027 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
4028 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
4029
4030 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
4031 if (!adev->rmmio)
4032 return -ENOMEM;
4033
4034 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
4035 DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
4036
4037 /*
4038 * Reset domain needs to be present early, before XGMI hive discovered
4039 * (if any) and intitialized to use reset sem and in_gpu reset flag
4040 * early on during init and before calling to RREG32.
4041 */
4042 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
4043 if (!adev->reset_domain)
4044 return -ENOMEM;
4045
4046 /* detect hw virtualization here */
4047 amdgpu_detect_virtualization(adev);
4048
4049 amdgpu_device_get_pcie_info(adev);
4050
4051 r = amdgpu_device_get_job_timeout_settings(adev);
4052 if (r) {
4053 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
4054 return r;
4055 }
4056
4057 amdgpu_device_set_mcbp(adev);
4058
4059 /* early init functions */
4060 r = amdgpu_device_ip_early_init(adev);
4061 if (r)
4062 return r;
4063
4064 /* Get rid of things like offb */
4065 r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
4066 if (r)
4067 return r;
4068
4069 /* Enable TMZ based on IP_VERSION */
4070 amdgpu_gmc_tmz_set(adev);
4071
4072 amdgpu_gmc_noretry_set(adev);
4073 /* Need to get xgmi info early to decide the reset behavior*/
4074 if (adev->gmc.xgmi.supported) {
4075 r = adev->gfxhub.funcs->get_xgmi_info(adev);
4076 if (r)
4077 return r;
4078 }
4079
4080 /* enable PCIE atomic ops */
4081 if (amdgpu_sriov_vf(adev)) {
4082 if (adev->virt.fw_reserve.p_pf2vf)
4083 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
4084 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
4085 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4086 /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4087 * internal path natively support atomics, set have_atomics_support to true.
4088 */
4089 } else if ((adev->flags & AMD_IS_APU) &&
4090 (amdgpu_ip_version(adev, GC_HWIP, 0) >
4091 IP_VERSION(9, 0, 0))) {
4092 adev->have_atomics_support = true;
4093 } else {
4094 adev->have_atomics_support =
4095 !pci_enable_atomic_ops_to_root(adev->pdev,
4096 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4097 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4098 }
4099
4100 if (!adev->have_atomics_support)
4101 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4102
4103 /* doorbell bar mapping and doorbell index init*/
4104 amdgpu_doorbell_init(adev);
4105
4106 if (amdgpu_emu_mode == 1) {
4107 /* post the asic on emulation mode */
4108 emu_soc_asic_init(adev);
4109 goto fence_driver_init;
4110 }
4111
4112 amdgpu_reset_init(adev);
4113
4114 /* detect if we are with an SRIOV vbios */
4115 if (adev->bios)
4116 amdgpu_device_detect_sriov_bios(adev);
4117
4118 /* check if we need to reset the asic
4119 * E.g., driver was not cleanly unloaded previously, etc.
4120 */
4121 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4122 if (adev->gmc.xgmi.num_physical_nodes) {
4123 dev_info(adev->dev, "Pending hive reset.\n");
4124 adev->gmc.xgmi.pending_reset = true;
4125 /* Only need to init necessary block for SMU to handle the reset */
4126 for (i = 0; i < adev->num_ip_blocks; i++) {
4127 if (!adev->ip_blocks[i].status.valid)
4128 continue;
4129 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
4130 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
4131 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
4132 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
4133 DRM_DEBUG("IP %s disabled for hw_init.\n",
4134 adev->ip_blocks[i].version->funcs->name);
4135 adev->ip_blocks[i].status.hw = true;
4136 }
4137 }
4138 } else if (amdgpu_ip_version(adev, MP1_HWIP, 0) == IP_VERSION(13, 0, 10) &&
4139 !amdgpu_device_has_display_hardware(adev)) {
4140 r = psp_gpu_reset(adev);
4141 } else {
4142 tmp = amdgpu_reset_method;
4143 /* It should do a default reset when loading or reloading the driver,
4144 * regardless of the module parameter reset_method.
4145 */
4146 amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4147 r = amdgpu_asic_reset(adev);
4148 amdgpu_reset_method = tmp;
4149 }
4150
4151 if (r) {
4152 dev_err(adev->dev, "asic reset on init failed\n");
4153 goto failed;
4154 }
4155 }
4156
4157 /* Post card if necessary */
4158 if (amdgpu_device_need_post(adev)) {
4159 if (!adev->bios) {
4160 dev_err(adev->dev, "no vBIOS found\n");
4161 r = -EINVAL;
4162 goto failed;
4163 }
4164 DRM_INFO("GPU posting now...\n");
4165 r = amdgpu_device_asic_init(adev);
4166 if (r) {
4167 dev_err(adev->dev, "gpu post error!\n");
4168 goto failed;
4169 }
4170 }
4171
4172 if (adev->bios) {
4173 if (adev->is_atom_fw) {
4174 /* Initialize clocks */
4175 r = amdgpu_atomfirmware_get_clock_info(adev);
4176 if (r) {
4177 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4178 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4179 goto failed;
4180 }
4181 } else {
4182 /* Initialize clocks */
4183 r = amdgpu_atombios_get_clock_info(adev);
4184 if (r) {
4185 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4186 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4187 goto failed;
4188 }
4189 /* init i2c buses */
4190 if (!amdgpu_device_has_dc_support(adev))
4191 amdgpu_atombios_i2c_init(adev);
4192 }
4193 }
4194
4195fence_driver_init:
4196 /* Fence driver */
4197 r = amdgpu_fence_driver_sw_init(adev);
4198 if (r) {
4199 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4200 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4201 goto failed;
4202 }
4203
4204 /* init the mode config */
4205 drm_mode_config_init(adev_to_drm(adev));
4206
4207 r = amdgpu_device_ip_init(adev);
4208 if (r) {
4209 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4210 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4211 goto release_ras_con;
4212 }
4213
4214 amdgpu_fence_driver_hw_init(adev);
4215
4216 dev_info(adev->dev,
4217 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4218 adev->gfx.config.max_shader_engines,
4219 adev->gfx.config.max_sh_per_se,
4220 adev->gfx.config.max_cu_per_sh,
4221 adev->gfx.cu_info.number);
4222
4223 adev->accel_working = true;
4224
4225 amdgpu_vm_check_compute_bug(adev);
4226
4227 /* Initialize the buffer migration limit. */
4228 if (amdgpu_moverate >= 0)
4229 max_MBps = amdgpu_moverate;
4230 else
4231 max_MBps = 8; /* Allow 8 MB/s. */
4232 /* Get a log2 for easy divisions. */
4233 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4234
4235 /*
4236 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4237 * Otherwise the mgpu fan boost feature will be skipped due to the
4238 * gpu instance is counted less.
4239 */
4240 amdgpu_register_gpu_instance(adev);
4241
4242 /* enable clockgating, etc. after ib tests, etc. since some blocks require
4243 * explicit gating rather than handling it automatically.
4244 */
4245 if (!adev->gmc.xgmi.pending_reset) {
4246 r = amdgpu_device_ip_late_init(adev);
4247 if (r) {
4248 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4249 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4250 goto release_ras_con;
4251 }
4252 /* must succeed. */
4253 amdgpu_ras_resume(adev);
4254 queue_delayed_work(system_wq, &adev->delayed_init_work,
4255 msecs_to_jiffies(AMDGPU_RESUME_MS));
4256 }
4257
4258 if (amdgpu_sriov_vf(adev)) {
4259 amdgpu_virt_release_full_gpu(adev, true);
4260 flush_delayed_work(&adev->delayed_init_work);
4261 }
4262
4263 /*
4264 * Place those sysfs registering after `late_init`. As some of those
4265 * operations performed in `late_init` might affect the sysfs
4266 * interfaces creating.
4267 */
4268 r = amdgpu_atombios_sysfs_init(adev);
4269 if (r)
4270 drm_err(&adev->ddev,
4271 "registering atombios sysfs failed (%d).\n", r);
4272
4273 r = amdgpu_pm_sysfs_init(adev);
4274 if (r)
4275 DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4276
4277 r = amdgpu_ucode_sysfs_init(adev);
4278 if (r) {
4279 adev->ucode_sysfs_en = false;
4280 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4281 } else
4282 adev->ucode_sysfs_en = true;
4283
4284 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4285 if (r)
4286 dev_err(adev->dev, "Could not create amdgpu device attr\n");
4287
4288 r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
4289 if (r)
4290 dev_err(adev->dev,
4291 "Could not create amdgpu board attributes\n");
4292
4293 amdgpu_fru_sysfs_init(adev);
4294 amdgpu_reg_state_sysfs_init(adev);
4295
4296 if (IS_ENABLED(CONFIG_PERF_EVENTS))
4297 r = amdgpu_pmu_init(adev);
4298 if (r)
4299 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4300
4301 /* Have stored pci confspace at hand for restore in sudden PCI error */
4302 if (amdgpu_device_cache_pci_state(adev->pdev))
4303 pci_restore_state(pdev);
4304
4305 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4306 /* this will fail for cards that aren't VGA class devices, just
4307 * ignore it
4308 */
4309 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4310 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4311
4312 px = amdgpu_device_supports_px(ddev);
4313
4314 if (px || (!dev_is_removable(&adev->pdev->dev) &&
4315 apple_gmux_detect(NULL, NULL)))
4316 vga_switcheroo_register_client(adev->pdev,
4317 &amdgpu_switcheroo_ops, px);
4318
4319 if (px)
4320 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4321
4322 if (adev->gmc.xgmi.pending_reset)
4323 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
4324 msecs_to_jiffies(AMDGPU_RESUME_MS));
4325
4326 amdgpu_device_check_iommu_direct_map(adev);
4327
4328 return 0;
4329
4330release_ras_con:
4331 if (amdgpu_sriov_vf(adev))
4332 amdgpu_virt_release_full_gpu(adev, true);
4333
4334 /* failed in exclusive mode due to timeout */
4335 if (amdgpu_sriov_vf(adev) &&
4336 !amdgpu_sriov_runtime(adev) &&
4337 amdgpu_virt_mmio_blocked(adev) &&
4338 !amdgpu_virt_wait_reset(adev)) {
4339 dev_err(adev->dev, "VF exclusive mode timeout\n");
4340 /* Don't send request since VF is inactive. */
4341 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4342 adev->virt.ops = NULL;
4343 r = -EAGAIN;
4344 }
4345 amdgpu_release_ras_context(adev);
4346
4347failed:
4348 amdgpu_vf_error_trans_all(adev);
4349
4350 return r;
4351}
4352
4353static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4354{
4355
4356 /* Clear all CPU mappings pointing to this device */
4357 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4358
4359 /* Unmap all mapped bars - Doorbell, registers and VRAM */
4360 amdgpu_doorbell_fini(adev);
4361
4362 iounmap(adev->rmmio);
4363 adev->rmmio = NULL;
4364 if (adev->mman.aper_base_kaddr)
4365 iounmap(adev->mman.aper_base_kaddr);
4366 adev->mman.aper_base_kaddr = NULL;
4367
4368 /* Memory manager related */
4369 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4370 arch_phys_wc_del(adev->gmc.vram_mtrr);
4371 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4372 }
4373}
4374
4375/**
4376 * amdgpu_device_fini_hw - tear down the driver
4377 *
4378 * @adev: amdgpu_device pointer
4379 *
4380 * Tear down the driver info (all asics).
4381 * Called at driver shutdown.
4382 */
4383void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4384{
4385 dev_info(adev->dev, "amdgpu: finishing device.\n");
4386 flush_delayed_work(&adev->delayed_init_work);
4387 adev->shutdown = true;
4388
4389 /* make sure IB test finished before entering exclusive mode
4390 * to avoid preemption on IB test
4391 */
4392 if (amdgpu_sriov_vf(adev)) {
4393 amdgpu_virt_request_full_gpu(adev, false);
4394 amdgpu_virt_fini_data_exchange(adev);
4395 }
4396
4397 /* disable all interrupts */
4398 amdgpu_irq_disable_all(adev);
4399 if (adev->mode_info.mode_config_initialized) {
4400 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4401 drm_helper_force_disable_all(adev_to_drm(adev));
4402 else
4403 drm_atomic_helper_shutdown(adev_to_drm(adev));
4404 }
4405 amdgpu_fence_driver_hw_fini(adev);
4406
4407 if (adev->mman.initialized)
4408 drain_workqueue(adev->mman.bdev.wq);
4409
4410 if (adev->pm.sysfs_initialized)
4411 amdgpu_pm_sysfs_fini(adev);
4412 if (adev->ucode_sysfs_en)
4413 amdgpu_ucode_sysfs_fini(adev);
4414 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4415 amdgpu_fru_sysfs_fini(adev);
4416
4417 amdgpu_reg_state_sysfs_fini(adev);
4418
4419 /* disable ras feature must before hw fini */
4420 amdgpu_ras_pre_fini(adev);
4421
4422 amdgpu_ttm_set_buffer_funcs_status(adev, false);
4423
4424 amdgpu_device_ip_fini_early(adev);
4425
4426 amdgpu_irq_fini_hw(adev);
4427
4428 if (adev->mman.initialized)
4429 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4430
4431 amdgpu_gart_dummy_page_fini(adev);
4432
4433 if (drm_dev_is_unplugged(adev_to_drm(adev)))
4434 amdgpu_device_unmap_mmio(adev);
4435
4436}
4437
4438void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4439{
4440 int idx;
4441 bool px;
4442
4443 amdgpu_fence_driver_sw_fini(adev);
4444 amdgpu_device_ip_fini(adev);
4445 amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4446 adev->accel_working = false;
4447 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4448
4449 amdgpu_reset_fini(adev);
4450
4451 /* free i2c buses */
4452 if (!amdgpu_device_has_dc_support(adev))
4453 amdgpu_i2c_fini(adev);
4454
4455 if (amdgpu_emu_mode != 1)
4456 amdgpu_atombios_fini(adev);
4457
4458 kfree(adev->bios);
4459 adev->bios = NULL;
4460
4461 kfree(adev->fru_info);
4462 adev->fru_info = NULL;
4463
4464 px = amdgpu_device_supports_px(adev_to_drm(adev));
4465
4466 if (px || (!dev_is_removable(&adev->pdev->dev) &&
4467 apple_gmux_detect(NULL, NULL)))
4468 vga_switcheroo_unregister_client(adev->pdev);
4469
4470 if (px)
4471 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4472
4473 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4474 vga_client_unregister(adev->pdev);
4475
4476 if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4477
4478 iounmap(adev->rmmio);
4479 adev->rmmio = NULL;
4480 amdgpu_doorbell_fini(adev);
4481 drm_dev_exit(idx);
4482 }
4483
4484 if (IS_ENABLED(CONFIG_PERF_EVENTS))
4485 amdgpu_pmu_fini(adev);
4486 if (adev->mman.discovery_bin)
4487 amdgpu_discovery_fini(adev);
4488
4489 amdgpu_reset_put_reset_domain(adev->reset_domain);
4490 adev->reset_domain = NULL;
4491
4492 kfree(adev->pci_state);
4493
4494}
4495
4496/**
4497 * amdgpu_device_evict_resources - evict device resources
4498 * @adev: amdgpu device object
4499 *
4500 * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4501 * of the vram memory type. Mainly used for evicting device resources
4502 * at suspend time.
4503 *
4504 */
4505static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4506{
4507 int ret;
4508
4509 /* No need to evict vram on APUs for suspend to ram or s2idle */
4510 if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4511 return 0;
4512
4513 ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4514 if (ret)
4515 DRM_WARN("evicting device resources failed\n");
4516 return ret;
4517}
4518
4519/*
4520 * Suspend & resume.
4521 */
4522/**
4523 * amdgpu_device_prepare - prepare for device suspend
4524 *
4525 * @dev: drm dev pointer
4526 *
4527 * Prepare to put the hw in the suspend state (all asics).
4528 * Returns 0 for success or an error on failure.
4529 * Called at driver suspend.
4530 */
4531int amdgpu_device_prepare(struct drm_device *dev)
4532{
4533 struct amdgpu_device *adev = drm_to_adev(dev);
4534 int i, r;
4535
4536 amdgpu_choose_low_power_state(adev);
4537
4538 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4539 return 0;
4540
4541 /* Evict the majority of BOs before starting suspend sequence */
4542 r = amdgpu_device_evict_resources(adev);
4543 if (r)
4544 goto unprepare;
4545
4546 flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4547
4548 for (i = 0; i < adev->num_ip_blocks; i++) {
4549 if (!adev->ip_blocks[i].status.valid)
4550 continue;
4551 if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4552 continue;
4553 r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev);
4554 if (r)
4555 goto unprepare;
4556 }
4557
4558 return 0;
4559
4560unprepare:
4561 adev->in_s0ix = adev->in_s3 = false;
4562
4563 return r;
4564}
4565
4566/**
4567 * amdgpu_device_suspend - initiate device suspend
4568 *
4569 * @dev: drm dev pointer
4570 * @fbcon : notify the fbdev of suspend
4571 *
4572 * Puts the hw in the suspend state (all asics).
4573 * Returns 0 for success or an error on failure.
4574 * Called at driver suspend.
4575 */
4576int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4577{
4578 struct amdgpu_device *adev = drm_to_adev(dev);
4579 int r = 0;
4580
4581 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4582 return 0;
4583
4584 adev->in_suspend = true;
4585
4586 if (amdgpu_sriov_vf(adev)) {
4587 amdgpu_virt_fini_data_exchange(adev);
4588 r = amdgpu_virt_request_full_gpu(adev, false);
4589 if (r)
4590 return r;
4591 }
4592
4593 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4594 DRM_WARN("smart shift update failed\n");
4595
4596 if (fbcon)
4597 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4598
4599 cancel_delayed_work_sync(&adev->delayed_init_work);
4600
4601 amdgpu_ras_suspend(adev);
4602
4603 amdgpu_device_ip_suspend_phase1(adev);
4604
4605 if (!adev->in_s0ix)
4606 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4607
4608 r = amdgpu_device_evict_resources(adev);
4609 if (r)
4610 return r;
4611
4612 amdgpu_ttm_set_buffer_funcs_status(adev, false);
4613
4614 amdgpu_fence_driver_hw_fini(adev);
4615
4616 amdgpu_device_ip_suspend_phase2(adev);
4617
4618 if (amdgpu_sriov_vf(adev))
4619 amdgpu_virt_release_full_gpu(adev, false);
4620
4621 r = amdgpu_dpm_notify_rlc_state(adev, false);
4622 if (r)
4623 return r;
4624
4625 return 0;
4626}
4627
4628/**
4629 * amdgpu_device_resume - initiate device resume
4630 *
4631 * @dev: drm dev pointer
4632 * @fbcon : notify the fbdev of resume
4633 *
4634 * Bring the hw back to operating state (all asics).
4635 * Returns 0 for success or an error on failure.
4636 * Called at driver resume.
4637 */
4638int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4639{
4640 struct amdgpu_device *adev = drm_to_adev(dev);
4641 int r = 0;
4642
4643 if (amdgpu_sriov_vf(adev)) {
4644 r = amdgpu_virt_request_full_gpu(adev, true);
4645 if (r)
4646 return r;
4647 }
4648
4649 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4650 return 0;
4651
4652 if (adev->in_s0ix)
4653 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4654
4655 /* post card */
4656 if (amdgpu_device_need_post(adev)) {
4657 r = amdgpu_device_asic_init(adev);
4658 if (r)
4659 dev_err(adev->dev, "amdgpu asic init failed\n");
4660 }
4661
4662 r = amdgpu_device_ip_resume(adev);
4663
4664 if (r) {
4665 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4666 goto exit;
4667 }
4668 amdgpu_fence_driver_hw_init(adev);
4669
4670 if (!adev->in_s0ix) {
4671 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4672 if (r)
4673 goto exit;
4674 }
4675
4676 r = amdgpu_device_ip_late_init(adev);
4677 if (r)
4678 goto exit;
4679
4680 queue_delayed_work(system_wq, &adev->delayed_init_work,
4681 msecs_to_jiffies(AMDGPU_RESUME_MS));
4682exit:
4683 if (amdgpu_sriov_vf(adev)) {
4684 amdgpu_virt_init_data_exchange(adev);
4685 amdgpu_virt_release_full_gpu(adev, true);
4686 }
4687
4688 if (r)
4689 return r;
4690
4691 /* Make sure IB tests flushed */
4692 flush_delayed_work(&adev->delayed_init_work);
4693
4694 if (fbcon)
4695 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4696
4697 amdgpu_ras_resume(adev);
4698
4699 if (adev->mode_info.num_crtc) {
4700 /*
4701 * Most of the connector probing functions try to acquire runtime pm
4702 * refs to ensure that the GPU is powered on when connector polling is
4703 * performed. Since we're calling this from a runtime PM callback,
4704 * trying to acquire rpm refs will cause us to deadlock.
4705 *
4706 * Since we're guaranteed to be holding the rpm lock, it's safe to
4707 * temporarily disable the rpm helpers so this doesn't deadlock us.
4708 */
4709#ifdef CONFIG_PM
4710 dev->dev->power.disable_depth++;
4711#endif
4712 if (!adev->dc_enabled)
4713 drm_helper_hpd_irq_event(dev);
4714 else
4715 drm_kms_helper_hotplug_event(dev);
4716#ifdef CONFIG_PM
4717 dev->dev->power.disable_depth--;
4718#endif
4719 }
4720 adev->in_suspend = false;
4721
4722 if (adev->enable_mes)
4723 amdgpu_mes_self_test(adev);
4724
4725 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4726 DRM_WARN("smart shift update failed\n");
4727
4728 return 0;
4729}
4730
4731/**
4732 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4733 *
4734 * @adev: amdgpu_device pointer
4735 *
4736 * The list of all the hardware IPs that make up the asic is walked and
4737 * the check_soft_reset callbacks are run. check_soft_reset determines
4738 * if the asic is still hung or not.
4739 * Returns true if any of the IPs are still in a hung state, false if not.
4740 */
4741static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4742{
4743 int i;
4744 bool asic_hang = false;
4745
4746 if (amdgpu_sriov_vf(adev))
4747 return true;
4748
4749 if (amdgpu_asic_need_full_reset(adev))
4750 return true;
4751
4752 for (i = 0; i < adev->num_ip_blocks; i++) {
4753 if (!adev->ip_blocks[i].status.valid)
4754 continue;
4755 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4756 adev->ip_blocks[i].status.hang =
4757 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4758 if (adev->ip_blocks[i].status.hang) {
4759 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4760 asic_hang = true;
4761 }
4762 }
4763 return asic_hang;
4764}
4765
4766/**
4767 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4768 *
4769 * @adev: amdgpu_device pointer
4770 *
4771 * The list of all the hardware IPs that make up the asic is walked and the
4772 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
4773 * handles any IP specific hardware or software state changes that are
4774 * necessary for a soft reset to succeed.
4775 * Returns 0 on success, negative error code on failure.
4776 */
4777static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4778{
4779 int i, r = 0;
4780
4781 for (i = 0; i < adev->num_ip_blocks; i++) {
4782 if (!adev->ip_blocks[i].status.valid)
4783 continue;
4784 if (adev->ip_blocks[i].status.hang &&
4785 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4786 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4787 if (r)
4788 return r;
4789 }
4790 }
4791
4792 return 0;
4793}
4794
4795/**
4796 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4797 *
4798 * @adev: amdgpu_device pointer
4799 *
4800 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
4801 * reset is necessary to recover.
4802 * Returns true if a full asic reset is required, false if not.
4803 */
4804static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4805{
4806 int i;
4807
4808 if (amdgpu_asic_need_full_reset(adev))
4809 return true;
4810
4811 for (i = 0; i < adev->num_ip_blocks; i++) {
4812 if (!adev->ip_blocks[i].status.valid)
4813 continue;
4814 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4815 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4816 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4817 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4818 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4819 if (adev->ip_blocks[i].status.hang) {
4820 dev_info(adev->dev, "Some block need full reset!\n");
4821 return true;
4822 }
4823 }
4824 }
4825 return false;
4826}
4827
4828/**
4829 * amdgpu_device_ip_soft_reset - do a soft reset
4830 *
4831 * @adev: amdgpu_device pointer
4832 *
4833 * The list of all the hardware IPs that make up the asic is walked and the
4834 * soft_reset callbacks are run if the block is hung. soft_reset handles any
4835 * IP specific hardware or software state changes that are necessary to soft
4836 * reset the IP.
4837 * Returns 0 on success, negative error code on failure.
4838 */
4839static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4840{
4841 int i, r = 0;
4842
4843 for (i = 0; i < adev->num_ip_blocks; i++) {
4844 if (!adev->ip_blocks[i].status.valid)
4845 continue;
4846 if (adev->ip_blocks[i].status.hang &&
4847 adev->ip_blocks[i].version->funcs->soft_reset) {
4848 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4849 if (r)
4850 return r;
4851 }
4852 }
4853
4854 return 0;
4855}
4856
4857/**
4858 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4859 *
4860 * @adev: amdgpu_device pointer
4861 *
4862 * The list of all the hardware IPs that make up the asic is walked and the
4863 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
4864 * handles any IP specific hardware or software state changes that are
4865 * necessary after the IP has been soft reset.
4866 * Returns 0 on success, negative error code on failure.
4867 */
4868static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4869{
4870 int i, r = 0;
4871
4872 for (i = 0; i < adev->num_ip_blocks; i++) {
4873 if (!adev->ip_blocks[i].status.valid)
4874 continue;
4875 if (adev->ip_blocks[i].status.hang &&
4876 adev->ip_blocks[i].version->funcs->post_soft_reset)
4877 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4878 if (r)
4879 return r;
4880 }
4881
4882 return 0;
4883}
4884
4885/**
4886 * amdgpu_device_recover_vram - Recover some VRAM contents
4887 *
4888 * @adev: amdgpu_device pointer
4889 *
4890 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
4891 * restore things like GPUVM page tables after a GPU reset where
4892 * the contents of VRAM might be lost.
4893 *
4894 * Returns:
4895 * 0 on success, negative error code on failure.
4896 */
4897static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4898{
4899 struct dma_fence *fence = NULL, *next = NULL;
4900 struct amdgpu_bo *shadow;
4901 struct amdgpu_bo_vm *vmbo;
4902 long r = 1, tmo;
4903
4904 if (amdgpu_sriov_runtime(adev))
4905 tmo = msecs_to_jiffies(8000);
4906 else
4907 tmo = msecs_to_jiffies(100);
4908
4909 dev_info(adev->dev, "recover vram bo from shadow start\n");
4910 mutex_lock(&adev->shadow_list_lock);
4911 list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4912 /* If vm is compute context or adev is APU, shadow will be NULL */
4913 if (!vmbo->shadow)
4914 continue;
4915 shadow = vmbo->shadow;
4916
4917 /* No need to recover an evicted BO */
4918 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4919 shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4920 shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4921 continue;
4922
4923 r = amdgpu_bo_restore_shadow(shadow, &next);
4924 if (r)
4925 break;
4926
4927 if (fence) {
4928 tmo = dma_fence_wait_timeout(fence, false, tmo);
4929 dma_fence_put(fence);
4930 fence = next;
4931 if (tmo == 0) {
4932 r = -ETIMEDOUT;
4933 break;
4934 } else if (tmo < 0) {
4935 r = tmo;
4936 break;
4937 }
4938 } else {
4939 fence = next;
4940 }
4941 }
4942 mutex_unlock(&adev->shadow_list_lock);
4943
4944 if (fence)
4945 tmo = dma_fence_wait_timeout(fence, false, tmo);
4946 dma_fence_put(fence);
4947
4948 if (r < 0 || tmo <= 0) {
4949 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4950 return -EIO;
4951 }
4952
4953 dev_info(adev->dev, "recover vram bo from shadow done\n");
4954 return 0;
4955}
4956
4957
4958/**
4959 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4960 *
4961 * @adev: amdgpu_device pointer
4962 * @from_hypervisor: request from hypervisor
4963 *
4964 * do VF FLR and reinitialize Asic
4965 * return 0 means succeeded otherwise failed
4966 */
4967static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4968 bool from_hypervisor)
4969{
4970 int r;
4971 struct amdgpu_hive_info *hive = NULL;
4972 int retry_limit = 0;
4973
4974retry:
4975 amdgpu_amdkfd_pre_reset(adev);
4976
4977 if (from_hypervisor)
4978 r = amdgpu_virt_request_full_gpu(adev, true);
4979 else
4980 r = amdgpu_virt_reset_gpu(adev);
4981 if (r)
4982 return r;
4983 amdgpu_irq_gpu_reset_resume_helper(adev);
4984
4985 /* some sw clean up VF needs to do before recover */
4986 amdgpu_virt_post_reset(adev);
4987
4988 /* Resume IP prior to SMC */
4989 r = amdgpu_device_ip_reinit_early_sriov(adev);
4990 if (r)
4991 goto error;
4992
4993 amdgpu_virt_init_data_exchange(adev);
4994
4995 r = amdgpu_device_fw_loading(adev);
4996 if (r)
4997 return r;
4998
4999 /* now we are okay to resume SMC/CP/SDMA */
5000 r = amdgpu_device_ip_reinit_late_sriov(adev);
5001 if (r)
5002 goto error;
5003
5004 hive = amdgpu_get_xgmi_hive(adev);
5005 /* Update PSP FW topology after reset */
5006 if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
5007 r = amdgpu_xgmi_update_topology(hive, adev);
5008
5009 if (hive)
5010 amdgpu_put_xgmi_hive(hive);
5011
5012 if (!r) {
5013 r = amdgpu_ib_ring_tests(adev);
5014
5015 amdgpu_amdkfd_post_reset(adev);
5016 }
5017
5018error:
5019 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
5020 amdgpu_inc_vram_lost(adev);
5021 r = amdgpu_device_recover_vram(adev);
5022 }
5023 amdgpu_virt_release_full_gpu(adev, true);
5024
5025 if (AMDGPU_RETRY_SRIOV_RESET(r)) {
5026 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
5027 retry_limit++;
5028 goto retry;
5029 } else
5030 DRM_ERROR("GPU reset retry is beyond the retry limit\n");
5031 }
5032
5033 return r;
5034}
5035
5036/**
5037 * amdgpu_device_has_job_running - check if there is any job in mirror list
5038 *
5039 * @adev: amdgpu_device pointer
5040 *
5041 * check if there is any job in mirror list
5042 */
5043bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
5044{
5045 int i;
5046 struct drm_sched_job *job;
5047
5048 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5049 struct amdgpu_ring *ring = adev->rings[i];
5050
5051 if (!amdgpu_ring_sched_ready(ring))
5052 continue;
5053
5054 spin_lock(&ring->sched.job_list_lock);
5055 job = list_first_entry_or_null(&ring->sched.pending_list,
5056 struct drm_sched_job, list);
5057 spin_unlock(&ring->sched.job_list_lock);
5058 if (job)
5059 return true;
5060 }
5061 return false;
5062}
5063
5064/**
5065 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5066 *
5067 * @adev: amdgpu_device pointer
5068 *
5069 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
5070 * a hung GPU.
5071 */
5072bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
5073{
5074
5075 if (amdgpu_gpu_recovery == 0)
5076 goto disabled;
5077
5078 /* Skip soft reset check in fatal error mode */
5079 if (!amdgpu_ras_is_poison_mode_supported(adev))
5080 return true;
5081
5082 if (amdgpu_sriov_vf(adev))
5083 return true;
5084
5085 if (amdgpu_gpu_recovery == -1) {
5086 switch (adev->asic_type) {
5087#ifdef CONFIG_DRM_AMDGPU_SI
5088 case CHIP_VERDE:
5089 case CHIP_TAHITI:
5090 case CHIP_PITCAIRN:
5091 case CHIP_OLAND:
5092 case CHIP_HAINAN:
5093#endif
5094#ifdef CONFIG_DRM_AMDGPU_CIK
5095 case CHIP_KAVERI:
5096 case CHIP_KABINI:
5097 case CHIP_MULLINS:
5098#endif
5099 case CHIP_CARRIZO:
5100 case CHIP_STONEY:
5101 case CHIP_CYAN_SKILLFISH:
5102 goto disabled;
5103 default:
5104 break;
5105 }
5106 }
5107
5108 return true;
5109
5110disabled:
5111 dev_info(adev->dev, "GPU recovery disabled.\n");
5112 return false;
5113}
5114
5115int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
5116{
5117 u32 i;
5118 int ret = 0;
5119
5120 amdgpu_atombios_scratch_regs_engine_hung(adev, true);
5121
5122 dev_info(adev->dev, "GPU mode1 reset\n");
5123
5124 /* disable BM */
5125 pci_clear_master(adev->pdev);
5126
5127 amdgpu_device_cache_pci_state(adev->pdev);
5128
5129 if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
5130 dev_info(adev->dev, "GPU smu mode1 reset\n");
5131 ret = amdgpu_dpm_mode1_reset(adev);
5132 } else {
5133 dev_info(adev->dev, "GPU psp mode1 reset\n");
5134 ret = psp_gpu_reset(adev);
5135 }
5136
5137 if (ret)
5138 goto mode1_reset_failed;
5139
5140 amdgpu_device_load_pci_state(adev->pdev);
5141 ret = amdgpu_psp_wait_for_bootloader(adev);
5142 if (ret)
5143 goto mode1_reset_failed;
5144
5145 /* wait for asic to come out of reset */
5146 for (i = 0; i < adev->usec_timeout; i++) {
5147 u32 memsize = adev->nbio.funcs->get_memsize(adev);
5148
5149 if (memsize != 0xffffffff)
5150 break;
5151 udelay(1);
5152 }
5153
5154 if (i >= adev->usec_timeout) {
5155 ret = -ETIMEDOUT;
5156 goto mode1_reset_failed;
5157 }
5158
5159 amdgpu_atombios_scratch_regs_engine_hung(adev, false);
5160
5161 return 0;
5162
5163mode1_reset_failed:
5164 dev_err(adev->dev, "GPU mode1 reset failed\n");
5165 return ret;
5166}
5167
5168int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5169 struct amdgpu_reset_context *reset_context)
5170{
5171 int i, r = 0;
5172 struct amdgpu_job *job = NULL;
5173 bool need_full_reset =
5174 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5175
5176 if (reset_context->reset_req_dev == adev)
5177 job = reset_context->job;
5178
5179 if (amdgpu_sriov_vf(adev)) {
5180 /* stop the data exchange thread */
5181 amdgpu_virt_fini_data_exchange(adev);
5182 }
5183
5184 amdgpu_fence_driver_isr_toggle(adev, true);
5185
5186 /* block all schedulers and reset given job's ring */
5187 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5188 struct amdgpu_ring *ring = adev->rings[i];
5189
5190 if (!amdgpu_ring_sched_ready(ring))
5191 continue;
5192
5193 /* Clear job fence from fence drv to avoid force_completion
5194 * leave NULL and vm flush fence in fence drv
5195 */
5196 amdgpu_fence_driver_clear_job_fences(ring);
5197
5198 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5199 amdgpu_fence_driver_force_completion(ring);
5200 }
5201
5202 amdgpu_fence_driver_isr_toggle(adev, false);
5203
5204 if (job && job->vm)
5205 drm_sched_increase_karma(&job->base);
5206
5207 r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5208 /* If reset handler not implemented, continue; otherwise return */
5209 if (r == -EOPNOTSUPP)
5210 r = 0;
5211 else
5212 return r;
5213
5214 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5215 if (!amdgpu_sriov_vf(adev)) {
5216
5217 if (!need_full_reset)
5218 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5219
5220 if (!need_full_reset && amdgpu_gpu_recovery &&
5221 amdgpu_device_ip_check_soft_reset(adev)) {
5222 amdgpu_device_ip_pre_soft_reset(adev);
5223 r = amdgpu_device_ip_soft_reset(adev);
5224 amdgpu_device_ip_post_soft_reset(adev);
5225 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5226 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5227 need_full_reset = true;
5228 }
5229 }
5230
5231 if (need_full_reset)
5232 r = amdgpu_device_ip_suspend(adev);
5233 if (need_full_reset)
5234 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5235 else
5236 clear_bit(AMDGPU_NEED_FULL_RESET,
5237 &reset_context->flags);
5238 }
5239
5240 return r;
5241}
5242
5243static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
5244{
5245 int i;
5246
5247 lockdep_assert_held(&adev->reset_domain->sem);
5248
5249 for (i = 0; i < adev->reset_info.num_regs; i++) {
5250 adev->reset_info.reset_dump_reg_value[i] =
5251 RREG32(adev->reset_info.reset_dump_reg_list[i]);
5252
5253 trace_amdgpu_reset_reg_dumps(adev->reset_info.reset_dump_reg_list[i],
5254 adev->reset_info.reset_dump_reg_value[i]);
5255 }
5256
5257 return 0;
5258}
5259
5260int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5261 struct amdgpu_reset_context *reset_context)
5262{
5263 struct amdgpu_device *tmp_adev = NULL;
5264 bool need_full_reset, skip_hw_reset, vram_lost = false;
5265 int r = 0;
5266
5267 /* Try reset handler method first */
5268 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5269 reset_list);
5270 amdgpu_reset_reg_dumps(tmp_adev);
5271
5272 reset_context->reset_device_list = device_list_handle;
5273 r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5274 /* If reset handler not implemented, continue; otherwise return */
5275 if (r == -EOPNOTSUPP)
5276 r = 0;
5277 else
5278 return r;
5279
5280 /* Reset handler not implemented, use the default method */
5281 need_full_reset =
5282 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5283 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5284
5285 /*
5286 * ASIC reset has to be done on all XGMI hive nodes ASAP
5287 * to allow proper links negotiation in FW (within 1 sec)
5288 */
5289 if (!skip_hw_reset && need_full_reset) {
5290 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5291 /* For XGMI run all resets in parallel to speed up the process */
5292 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5293 tmp_adev->gmc.xgmi.pending_reset = false;
5294 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
5295 r = -EALREADY;
5296 } else
5297 r = amdgpu_asic_reset(tmp_adev);
5298
5299 if (r) {
5300 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
5301 r, adev_to_drm(tmp_adev)->unique);
5302 goto out;
5303 }
5304 }
5305
5306 /* For XGMI wait for all resets to complete before proceed */
5307 if (!r) {
5308 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5309 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5310 flush_work(&tmp_adev->xgmi_reset_work);
5311 r = tmp_adev->asic_reset_res;
5312 if (r)
5313 break;
5314 }
5315 }
5316 }
5317 }
5318
5319 if (!r && amdgpu_ras_intr_triggered()) {
5320 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5321 amdgpu_ras_reset_error_count(tmp_adev, AMDGPU_RAS_BLOCK__MMHUB);
5322 }
5323
5324 amdgpu_ras_intr_cleared();
5325 }
5326
5327 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5328 if (need_full_reset) {
5329 /* post card */
5330 amdgpu_ras_set_fed(tmp_adev, false);
5331 r = amdgpu_device_asic_init(tmp_adev);
5332 if (r) {
5333 dev_warn(tmp_adev->dev, "asic atom init failed!");
5334 } else {
5335 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5336
5337 r = amdgpu_device_ip_resume_phase1(tmp_adev);
5338 if (r)
5339 goto out;
5340
5341 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5342
5343 amdgpu_coredump(tmp_adev, vram_lost, reset_context);
5344
5345 if (vram_lost) {
5346 DRM_INFO("VRAM is lost due to GPU reset!\n");
5347 amdgpu_inc_vram_lost(tmp_adev);
5348 }
5349
5350 r = amdgpu_device_fw_loading(tmp_adev);
5351 if (r)
5352 return r;
5353
5354 r = amdgpu_xcp_restore_partition_mode(
5355 tmp_adev->xcp_mgr);
5356 if (r)
5357 goto out;
5358
5359 r = amdgpu_device_ip_resume_phase2(tmp_adev);
5360 if (r)
5361 goto out;
5362
5363 if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5364 amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
5365
5366 if (vram_lost)
5367 amdgpu_device_fill_reset_magic(tmp_adev);
5368
5369 /*
5370 * Add this ASIC as tracked as reset was already
5371 * complete successfully.
5372 */
5373 amdgpu_register_gpu_instance(tmp_adev);
5374
5375 if (!reset_context->hive &&
5376 tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5377 amdgpu_xgmi_add_device(tmp_adev);
5378
5379 r = amdgpu_device_ip_late_init(tmp_adev);
5380 if (r)
5381 goto out;
5382
5383 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
5384
5385 /*
5386 * The GPU enters bad state once faulty pages
5387 * by ECC has reached the threshold, and ras
5388 * recovery is scheduled next. So add one check
5389 * here to break recovery if it indeed exceeds
5390 * bad page threshold, and remind user to
5391 * retire this GPU or setting one bigger
5392 * bad_page_threshold value to fix this once
5393 * probing driver again.
5394 */
5395 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5396 /* must succeed. */
5397 amdgpu_ras_resume(tmp_adev);
5398 } else {
5399 r = -EINVAL;
5400 goto out;
5401 }
5402
5403 /* Update PSP FW topology after reset */
5404 if (reset_context->hive &&
5405 tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5406 r = amdgpu_xgmi_update_topology(
5407 reset_context->hive, tmp_adev);
5408 }
5409 }
5410
5411out:
5412 if (!r) {
5413 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5414 r = amdgpu_ib_ring_tests(tmp_adev);
5415 if (r) {
5416 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5417 need_full_reset = true;
5418 r = -EAGAIN;
5419 goto end;
5420 }
5421 }
5422
5423 if (!r)
5424 r = amdgpu_device_recover_vram(tmp_adev);
5425 else
5426 tmp_adev->asic_reset_res = r;
5427 }
5428
5429end:
5430 if (need_full_reset)
5431 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5432 else
5433 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5434 return r;
5435}
5436
5437static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5438{
5439
5440 switch (amdgpu_asic_reset_method(adev)) {
5441 case AMD_RESET_METHOD_MODE1:
5442 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5443 break;
5444 case AMD_RESET_METHOD_MODE2:
5445 adev->mp1_state = PP_MP1_STATE_RESET;
5446 break;
5447 default:
5448 adev->mp1_state = PP_MP1_STATE_NONE;
5449 break;
5450 }
5451}
5452
5453static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5454{
5455 amdgpu_vf_error_trans_all(adev);
5456 adev->mp1_state = PP_MP1_STATE_NONE;
5457}
5458
5459static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5460{
5461 struct pci_dev *p = NULL;
5462
5463 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5464 adev->pdev->bus->number, 1);
5465 if (p) {
5466 pm_runtime_enable(&(p->dev));
5467 pm_runtime_resume(&(p->dev));
5468 }
5469
5470 pci_dev_put(p);
5471}
5472
5473static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5474{
5475 enum amd_reset_method reset_method;
5476 struct pci_dev *p = NULL;
5477 u64 expires;
5478
5479 /*
5480 * For now, only BACO and mode1 reset are confirmed
5481 * to suffer the audio issue without proper suspended.
5482 */
5483 reset_method = amdgpu_asic_reset_method(adev);
5484 if ((reset_method != AMD_RESET_METHOD_BACO) &&
5485 (reset_method != AMD_RESET_METHOD_MODE1))
5486 return -EINVAL;
5487
5488 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5489 adev->pdev->bus->number, 1);
5490 if (!p)
5491 return -ENODEV;
5492
5493 expires = pm_runtime_autosuspend_expiration(&(p->dev));
5494 if (!expires)
5495 /*
5496 * If we cannot get the audio device autosuspend delay,
5497 * a fixed 4S interval will be used. Considering 3S is
5498 * the audio controller default autosuspend delay setting.
5499 * 4S used here is guaranteed to cover that.
5500 */
5501 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5502
5503 while (!pm_runtime_status_suspended(&(p->dev))) {
5504 if (!pm_runtime_suspend(&(p->dev)))
5505 break;
5506
5507 if (expires < ktime_get_mono_fast_ns()) {
5508 dev_warn(adev->dev, "failed to suspend display audio\n");
5509 pci_dev_put(p);
5510 /* TODO: abort the succeeding gpu reset? */
5511 return -ETIMEDOUT;
5512 }
5513 }
5514
5515 pm_runtime_disable(&(p->dev));
5516
5517 pci_dev_put(p);
5518 return 0;
5519}
5520
5521static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5522{
5523 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5524
5525#if defined(CONFIG_DEBUG_FS)
5526 if (!amdgpu_sriov_vf(adev))
5527 cancel_work(&adev->reset_work);
5528#endif
5529
5530 if (adev->kfd.dev)
5531 cancel_work(&adev->kfd.reset_work);
5532
5533 if (amdgpu_sriov_vf(adev))
5534 cancel_work(&adev->virt.flr_work);
5535
5536 if (con && adev->ras_enabled)
5537 cancel_work(&con->recovery_work);
5538
5539}
5540
5541/**
5542 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5543 *
5544 * @adev: amdgpu_device pointer
5545 * @job: which job trigger hang
5546 * @reset_context: amdgpu reset context pointer
5547 *
5548 * Attempt to reset the GPU if it has hung (all asics).
5549 * Attempt to do soft-reset or full-reset and reinitialize Asic
5550 * Returns 0 for success or an error on failure.
5551 */
5552
5553int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5554 struct amdgpu_job *job,
5555 struct amdgpu_reset_context *reset_context)
5556{
5557 struct list_head device_list, *device_list_handle = NULL;
5558 bool job_signaled = false;
5559 struct amdgpu_hive_info *hive = NULL;
5560 struct amdgpu_device *tmp_adev = NULL;
5561 int i, r = 0;
5562 bool need_emergency_restart = false;
5563 bool audio_suspended = false;
5564
5565 /*
5566 * Special case: RAS triggered and full reset isn't supported
5567 */
5568 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5569
5570 /*
5571 * Flush RAM to disk so that after reboot
5572 * the user can read log and see why the system rebooted.
5573 */
5574 if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5575 amdgpu_ras_get_context(adev)->reboot) {
5576 DRM_WARN("Emergency reboot.");
5577
5578 ksys_sync_helper();
5579 emergency_restart();
5580 }
5581
5582 dev_info(adev->dev, "GPU %s begin!\n",
5583 need_emergency_restart ? "jobs stop":"reset");
5584
5585 if (!amdgpu_sriov_vf(adev))
5586 hive = amdgpu_get_xgmi_hive(adev);
5587 if (hive)
5588 mutex_lock(&hive->hive_lock);
5589
5590 reset_context->job = job;
5591 reset_context->hive = hive;
5592 /*
5593 * Build list of devices to reset.
5594 * In case we are in XGMI hive mode, resort the device list
5595 * to put adev in the 1st position.
5596 */
5597 INIT_LIST_HEAD(&device_list);
5598 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5599 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5600 list_add_tail(&tmp_adev->reset_list, &device_list);
5601 if (adev->shutdown)
5602 tmp_adev->shutdown = true;
5603 }
5604 if (!list_is_first(&adev->reset_list, &device_list))
5605 list_rotate_to_front(&adev->reset_list, &device_list);
5606 device_list_handle = &device_list;
5607 } else {
5608 list_add_tail(&adev->reset_list, &device_list);
5609 device_list_handle = &device_list;
5610 }
5611
5612 /* We need to lock reset domain only once both for XGMI and single device */
5613 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5614 reset_list);
5615 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5616
5617 /* block all schedulers and reset given job's ring */
5618 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5619
5620 amdgpu_device_set_mp1_state(tmp_adev);
5621
5622 /*
5623 * Try to put the audio codec into suspend state
5624 * before gpu reset started.
5625 *
5626 * Due to the power domain of the graphics device
5627 * is shared with AZ power domain. Without this,
5628 * we may change the audio hardware from behind
5629 * the audio driver's back. That will trigger
5630 * some audio codec errors.
5631 */
5632 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5633 audio_suspended = true;
5634
5635 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5636
5637 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5638
5639 if (!amdgpu_sriov_vf(tmp_adev))
5640 amdgpu_amdkfd_pre_reset(tmp_adev);
5641
5642 /*
5643 * Mark these ASICs to be reseted as untracked first
5644 * And add them back after reset completed
5645 */
5646 amdgpu_unregister_gpu_instance(tmp_adev);
5647
5648 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5649
5650 /* disable ras on ALL IPs */
5651 if (!need_emergency_restart &&
5652 amdgpu_device_ip_need_full_reset(tmp_adev))
5653 amdgpu_ras_suspend(tmp_adev);
5654
5655 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5656 struct amdgpu_ring *ring = tmp_adev->rings[i];
5657
5658 if (!amdgpu_ring_sched_ready(ring))
5659 continue;
5660
5661 drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5662
5663 if (need_emergency_restart)
5664 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5665 }
5666 atomic_inc(&tmp_adev->gpu_reset_counter);
5667 }
5668
5669 if (need_emergency_restart)
5670 goto skip_sched_resume;
5671
5672 /*
5673 * Must check guilty signal here since after this point all old
5674 * HW fences are force signaled.
5675 *
5676 * job->base holds a reference to parent fence
5677 */
5678 if (job && dma_fence_is_signaled(&job->hw_fence)) {
5679 job_signaled = true;
5680 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5681 goto skip_hw_reset;
5682 }
5683
5684retry: /* Rest of adevs pre asic reset from XGMI hive. */
5685 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5686 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5687 /*TODO Should we stop ?*/
5688 if (r) {
5689 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5690 r, adev_to_drm(tmp_adev)->unique);
5691 tmp_adev->asic_reset_res = r;
5692 }
5693
5694 /*
5695 * Drop all pending non scheduler resets. Scheduler resets
5696 * were already dropped during drm_sched_stop
5697 */
5698 amdgpu_device_stop_pending_resets(tmp_adev);
5699 }
5700
5701 /* Actual ASIC resets if needed.*/
5702 /* Host driver will handle XGMI hive reset for SRIOV */
5703 if (amdgpu_sriov_vf(adev)) {
5704 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5705 if (r)
5706 adev->asic_reset_res = r;
5707
5708 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5709 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
5710 IP_VERSION(9, 4, 2) ||
5711 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
5712 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
5713 amdgpu_ras_resume(adev);
5714 } else {
5715 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5716 if (r && r == -EAGAIN)
5717 goto retry;
5718 }
5719
5720skip_hw_reset:
5721
5722 /* Post ASIC reset for all devs .*/
5723 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5724
5725 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5726 struct amdgpu_ring *ring = tmp_adev->rings[i];
5727
5728 if (!amdgpu_ring_sched_ready(ring))
5729 continue;
5730
5731 drm_sched_start(&ring->sched, true);
5732 }
5733
5734 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
5735 drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5736
5737 if (tmp_adev->asic_reset_res)
5738 r = tmp_adev->asic_reset_res;
5739
5740 tmp_adev->asic_reset_res = 0;
5741
5742 if (r) {
5743 /* bad news, how to tell it to userspace ? */
5744 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5745 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5746 } else {
5747 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5748 if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5749 DRM_WARN("smart shift update failed\n");
5750 }
5751 }
5752
5753skip_sched_resume:
5754 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5755 /* unlock kfd: SRIOV would do it separately */
5756 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5757 amdgpu_amdkfd_post_reset(tmp_adev);
5758
5759 /* kfd_post_reset will do nothing if kfd device is not initialized,
5760 * need to bring up kfd here if it's not be initialized before
5761 */
5762 if (!adev->kfd.init_complete)
5763 amdgpu_amdkfd_device_init(adev);
5764
5765 if (audio_suspended)
5766 amdgpu_device_resume_display_audio(tmp_adev);
5767
5768 amdgpu_device_unset_mp1_state(tmp_adev);
5769
5770 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5771 }
5772
5773 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5774 reset_list);
5775 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5776
5777 if (hive) {
5778 mutex_unlock(&hive->hive_lock);
5779 amdgpu_put_xgmi_hive(hive);
5780 }
5781
5782 if (r)
5783 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5784
5785 atomic_set(&adev->reset_domain->reset_res, r);
5786 return r;
5787}
5788
5789/**
5790 * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5791 *
5792 * @adev: amdgpu_device pointer
5793 * @speed: pointer to the speed of the link
5794 * @width: pointer to the width of the link
5795 *
5796 * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
5797 * first physical partner to an AMD dGPU.
5798 * This will exclude any virtual switches and links.
5799 */
5800static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
5801 enum pci_bus_speed *speed,
5802 enum pcie_link_width *width)
5803{
5804 struct pci_dev *parent = adev->pdev;
5805
5806 if (!speed || !width)
5807 return;
5808
5809 *speed = PCI_SPEED_UNKNOWN;
5810 *width = PCIE_LNK_WIDTH_UNKNOWN;
5811
5812 if (amdgpu_device_pcie_dynamic_switching_supported(adev)) {
5813 while ((parent = pci_upstream_bridge(parent))) {
5814 /* skip upstream/downstream switches internal to dGPU*/
5815 if (parent->vendor == PCI_VENDOR_ID_ATI)
5816 continue;
5817 *speed = pcie_get_speed_cap(parent);
5818 *width = pcie_get_width_cap(parent);
5819 break;
5820 }
5821 } else {
5822 /* use the current speeds rather than max if switching is not supported */
5823 pcie_bandwidth_available(adev->pdev, NULL, speed, width);
5824 }
5825}
5826
5827/**
5828 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5829 *
5830 * @adev: amdgpu_device pointer
5831 *
5832 * Fetchs and stores in the driver the PCIE capabilities (gen speed
5833 * and lanes) of the slot the device is in. Handles APUs and
5834 * virtualized environments where PCIE config space may not be available.
5835 */
5836static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5837{
5838 struct pci_dev *pdev;
5839 enum pci_bus_speed speed_cap, platform_speed_cap;
5840 enum pcie_link_width platform_link_width;
5841
5842 if (amdgpu_pcie_gen_cap)
5843 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5844
5845 if (amdgpu_pcie_lane_cap)
5846 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5847
5848 /* covers APUs as well */
5849 if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5850 if (adev->pm.pcie_gen_mask == 0)
5851 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5852 if (adev->pm.pcie_mlw_mask == 0)
5853 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5854 return;
5855 }
5856
5857 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5858 return;
5859
5860 amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
5861 &platform_link_width);
5862
5863 if (adev->pm.pcie_gen_mask == 0) {
5864 /* asic caps */
5865 pdev = adev->pdev;
5866 speed_cap = pcie_get_speed_cap(pdev);
5867 if (speed_cap == PCI_SPEED_UNKNOWN) {
5868 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5869 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5870 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5871 } else {
5872 if (speed_cap == PCIE_SPEED_32_0GT)
5873 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5874 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5875 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5876 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5877 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5878 else if (speed_cap == PCIE_SPEED_16_0GT)
5879 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5880 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5881 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5882 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5883 else if (speed_cap == PCIE_SPEED_8_0GT)
5884 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5885 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5886 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5887 else if (speed_cap == PCIE_SPEED_5_0GT)
5888 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5889 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5890 else
5891 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5892 }
5893 /* platform caps */
5894 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5895 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5896 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5897 } else {
5898 if (platform_speed_cap == PCIE_SPEED_32_0GT)
5899 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5900 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5901 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5902 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5903 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5904 else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5905 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5906 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5907 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5908 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5909 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5910 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5911 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5912 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5913 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5914 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5915 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5916 else
5917 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5918
5919 }
5920 }
5921 if (adev->pm.pcie_mlw_mask == 0) {
5922 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5923 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5924 } else {
5925 switch (platform_link_width) {
5926 case PCIE_LNK_X32:
5927 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5928 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5929 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5930 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5931 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5932 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5933 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5934 break;
5935 case PCIE_LNK_X16:
5936 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5937 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5938 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5939 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5940 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5941 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5942 break;
5943 case PCIE_LNK_X12:
5944 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5945 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5946 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5947 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5948 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5949 break;
5950 case PCIE_LNK_X8:
5951 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5952 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5953 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5954 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5955 break;
5956 case PCIE_LNK_X4:
5957 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5958 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5959 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5960 break;
5961 case PCIE_LNK_X2:
5962 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5963 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5964 break;
5965 case PCIE_LNK_X1:
5966 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5967 break;
5968 default:
5969 break;
5970 }
5971 }
5972 }
5973}
5974
5975/**
5976 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5977 *
5978 * @adev: amdgpu_device pointer
5979 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5980 *
5981 * Return true if @peer_adev can access (DMA) @adev through the PCIe
5982 * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5983 * @peer_adev.
5984 */
5985bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5986 struct amdgpu_device *peer_adev)
5987{
5988#ifdef CONFIG_HSA_AMD_P2P
5989 uint64_t address_mask = peer_adev->dev->dma_mask ?
5990 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5991 resource_size_t aper_limit =
5992 adev->gmc.aper_base + adev->gmc.aper_size - 1;
5993 bool p2p_access =
5994 !adev->gmc.xgmi.connected_to_cpu &&
5995 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5996
5997 return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5998 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5999 !(adev->gmc.aper_base & address_mask ||
6000 aper_limit & address_mask));
6001#else
6002 return false;
6003#endif
6004}
6005
6006int amdgpu_device_baco_enter(struct drm_device *dev)
6007{
6008 struct amdgpu_device *adev = drm_to_adev(dev);
6009 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6010
6011 if (!amdgpu_device_supports_baco(dev))
6012 return -ENOTSUPP;
6013
6014 if (ras && adev->ras_enabled &&
6015 adev->nbio.funcs->enable_doorbell_interrupt)
6016 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6017
6018 return amdgpu_dpm_baco_enter(adev);
6019}
6020
6021int amdgpu_device_baco_exit(struct drm_device *dev)
6022{
6023 struct amdgpu_device *adev = drm_to_adev(dev);
6024 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6025 int ret = 0;
6026
6027 if (!amdgpu_device_supports_baco(dev))
6028 return -ENOTSUPP;
6029
6030 ret = amdgpu_dpm_baco_exit(adev);
6031 if (ret)
6032 return ret;
6033
6034 if (ras && adev->ras_enabled &&
6035 adev->nbio.funcs->enable_doorbell_interrupt)
6036 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6037
6038 if (amdgpu_passthrough(adev) &&
6039 adev->nbio.funcs->clear_doorbell_interrupt)
6040 adev->nbio.funcs->clear_doorbell_interrupt(adev);
6041
6042 return 0;
6043}
6044
6045/**
6046 * amdgpu_pci_error_detected - Called when a PCI error is detected.
6047 * @pdev: PCI device struct
6048 * @state: PCI channel state
6049 *
6050 * Description: Called when a PCI error is detected.
6051 *
6052 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6053 */
6054pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6055{
6056 struct drm_device *dev = pci_get_drvdata(pdev);
6057 struct amdgpu_device *adev = drm_to_adev(dev);
6058 int i;
6059
6060 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
6061
6062 if (adev->gmc.xgmi.num_physical_nodes > 1) {
6063 DRM_WARN("No support for XGMI hive yet...");
6064 return PCI_ERS_RESULT_DISCONNECT;
6065 }
6066
6067 adev->pci_channel_state = state;
6068
6069 switch (state) {
6070 case pci_channel_io_normal:
6071 return PCI_ERS_RESULT_CAN_RECOVER;
6072 /* Fatal error, prepare for slot reset */
6073 case pci_channel_io_frozen:
6074 /*
6075 * Locking adev->reset_domain->sem will prevent any external access
6076 * to GPU during PCI error recovery
6077 */
6078 amdgpu_device_lock_reset_domain(adev->reset_domain);
6079 amdgpu_device_set_mp1_state(adev);
6080
6081 /*
6082 * Block any work scheduling as we do for regular GPU reset
6083 * for the duration of the recovery
6084 */
6085 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6086 struct amdgpu_ring *ring = adev->rings[i];
6087
6088 if (!amdgpu_ring_sched_ready(ring))
6089 continue;
6090
6091 drm_sched_stop(&ring->sched, NULL);
6092 }
6093 atomic_inc(&adev->gpu_reset_counter);
6094 return PCI_ERS_RESULT_NEED_RESET;
6095 case pci_channel_io_perm_failure:
6096 /* Permanent error, prepare for device removal */
6097 return PCI_ERS_RESULT_DISCONNECT;
6098 }
6099
6100 return PCI_ERS_RESULT_NEED_RESET;
6101}
6102
6103/**
6104 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6105 * @pdev: pointer to PCI device
6106 */
6107pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6108{
6109
6110 DRM_INFO("PCI error: mmio enabled callback!!\n");
6111
6112 /* TODO - dump whatever for debugging purposes */
6113
6114 /* This called only if amdgpu_pci_error_detected returns
6115 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6116 * works, no need to reset slot.
6117 */
6118
6119 return PCI_ERS_RESULT_RECOVERED;
6120}
6121
6122/**
6123 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6124 * @pdev: PCI device struct
6125 *
6126 * Description: This routine is called by the pci error recovery
6127 * code after the PCI slot has been reset, just before we
6128 * should resume normal operations.
6129 */
6130pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6131{
6132 struct drm_device *dev = pci_get_drvdata(pdev);
6133 struct amdgpu_device *adev = drm_to_adev(dev);
6134 int r, i;
6135 struct amdgpu_reset_context reset_context;
6136 u32 memsize;
6137 struct list_head device_list;
6138 struct amdgpu_hive_info *hive;
6139 int hive_ras_recovery = 0;
6140 struct amdgpu_ras *ras;
6141
6142 /* PCI error slot reset should be skipped During RAS recovery */
6143 hive = amdgpu_get_xgmi_hive(adev);
6144 if (hive) {
6145 hive_ras_recovery = atomic_read(&hive->ras_recovery);
6146 amdgpu_put_xgmi_hive(hive);
6147 }
6148 ras = amdgpu_ras_get_context(adev);
6149 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3)) &&
6150 ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery))
6151 return PCI_ERS_RESULT_RECOVERED;
6152
6153 DRM_INFO("PCI error: slot reset callback!!\n");
6154
6155 memset(&reset_context, 0, sizeof(reset_context));
6156
6157 INIT_LIST_HEAD(&device_list);
6158 list_add_tail(&adev->reset_list, &device_list);
6159
6160 /* wait for asic to come out of reset */
6161 msleep(500);
6162
6163 /* Restore PCI confspace */
6164 amdgpu_device_load_pci_state(pdev);
6165
6166 /* confirm ASIC came out of reset */
6167 for (i = 0; i < adev->usec_timeout; i++) {
6168 memsize = amdgpu_asic_get_config_memsize(adev);
6169
6170 if (memsize != 0xffffffff)
6171 break;
6172 udelay(1);
6173 }
6174 if (memsize == 0xffffffff) {
6175 r = -ETIME;
6176 goto out;
6177 }
6178
6179 reset_context.method = AMD_RESET_METHOD_NONE;
6180 reset_context.reset_req_dev = adev;
6181 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6182 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6183
6184 adev->no_hw_access = true;
6185 r = amdgpu_device_pre_asic_reset(adev, &reset_context);
6186 adev->no_hw_access = false;
6187 if (r)
6188 goto out;
6189
6190 r = amdgpu_do_asic_reset(&device_list, &reset_context);
6191
6192out:
6193 if (!r) {
6194 if (amdgpu_device_cache_pci_state(adev->pdev))
6195 pci_restore_state(adev->pdev);
6196
6197 DRM_INFO("PCIe error recovery succeeded\n");
6198 } else {
6199 DRM_ERROR("PCIe error recovery failed, err:%d", r);
6200 amdgpu_device_unset_mp1_state(adev);
6201 amdgpu_device_unlock_reset_domain(adev->reset_domain);
6202 }
6203
6204 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6205}
6206
6207/**
6208 * amdgpu_pci_resume() - resume normal ops after PCI reset
6209 * @pdev: pointer to PCI device
6210 *
6211 * Called when the error recovery driver tells us that its
6212 * OK to resume normal operation.
6213 */
6214void amdgpu_pci_resume(struct pci_dev *pdev)
6215{
6216 struct drm_device *dev = pci_get_drvdata(pdev);
6217 struct amdgpu_device *adev = drm_to_adev(dev);
6218 int i;
6219
6220
6221 DRM_INFO("PCI error: resume callback!!\n");
6222
6223 /* Only continue execution for the case of pci_channel_io_frozen */
6224 if (adev->pci_channel_state != pci_channel_io_frozen)
6225 return;
6226
6227 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6228 struct amdgpu_ring *ring = adev->rings[i];
6229
6230 if (!amdgpu_ring_sched_ready(ring))
6231 continue;
6232
6233 drm_sched_start(&ring->sched, true);
6234 }
6235
6236 amdgpu_device_unset_mp1_state(adev);
6237 amdgpu_device_unlock_reset_domain(adev->reset_domain);
6238}
6239
6240bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6241{
6242 struct drm_device *dev = pci_get_drvdata(pdev);
6243 struct amdgpu_device *adev = drm_to_adev(dev);
6244 int r;
6245
6246 r = pci_save_state(pdev);
6247 if (!r) {
6248 kfree(adev->pci_state);
6249
6250 adev->pci_state = pci_store_saved_state(pdev);
6251
6252 if (!adev->pci_state) {
6253 DRM_ERROR("Failed to store PCI saved state");
6254 return false;
6255 }
6256 } else {
6257 DRM_WARN("Failed to save PCI state, err:%d\n", r);
6258 return false;
6259 }
6260
6261 return true;
6262}
6263
6264bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6265{
6266 struct drm_device *dev = pci_get_drvdata(pdev);
6267 struct amdgpu_device *adev = drm_to_adev(dev);
6268 int r;
6269
6270 if (!adev->pci_state)
6271 return false;
6272
6273 r = pci_load_saved_state(pdev, adev->pci_state);
6274
6275 if (!r) {
6276 pci_restore_state(pdev);
6277 } else {
6278 DRM_WARN("Failed to load PCI state, err:%d\n", r);
6279 return false;
6280 }
6281
6282 return true;
6283}
6284
6285void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6286 struct amdgpu_ring *ring)
6287{
6288#ifdef CONFIG_X86_64
6289 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6290 return;
6291#endif
6292 if (adev->gmc.xgmi.connected_to_cpu)
6293 return;
6294
6295 if (ring && ring->funcs->emit_hdp_flush)
6296 amdgpu_ring_emit_hdp_flush(ring);
6297 else
6298 amdgpu_asic_flush_hdp(adev, ring);
6299}
6300
6301void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6302 struct amdgpu_ring *ring)
6303{
6304#ifdef CONFIG_X86_64
6305 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6306 return;
6307#endif
6308 if (adev->gmc.xgmi.connected_to_cpu)
6309 return;
6310
6311 amdgpu_asic_invalidate_hdp(adev, ring);
6312}
6313
6314int amdgpu_in_reset(struct amdgpu_device *adev)
6315{
6316 return atomic_read(&adev->reset_domain->in_gpu_reset);
6317}
6318
6319/**
6320 * amdgpu_device_halt() - bring hardware to some kind of halt state
6321 *
6322 * @adev: amdgpu_device pointer
6323 *
6324 * Bring hardware to some kind of halt state so that no one can touch it
6325 * any more. It will help to maintain error context when error occurred.
6326 * Compare to a simple hang, the system will keep stable at least for SSH
6327 * access. Then it should be trivial to inspect the hardware state and
6328 * see what's going on. Implemented as following:
6329 *
6330 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6331 * clears all CPU mappings to device, disallows remappings through page faults
6332 * 2. amdgpu_irq_disable_all() disables all interrupts
6333 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6334 * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6335 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6336 * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6337 * flush any in flight DMA operations
6338 */
6339void amdgpu_device_halt(struct amdgpu_device *adev)
6340{
6341 struct pci_dev *pdev = adev->pdev;
6342 struct drm_device *ddev = adev_to_drm(adev);
6343
6344 amdgpu_xcp_dev_unplug(adev);
6345 drm_dev_unplug(ddev);
6346
6347 amdgpu_irq_disable_all(adev);
6348
6349 amdgpu_fence_driver_hw_fini(adev);
6350
6351 adev->no_hw_access = true;
6352
6353 amdgpu_device_unmap_mmio(adev);
6354
6355 pci_disable_device(pdev);
6356 pci_wait_for_pending_transaction(pdev);
6357}
6358
6359u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6360 u32 reg)
6361{
6362 unsigned long flags, address, data;
6363 u32 r;
6364
6365 address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6366 data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6367
6368 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6369 WREG32(address, reg * 4);
6370 (void)RREG32(address);
6371 r = RREG32(data);
6372 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6373 return r;
6374}
6375
6376void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6377 u32 reg, u32 v)
6378{
6379 unsigned long flags, address, data;
6380
6381 address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6382 data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6383
6384 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6385 WREG32(address, reg * 4);
6386 (void)RREG32(address);
6387 WREG32(data, v);
6388 (void)RREG32(data);
6389 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6390}
6391
6392/**
6393 * amdgpu_device_switch_gang - switch to a new gang
6394 * @adev: amdgpu_device pointer
6395 * @gang: the gang to switch to
6396 *
6397 * Try to switch to a new gang.
6398 * Returns: NULL if we switched to the new gang or a reference to the current
6399 * gang leader.
6400 */
6401struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6402 struct dma_fence *gang)
6403{
6404 struct dma_fence *old = NULL;
6405
6406 do {
6407 dma_fence_put(old);
6408 rcu_read_lock();
6409 old = dma_fence_get_rcu_safe(&adev->gang_submit);
6410 rcu_read_unlock();
6411
6412 if (old == gang)
6413 break;
6414
6415 if (!dma_fence_is_signaled(old))
6416 return old;
6417
6418 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6419 old, gang) != old);
6420
6421 dma_fence_put(old);
6422 return NULL;
6423}
6424
6425bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6426{
6427 switch (adev->asic_type) {
6428#ifdef CONFIG_DRM_AMDGPU_SI
6429 case CHIP_HAINAN:
6430#endif
6431 case CHIP_TOPAZ:
6432 /* chips with no display hardware */
6433 return false;
6434#ifdef CONFIG_DRM_AMDGPU_SI
6435 case CHIP_TAHITI:
6436 case CHIP_PITCAIRN:
6437 case CHIP_VERDE:
6438 case CHIP_OLAND:
6439#endif
6440#ifdef CONFIG_DRM_AMDGPU_CIK
6441 case CHIP_BONAIRE:
6442 case CHIP_HAWAII:
6443 case CHIP_KAVERI:
6444 case CHIP_KABINI:
6445 case CHIP_MULLINS:
6446#endif
6447 case CHIP_TONGA:
6448 case CHIP_FIJI:
6449 case CHIP_POLARIS10:
6450 case CHIP_POLARIS11:
6451 case CHIP_POLARIS12:
6452 case CHIP_VEGAM:
6453 case CHIP_CARRIZO:
6454 case CHIP_STONEY:
6455 /* chips with display hardware */
6456 return true;
6457 default:
6458 /* IP discovery */
6459 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6460 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6461 return false;
6462 return true;
6463 }
6464}
6465
6466uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6467 uint32_t inst, uint32_t reg_addr, char reg_name[],
6468 uint32_t expected_value, uint32_t mask)
6469{
6470 uint32_t ret = 0;
6471 uint32_t old_ = 0;
6472 uint32_t tmp_ = RREG32(reg_addr);
6473 uint32_t loop = adev->usec_timeout;
6474
6475 while ((tmp_ & (mask)) != (expected_value)) {
6476 if (old_ != tmp_) {
6477 loop = adev->usec_timeout;
6478 old_ = tmp_;
6479 } else
6480 udelay(1);
6481 tmp_ = RREG32(reg_addr);
6482 loop--;
6483 if (!loop) {
6484 DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6485 inst, reg_name, (uint32_t)expected_value,
6486 (uint32_t)(tmp_ & (mask)));
6487 ret = -ETIMEDOUT;
6488 break;
6489 }
6490 }
6491 return ret;
6492}
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/power_supply.h>
29#include <linux/kthread.h>
30#include <linux/module.h>
31#include <linux/console.h>
32#include <linux/slab.h>
33
34#include <drm/drm_atomic_helper.h>
35#include <drm/drm_probe_helper.h>
36#include <drm/amdgpu_drm.h>
37#include <linux/vgaarb.h>
38#include <linux/vga_switcheroo.h>
39#include <linux/efi.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_i2c.h"
43#include "atom.h"
44#include "amdgpu_atombios.h"
45#include "amdgpu_atomfirmware.h"
46#include "amd_pcie.h"
47#ifdef CONFIG_DRM_AMDGPU_SI
48#include "si.h"
49#endif
50#ifdef CONFIG_DRM_AMDGPU_CIK
51#include "cik.h"
52#endif
53#include "vi.h"
54#include "soc15.h"
55#include "nv.h"
56#include "bif/bif_4_1_d.h"
57#include <linux/pci.h>
58#include <linux/firmware.h>
59#include "amdgpu_vf_error.h"
60
61#include "amdgpu_amdkfd.h"
62#include "amdgpu_pm.h"
63
64#include "amdgpu_xgmi.h"
65#include "amdgpu_ras.h"
66#include "amdgpu_pmu.h"
67
68MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
69MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
70MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
71MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
72MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
73MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
74MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
75MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
76MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
77MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
78
79#define AMDGPU_RESUME_MS 2000
80
81static const char *amdgpu_asic_name[] = {
82 "TAHITI",
83 "PITCAIRN",
84 "VERDE",
85 "OLAND",
86 "HAINAN",
87 "BONAIRE",
88 "KAVERI",
89 "KABINI",
90 "HAWAII",
91 "MULLINS",
92 "TOPAZ",
93 "TONGA",
94 "FIJI",
95 "CARRIZO",
96 "STONEY",
97 "POLARIS10",
98 "POLARIS11",
99 "POLARIS12",
100 "VEGAM",
101 "VEGA10",
102 "VEGA12",
103 "VEGA20",
104 "RAVEN",
105 "ARCTURUS",
106 "RENOIR",
107 "NAVI10",
108 "NAVI14",
109 "NAVI12",
110 "LAST",
111};
112
113/**
114 * DOC: pcie_replay_count
115 *
116 * The amdgpu driver provides a sysfs API for reporting the total number
117 * of PCIe replays (NAKs)
118 * The file pcie_replay_count is used for this and returns the total
119 * number of replays as a sum of the NAKs generated and NAKs received
120 */
121
122static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
123 struct device_attribute *attr, char *buf)
124{
125 struct drm_device *ddev = dev_get_drvdata(dev);
126 struct amdgpu_device *adev = ddev->dev_private;
127 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
128
129 return snprintf(buf, PAGE_SIZE, "%llu\n", cnt);
130}
131
132static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
133 amdgpu_device_get_pcie_replay_count, NULL);
134
135static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
136
137/**
138 * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control
139 *
140 * @dev: drm_device pointer
141 *
142 * Returns true if the device is a dGPU with HG/PX power control,
143 * otherwise return false.
144 */
145bool amdgpu_device_is_px(struct drm_device *dev)
146{
147 struct amdgpu_device *adev = dev->dev_private;
148
149 if (adev->flags & AMD_IS_PX)
150 return true;
151 return false;
152}
153
154/*
155 * MMIO register access helper functions.
156 */
157/**
158 * amdgpu_mm_rreg - read a memory mapped IO register
159 *
160 * @adev: amdgpu_device pointer
161 * @reg: dword aligned register offset
162 * @acc_flags: access flags which require special behavior
163 *
164 * Returns the 32 bit value from the offset specified.
165 */
166uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
167 uint32_t acc_flags)
168{
169 uint32_t ret;
170
171 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
172 return amdgpu_virt_kiq_rreg(adev, reg);
173
174 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
175 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
176 else {
177 unsigned long flags;
178
179 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
180 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
181 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
182 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
183 }
184 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
185 return ret;
186}
187
188/*
189 * MMIO register read with bytes helper functions
190 * @offset:bytes offset from MMIO start
191 *
192*/
193
194/**
195 * amdgpu_mm_rreg8 - read a memory mapped IO register
196 *
197 * @adev: amdgpu_device pointer
198 * @offset: byte aligned register offset
199 *
200 * Returns the 8 bit value from the offset specified.
201 */
202uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
203 if (offset < adev->rmmio_size)
204 return (readb(adev->rmmio + offset));
205 BUG();
206}
207
208/*
209 * MMIO register write with bytes helper functions
210 * @offset:bytes offset from MMIO start
211 * @value: the value want to be written to the register
212 *
213*/
214/**
215 * amdgpu_mm_wreg8 - read a memory mapped IO register
216 *
217 * @adev: amdgpu_device pointer
218 * @offset: byte aligned register offset
219 * @value: 8 bit value to write
220 *
221 * Writes the value specified to the offset specified.
222 */
223void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
224 if (offset < adev->rmmio_size)
225 writeb(value, adev->rmmio + offset);
226 else
227 BUG();
228}
229
230/**
231 * amdgpu_mm_wreg - write to a memory mapped IO register
232 *
233 * @adev: amdgpu_device pointer
234 * @reg: dword aligned register offset
235 * @v: 32 bit value to write to the register
236 * @acc_flags: access flags which require special behavior
237 *
238 * Writes the value specified to the offset specified.
239 */
240void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
241 uint32_t acc_flags)
242{
243 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
244
245 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
246 adev->last_mm_index = v;
247 }
248
249 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
250 return amdgpu_virt_kiq_wreg(adev, reg, v);
251
252 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
253 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
254 else {
255 unsigned long flags;
256
257 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
258 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
259 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
260 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
261 }
262
263 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
264 udelay(500);
265 }
266}
267
268/**
269 * amdgpu_io_rreg - read an IO register
270 *
271 * @adev: amdgpu_device pointer
272 * @reg: dword aligned register offset
273 *
274 * Returns the 32 bit value from the offset specified.
275 */
276u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
277{
278 if ((reg * 4) < adev->rio_mem_size)
279 return ioread32(adev->rio_mem + (reg * 4));
280 else {
281 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
282 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
283 }
284}
285
286/**
287 * amdgpu_io_wreg - write to an IO register
288 *
289 * @adev: amdgpu_device pointer
290 * @reg: dword aligned register offset
291 * @v: 32 bit value to write to the register
292 *
293 * Writes the value specified to the offset specified.
294 */
295void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
296{
297 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
298 adev->last_mm_index = v;
299 }
300
301 if ((reg * 4) < adev->rio_mem_size)
302 iowrite32(v, adev->rio_mem + (reg * 4));
303 else {
304 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
305 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
306 }
307
308 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
309 udelay(500);
310 }
311}
312
313/**
314 * amdgpu_mm_rdoorbell - read a doorbell dword
315 *
316 * @adev: amdgpu_device pointer
317 * @index: doorbell index
318 *
319 * Returns the value in the doorbell aperture at the
320 * requested doorbell index (CIK).
321 */
322u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
323{
324 if (index < adev->doorbell.num_doorbells) {
325 return readl(adev->doorbell.ptr + index);
326 } else {
327 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
328 return 0;
329 }
330}
331
332/**
333 * amdgpu_mm_wdoorbell - write a doorbell dword
334 *
335 * @adev: amdgpu_device pointer
336 * @index: doorbell index
337 * @v: value to write
338 *
339 * Writes @v to the doorbell aperture at the
340 * requested doorbell index (CIK).
341 */
342void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
343{
344 if (index < adev->doorbell.num_doorbells) {
345 writel(v, adev->doorbell.ptr + index);
346 } else {
347 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
348 }
349}
350
351/**
352 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
353 *
354 * @adev: amdgpu_device pointer
355 * @index: doorbell index
356 *
357 * Returns the value in the doorbell aperture at the
358 * requested doorbell index (VEGA10+).
359 */
360u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
361{
362 if (index < adev->doorbell.num_doorbells) {
363 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
364 } else {
365 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
366 return 0;
367 }
368}
369
370/**
371 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
372 *
373 * @adev: amdgpu_device pointer
374 * @index: doorbell index
375 * @v: value to write
376 *
377 * Writes @v to the doorbell aperture at the
378 * requested doorbell index (VEGA10+).
379 */
380void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
381{
382 if (index < adev->doorbell.num_doorbells) {
383 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
384 } else {
385 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
386 }
387}
388
389/**
390 * amdgpu_invalid_rreg - dummy reg read function
391 *
392 * @adev: amdgpu device pointer
393 * @reg: offset of register
394 *
395 * Dummy register read function. Used for register blocks
396 * that certain asics don't have (all asics).
397 * Returns the value in the register.
398 */
399static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
400{
401 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
402 BUG();
403 return 0;
404}
405
406/**
407 * amdgpu_invalid_wreg - dummy reg write function
408 *
409 * @adev: amdgpu device pointer
410 * @reg: offset of register
411 * @v: value to write to the register
412 *
413 * Dummy register read function. Used for register blocks
414 * that certain asics don't have (all asics).
415 */
416static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
417{
418 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
419 reg, v);
420 BUG();
421}
422
423/**
424 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
425 *
426 * @adev: amdgpu device pointer
427 * @reg: offset of register
428 *
429 * Dummy register read function. Used for register blocks
430 * that certain asics don't have (all asics).
431 * Returns the value in the register.
432 */
433static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
434{
435 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
436 BUG();
437 return 0;
438}
439
440/**
441 * amdgpu_invalid_wreg64 - dummy reg write function
442 *
443 * @adev: amdgpu device pointer
444 * @reg: offset of register
445 * @v: value to write to the register
446 *
447 * Dummy register read function. Used for register blocks
448 * that certain asics don't have (all asics).
449 */
450static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
451{
452 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
453 reg, v);
454 BUG();
455}
456
457/**
458 * amdgpu_block_invalid_rreg - dummy reg read function
459 *
460 * @adev: amdgpu device pointer
461 * @block: offset of instance
462 * @reg: offset of register
463 *
464 * Dummy register read function. Used for register blocks
465 * that certain asics don't have (all asics).
466 * Returns the value in the register.
467 */
468static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
469 uint32_t block, uint32_t reg)
470{
471 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
472 reg, block);
473 BUG();
474 return 0;
475}
476
477/**
478 * amdgpu_block_invalid_wreg - dummy reg write function
479 *
480 * @adev: amdgpu device pointer
481 * @block: offset of instance
482 * @reg: offset of register
483 * @v: value to write to the register
484 *
485 * Dummy register read function. Used for register blocks
486 * that certain asics don't have (all asics).
487 */
488static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
489 uint32_t block,
490 uint32_t reg, uint32_t v)
491{
492 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
493 reg, block, v);
494 BUG();
495}
496
497/**
498 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
499 *
500 * @adev: amdgpu device pointer
501 *
502 * Allocates a scratch page of VRAM for use by various things in the
503 * driver.
504 */
505static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
506{
507 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
508 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
509 &adev->vram_scratch.robj,
510 &adev->vram_scratch.gpu_addr,
511 (void **)&adev->vram_scratch.ptr);
512}
513
514/**
515 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
516 *
517 * @adev: amdgpu device pointer
518 *
519 * Frees the VRAM scratch page.
520 */
521static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
522{
523 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
524}
525
526/**
527 * amdgpu_device_program_register_sequence - program an array of registers.
528 *
529 * @adev: amdgpu_device pointer
530 * @registers: pointer to the register array
531 * @array_size: size of the register array
532 *
533 * Programs an array or registers with and and or masks.
534 * This is a helper for setting golden registers.
535 */
536void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
537 const u32 *registers,
538 const u32 array_size)
539{
540 u32 tmp, reg, and_mask, or_mask;
541 int i;
542
543 if (array_size % 3)
544 return;
545
546 for (i = 0; i < array_size; i +=3) {
547 reg = registers[i + 0];
548 and_mask = registers[i + 1];
549 or_mask = registers[i + 2];
550
551 if (and_mask == 0xffffffff) {
552 tmp = or_mask;
553 } else {
554 tmp = RREG32(reg);
555 tmp &= ~and_mask;
556 if (adev->family >= AMDGPU_FAMILY_AI)
557 tmp |= (or_mask & and_mask);
558 else
559 tmp |= or_mask;
560 }
561 WREG32(reg, tmp);
562 }
563}
564
565/**
566 * amdgpu_device_pci_config_reset - reset the GPU
567 *
568 * @adev: amdgpu_device pointer
569 *
570 * Resets the GPU using the pci config reset sequence.
571 * Only applicable to asics prior to vega10.
572 */
573void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
574{
575 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
576}
577
578/*
579 * GPU doorbell aperture helpers function.
580 */
581/**
582 * amdgpu_device_doorbell_init - Init doorbell driver information.
583 *
584 * @adev: amdgpu_device pointer
585 *
586 * Init doorbell driver information (CIK)
587 * Returns 0 on success, error on failure.
588 */
589static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
590{
591
592 /* No doorbell on SI hardware generation */
593 if (adev->asic_type < CHIP_BONAIRE) {
594 adev->doorbell.base = 0;
595 adev->doorbell.size = 0;
596 adev->doorbell.num_doorbells = 0;
597 adev->doorbell.ptr = NULL;
598 return 0;
599 }
600
601 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
602 return -EINVAL;
603
604 amdgpu_asic_init_doorbell_index(adev);
605
606 /* doorbell bar mapping */
607 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
608 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
609
610 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
611 adev->doorbell_index.max_assignment+1);
612 if (adev->doorbell.num_doorbells == 0)
613 return -EINVAL;
614
615 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
616 * paging queue doorbell use the second page. The
617 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
618 * doorbells are in the first page. So with paging queue enabled,
619 * the max num_doorbells should + 1 page (0x400 in dword)
620 */
621 if (adev->asic_type >= CHIP_VEGA10)
622 adev->doorbell.num_doorbells += 0x400;
623
624 adev->doorbell.ptr = ioremap(adev->doorbell.base,
625 adev->doorbell.num_doorbells *
626 sizeof(u32));
627 if (adev->doorbell.ptr == NULL)
628 return -ENOMEM;
629
630 return 0;
631}
632
633/**
634 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
635 *
636 * @adev: amdgpu_device pointer
637 *
638 * Tear down doorbell driver information (CIK)
639 */
640static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
641{
642 iounmap(adev->doorbell.ptr);
643 adev->doorbell.ptr = NULL;
644}
645
646
647
648/*
649 * amdgpu_device_wb_*()
650 * Writeback is the method by which the GPU updates special pages in memory
651 * with the status of certain GPU events (fences, ring pointers,etc.).
652 */
653
654/**
655 * amdgpu_device_wb_fini - Disable Writeback and free memory
656 *
657 * @adev: amdgpu_device pointer
658 *
659 * Disables Writeback and frees the Writeback memory (all asics).
660 * Used at driver shutdown.
661 */
662static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
663{
664 if (adev->wb.wb_obj) {
665 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
666 &adev->wb.gpu_addr,
667 (void **)&adev->wb.wb);
668 adev->wb.wb_obj = NULL;
669 }
670}
671
672/**
673 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
674 *
675 * @adev: amdgpu_device pointer
676 *
677 * Initializes writeback and allocates writeback memory (all asics).
678 * Used at driver startup.
679 * Returns 0 on success or an -error on failure.
680 */
681static int amdgpu_device_wb_init(struct amdgpu_device *adev)
682{
683 int r;
684
685 if (adev->wb.wb_obj == NULL) {
686 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
687 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
688 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
689 &adev->wb.wb_obj, &adev->wb.gpu_addr,
690 (void **)&adev->wb.wb);
691 if (r) {
692 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
693 return r;
694 }
695
696 adev->wb.num_wb = AMDGPU_MAX_WB;
697 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
698
699 /* clear wb memory */
700 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
701 }
702
703 return 0;
704}
705
706/**
707 * amdgpu_device_wb_get - Allocate a wb entry
708 *
709 * @adev: amdgpu_device pointer
710 * @wb: wb index
711 *
712 * Allocate a wb slot for use by the driver (all asics).
713 * Returns 0 on success or -EINVAL on failure.
714 */
715int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
716{
717 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
718
719 if (offset < adev->wb.num_wb) {
720 __set_bit(offset, adev->wb.used);
721 *wb = offset << 3; /* convert to dw offset */
722 return 0;
723 } else {
724 return -EINVAL;
725 }
726}
727
728/**
729 * amdgpu_device_wb_free - Free a wb entry
730 *
731 * @adev: amdgpu_device pointer
732 * @wb: wb index
733 *
734 * Free a wb slot allocated for use by the driver (all asics)
735 */
736void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
737{
738 wb >>= 3;
739 if (wb < adev->wb.num_wb)
740 __clear_bit(wb, adev->wb.used);
741}
742
743/**
744 * amdgpu_device_resize_fb_bar - try to resize FB BAR
745 *
746 * @adev: amdgpu_device pointer
747 *
748 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
749 * to fail, but if any of the BARs is not accessible after the size we abort
750 * driver loading by returning -ENODEV.
751 */
752int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
753{
754 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
755 u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
756 struct pci_bus *root;
757 struct resource *res;
758 unsigned i;
759 u16 cmd;
760 int r;
761
762 /* Bypass for VF */
763 if (amdgpu_sriov_vf(adev))
764 return 0;
765
766 /* Check if the root BUS has 64bit memory resources */
767 root = adev->pdev->bus;
768 while (root->parent)
769 root = root->parent;
770
771 pci_bus_for_each_resource(root, res, i) {
772 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
773 res->start > 0x100000000ull)
774 break;
775 }
776
777 /* Trying to resize is pointless without a root hub window above 4GB */
778 if (!res)
779 return 0;
780
781 /* Disable memory decoding while we change the BAR addresses and size */
782 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
783 pci_write_config_word(adev->pdev, PCI_COMMAND,
784 cmd & ~PCI_COMMAND_MEMORY);
785
786 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
787 amdgpu_device_doorbell_fini(adev);
788 if (adev->asic_type >= CHIP_BONAIRE)
789 pci_release_resource(adev->pdev, 2);
790
791 pci_release_resource(adev->pdev, 0);
792
793 r = pci_resize_resource(adev->pdev, 0, rbar_size);
794 if (r == -ENOSPC)
795 DRM_INFO("Not enough PCI address space for a large BAR.");
796 else if (r && r != -ENOTSUPP)
797 DRM_ERROR("Problem resizing BAR0 (%d).", r);
798
799 pci_assign_unassigned_bus_resources(adev->pdev->bus);
800
801 /* When the doorbell or fb BAR isn't available we have no chance of
802 * using the device.
803 */
804 r = amdgpu_device_doorbell_init(adev);
805 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
806 return -ENODEV;
807
808 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
809
810 return 0;
811}
812
813/*
814 * GPU helpers function.
815 */
816/**
817 * amdgpu_device_need_post - check if the hw need post or not
818 *
819 * @adev: amdgpu_device pointer
820 *
821 * Check if the asic has been initialized (all asics) at driver startup
822 * or post is needed if hw reset is performed.
823 * Returns true if need or false if not.
824 */
825bool amdgpu_device_need_post(struct amdgpu_device *adev)
826{
827 uint32_t reg;
828
829 if (amdgpu_sriov_vf(adev))
830 return false;
831
832 if (amdgpu_passthrough(adev)) {
833 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
834 * some old smc fw still need driver do vPost otherwise gpu hang, while
835 * those smc fw version above 22.15 doesn't have this flaw, so we force
836 * vpost executed for smc version below 22.15
837 */
838 if (adev->asic_type == CHIP_FIJI) {
839 int err;
840 uint32_t fw_ver;
841 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
842 /* force vPost if error occured */
843 if (err)
844 return true;
845
846 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
847 if (fw_ver < 0x00160e00)
848 return true;
849 }
850 }
851
852 if (adev->has_hw_reset) {
853 adev->has_hw_reset = false;
854 return true;
855 }
856
857 /* bios scratch used on CIK+ */
858 if (adev->asic_type >= CHIP_BONAIRE)
859 return amdgpu_atombios_scratch_need_asic_init(adev);
860
861 /* check MEM_SIZE for older asics */
862 reg = amdgpu_asic_get_config_memsize(adev);
863
864 if ((reg != 0) && (reg != 0xffffffff))
865 return false;
866
867 return true;
868}
869
870/* if we get transitioned to only one device, take VGA back */
871/**
872 * amdgpu_device_vga_set_decode - enable/disable vga decode
873 *
874 * @cookie: amdgpu_device pointer
875 * @state: enable/disable vga decode
876 *
877 * Enable/disable vga decode (all asics).
878 * Returns VGA resource flags.
879 */
880static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
881{
882 struct amdgpu_device *adev = cookie;
883 amdgpu_asic_set_vga_state(adev, state);
884 if (state)
885 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
886 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
887 else
888 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
889}
890
891/**
892 * amdgpu_device_check_block_size - validate the vm block size
893 *
894 * @adev: amdgpu_device pointer
895 *
896 * Validates the vm block size specified via module parameter.
897 * The vm block size defines number of bits in page table versus page directory,
898 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
899 * page table and the remaining bits are in the page directory.
900 */
901static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
902{
903 /* defines number of bits in page table versus page directory,
904 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
905 * page table and the remaining bits are in the page directory */
906 if (amdgpu_vm_block_size == -1)
907 return;
908
909 if (amdgpu_vm_block_size < 9) {
910 dev_warn(adev->dev, "VM page table size (%d) too small\n",
911 amdgpu_vm_block_size);
912 amdgpu_vm_block_size = -1;
913 }
914}
915
916/**
917 * amdgpu_device_check_vm_size - validate the vm size
918 *
919 * @adev: amdgpu_device pointer
920 *
921 * Validates the vm size in GB specified via module parameter.
922 * The VM size is the size of the GPU virtual memory space in GB.
923 */
924static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
925{
926 /* no need to check the default value */
927 if (amdgpu_vm_size == -1)
928 return;
929
930 if (amdgpu_vm_size < 1) {
931 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
932 amdgpu_vm_size);
933 amdgpu_vm_size = -1;
934 }
935}
936
937static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
938{
939 struct sysinfo si;
940 bool is_os_64 = (sizeof(void *) == 8) ? true : false;
941 uint64_t total_memory;
942 uint64_t dram_size_seven_GB = 0x1B8000000;
943 uint64_t dram_size_three_GB = 0xB8000000;
944
945 if (amdgpu_smu_memory_pool_size == 0)
946 return;
947
948 if (!is_os_64) {
949 DRM_WARN("Not 64-bit OS, feature not supported\n");
950 goto def_value;
951 }
952 si_meminfo(&si);
953 total_memory = (uint64_t)si.totalram * si.mem_unit;
954
955 if ((amdgpu_smu_memory_pool_size == 1) ||
956 (amdgpu_smu_memory_pool_size == 2)) {
957 if (total_memory < dram_size_three_GB)
958 goto def_value1;
959 } else if ((amdgpu_smu_memory_pool_size == 4) ||
960 (amdgpu_smu_memory_pool_size == 8)) {
961 if (total_memory < dram_size_seven_GB)
962 goto def_value1;
963 } else {
964 DRM_WARN("Smu memory pool size not supported\n");
965 goto def_value;
966 }
967 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
968
969 return;
970
971def_value1:
972 DRM_WARN("No enough system memory\n");
973def_value:
974 adev->pm.smu_prv_buffer_size = 0;
975}
976
977/**
978 * amdgpu_device_check_arguments - validate module params
979 *
980 * @adev: amdgpu_device pointer
981 *
982 * Validates certain module parameters and updates
983 * the associated values used by the driver (all asics).
984 */
985static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
986{
987 int ret = 0;
988
989 if (amdgpu_sched_jobs < 4) {
990 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
991 amdgpu_sched_jobs);
992 amdgpu_sched_jobs = 4;
993 } else if (!is_power_of_2(amdgpu_sched_jobs)){
994 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
995 amdgpu_sched_jobs);
996 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
997 }
998
999 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1000 /* gart size must be greater or equal to 32M */
1001 dev_warn(adev->dev, "gart size (%d) too small\n",
1002 amdgpu_gart_size);
1003 amdgpu_gart_size = -1;
1004 }
1005
1006 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1007 /* gtt size must be greater or equal to 32M */
1008 dev_warn(adev->dev, "gtt size (%d) too small\n",
1009 amdgpu_gtt_size);
1010 amdgpu_gtt_size = -1;
1011 }
1012
1013 /* valid range is between 4 and 9 inclusive */
1014 if (amdgpu_vm_fragment_size != -1 &&
1015 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1016 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1017 amdgpu_vm_fragment_size = -1;
1018 }
1019
1020 amdgpu_device_check_smu_prv_buffer_size(adev);
1021
1022 amdgpu_device_check_vm_size(adev);
1023
1024 amdgpu_device_check_block_size(adev);
1025
1026 ret = amdgpu_device_get_job_timeout_settings(adev);
1027 if (ret) {
1028 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
1029 return ret;
1030 }
1031
1032 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1033
1034 return ret;
1035}
1036
1037/**
1038 * amdgpu_switcheroo_set_state - set switcheroo state
1039 *
1040 * @pdev: pci dev pointer
1041 * @state: vga_switcheroo state
1042 *
1043 * Callback for the switcheroo driver. Suspends or resumes the
1044 * the asics before or after it is powered up using ACPI methods.
1045 */
1046static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1047{
1048 struct drm_device *dev = pci_get_drvdata(pdev);
1049
1050 if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1051 return;
1052
1053 if (state == VGA_SWITCHEROO_ON) {
1054 pr_info("amdgpu: switched on\n");
1055 /* don't suspend or resume card normally */
1056 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1057
1058 amdgpu_device_resume(dev, true, true);
1059
1060 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1061 drm_kms_helper_poll_enable(dev);
1062 } else {
1063 pr_info("amdgpu: switched off\n");
1064 drm_kms_helper_poll_disable(dev);
1065 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1066 amdgpu_device_suspend(dev, true, true);
1067 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1068 }
1069}
1070
1071/**
1072 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1073 *
1074 * @pdev: pci dev pointer
1075 *
1076 * Callback for the switcheroo driver. Check of the switcheroo
1077 * state can be changed.
1078 * Returns true if the state can be changed, false if not.
1079 */
1080static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1081{
1082 struct drm_device *dev = pci_get_drvdata(pdev);
1083
1084 /*
1085 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1086 * locking inversion with the driver load path. And the access here is
1087 * completely racy anyway. So don't bother with locking for now.
1088 */
1089 return dev->open_count == 0;
1090}
1091
1092static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1093 .set_gpu_state = amdgpu_switcheroo_set_state,
1094 .reprobe = NULL,
1095 .can_switch = amdgpu_switcheroo_can_switch,
1096};
1097
1098/**
1099 * amdgpu_device_ip_set_clockgating_state - set the CG state
1100 *
1101 * @dev: amdgpu_device pointer
1102 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1103 * @state: clockgating state (gate or ungate)
1104 *
1105 * Sets the requested clockgating state for all instances of
1106 * the hardware IP specified.
1107 * Returns the error code from the last instance.
1108 */
1109int amdgpu_device_ip_set_clockgating_state(void *dev,
1110 enum amd_ip_block_type block_type,
1111 enum amd_clockgating_state state)
1112{
1113 struct amdgpu_device *adev = dev;
1114 int i, r = 0;
1115
1116 for (i = 0; i < adev->num_ip_blocks; i++) {
1117 if (!adev->ip_blocks[i].status.valid)
1118 continue;
1119 if (adev->ip_blocks[i].version->type != block_type)
1120 continue;
1121 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1122 continue;
1123 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1124 (void *)adev, state);
1125 if (r)
1126 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1127 adev->ip_blocks[i].version->funcs->name, r);
1128 }
1129 return r;
1130}
1131
1132/**
1133 * amdgpu_device_ip_set_powergating_state - set the PG state
1134 *
1135 * @dev: amdgpu_device pointer
1136 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1137 * @state: powergating state (gate or ungate)
1138 *
1139 * Sets the requested powergating state for all instances of
1140 * the hardware IP specified.
1141 * Returns the error code from the last instance.
1142 */
1143int amdgpu_device_ip_set_powergating_state(void *dev,
1144 enum amd_ip_block_type block_type,
1145 enum amd_powergating_state state)
1146{
1147 struct amdgpu_device *adev = dev;
1148 int i, r = 0;
1149
1150 for (i = 0; i < adev->num_ip_blocks; i++) {
1151 if (!adev->ip_blocks[i].status.valid)
1152 continue;
1153 if (adev->ip_blocks[i].version->type != block_type)
1154 continue;
1155 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1156 continue;
1157 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1158 (void *)adev, state);
1159 if (r)
1160 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1161 adev->ip_blocks[i].version->funcs->name, r);
1162 }
1163 return r;
1164}
1165
1166/**
1167 * amdgpu_device_ip_get_clockgating_state - get the CG state
1168 *
1169 * @adev: amdgpu_device pointer
1170 * @flags: clockgating feature flags
1171 *
1172 * Walks the list of IPs on the device and updates the clockgating
1173 * flags for each IP.
1174 * Updates @flags with the feature flags for each hardware IP where
1175 * clockgating is enabled.
1176 */
1177void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1178 u32 *flags)
1179{
1180 int i;
1181
1182 for (i = 0; i < adev->num_ip_blocks; i++) {
1183 if (!adev->ip_blocks[i].status.valid)
1184 continue;
1185 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1186 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1187 }
1188}
1189
1190/**
1191 * amdgpu_device_ip_wait_for_idle - wait for idle
1192 *
1193 * @adev: amdgpu_device pointer
1194 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1195 *
1196 * Waits for the request hardware IP to be idle.
1197 * Returns 0 for success or a negative error code on failure.
1198 */
1199int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1200 enum amd_ip_block_type block_type)
1201{
1202 int i, r;
1203
1204 for (i = 0; i < adev->num_ip_blocks; i++) {
1205 if (!adev->ip_blocks[i].status.valid)
1206 continue;
1207 if (adev->ip_blocks[i].version->type == block_type) {
1208 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1209 if (r)
1210 return r;
1211 break;
1212 }
1213 }
1214 return 0;
1215
1216}
1217
1218/**
1219 * amdgpu_device_ip_is_idle - is the hardware IP idle
1220 *
1221 * @adev: amdgpu_device pointer
1222 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1223 *
1224 * Check if the hardware IP is idle or not.
1225 * Returns true if it the IP is idle, false if not.
1226 */
1227bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1228 enum amd_ip_block_type block_type)
1229{
1230 int i;
1231
1232 for (i = 0; i < adev->num_ip_blocks; i++) {
1233 if (!adev->ip_blocks[i].status.valid)
1234 continue;
1235 if (adev->ip_blocks[i].version->type == block_type)
1236 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1237 }
1238 return true;
1239
1240}
1241
1242/**
1243 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1244 *
1245 * @adev: amdgpu_device pointer
1246 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1247 *
1248 * Returns a pointer to the hardware IP block structure
1249 * if it exists for the asic, otherwise NULL.
1250 */
1251struct amdgpu_ip_block *
1252amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1253 enum amd_ip_block_type type)
1254{
1255 int i;
1256
1257 for (i = 0; i < adev->num_ip_blocks; i++)
1258 if (adev->ip_blocks[i].version->type == type)
1259 return &adev->ip_blocks[i];
1260
1261 return NULL;
1262}
1263
1264/**
1265 * amdgpu_device_ip_block_version_cmp
1266 *
1267 * @adev: amdgpu_device pointer
1268 * @type: enum amd_ip_block_type
1269 * @major: major version
1270 * @minor: minor version
1271 *
1272 * return 0 if equal or greater
1273 * return 1 if smaller or the ip_block doesn't exist
1274 */
1275int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1276 enum amd_ip_block_type type,
1277 u32 major, u32 minor)
1278{
1279 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1280
1281 if (ip_block && ((ip_block->version->major > major) ||
1282 ((ip_block->version->major == major) &&
1283 (ip_block->version->minor >= minor))))
1284 return 0;
1285
1286 return 1;
1287}
1288
1289/**
1290 * amdgpu_device_ip_block_add
1291 *
1292 * @adev: amdgpu_device pointer
1293 * @ip_block_version: pointer to the IP to add
1294 *
1295 * Adds the IP block driver information to the collection of IPs
1296 * on the asic.
1297 */
1298int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1299 const struct amdgpu_ip_block_version *ip_block_version)
1300{
1301 if (!ip_block_version)
1302 return -EINVAL;
1303
1304 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1305 ip_block_version->funcs->name);
1306
1307 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1308
1309 return 0;
1310}
1311
1312/**
1313 * amdgpu_device_enable_virtual_display - enable virtual display feature
1314 *
1315 * @adev: amdgpu_device pointer
1316 *
1317 * Enabled the virtual display feature if the user has enabled it via
1318 * the module parameter virtual_display. This feature provides a virtual
1319 * display hardware on headless boards or in virtualized environments.
1320 * This function parses and validates the configuration string specified by
1321 * the user and configues the virtual display configuration (number of
1322 * virtual connectors, crtcs, etc.) specified.
1323 */
1324static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1325{
1326 adev->enable_virtual_display = false;
1327
1328 if (amdgpu_virtual_display) {
1329 struct drm_device *ddev = adev->ddev;
1330 const char *pci_address_name = pci_name(ddev->pdev);
1331 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1332
1333 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1334 pciaddstr_tmp = pciaddstr;
1335 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1336 pciaddname = strsep(&pciaddname_tmp, ",");
1337 if (!strcmp("all", pciaddname)
1338 || !strcmp(pci_address_name, pciaddname)) {
1339 long num_crtc;
1340 int res = -1;
1341
1342 adev->enable_virtual_display = true;
1343
1344 if (pciaddname_tmp)
1345 res = kstrtol(pciaddname_tmp, 10,
1346 &num_crtc);
1347
1348 if (!res) {
1349 if (num_crtc < 1)
1350 num_crtc = 1;
1351 if (num_crtc > 6)
1352 num_crtc = 6;
1353 adev->mode_info.num_crtc = num_crtc;
1354 } else {
1355 adev->mode_info.num_crtc = 1;
1356 }
1357 break;
1358 }
1359 }
1360
1361 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1362 amdgpu_virtual_display, pci_address_name,
1363 adev->enable_virtual_display, adev->mode_info.num_crtc);
1364
1365 kfree(pciaddstr);
1366 }
1367}
1368
1369/**
1370 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1371 *
1372 * @adev: amdgpu_device pointer
1373 *
1374 * Parses the asic configuration parameters specified in the gpu info
1375 * firmware and makes them availale to the driver for use in configuring
1376 * the asic.
1377 * Returns 0 on success, -EINVAL on failure.
1378 */
1379static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1380{
1381 const char *chip_name;
1382 char fw_name[30];
1383 int err;
1384 const struct gpu_info_firmware_header_v1_0 *hdr;
1385
1386 adev->firmware.gpu_info_fw = NULL;
1387
1388 switch (adev->asic_type) {
1389 case CHIP_TOPAZ:
1390 case CHIP_TONGA:
1391 case CHIP_FIJI:
1392 case CHIP_POLARIS10:
1393 case CHIP_POLARIS11:
1394 case CHIP_POLARIS12:
1395 case CHIP_VEGAM:
1396 case CHIP_CARRIZO:
1397 case CHIP_STONEY:
1398#ifdef CONFIG_DRM_AMDGPU_SI
1399 case CHIP_VERDE:
1400 case CHIP_TAHITI:
1401 case CHIP_PITCAIRN:
1402 case CHIP_OLAND:
1403 case CHIP_HAINAN:
1404#endif
1405#ifdef CONFIG_DRM_AMDGPU_CIK
1406 case CHIP_BONAIRE:
1407 case CHIP_HAWAII:
1408 case CHIP_KAVERI:
1409 case CHIP_KABINI:
1410 case CHIP_MULLINS:
1411#endif
1412 case CHIP_VEGA20:
1413 default:
1414 return 0;
1415 case CHIP_VEGA10:
1416 chip_name = "vega10";
1417 break;
1418 case CHIP_VEGA12:
1419 chip_name = "vega12";
1420 break;
1421 case CHIP_RAVEN:
1422 if (adev->rev_id >= 8)
1423 chip_name = "raven2";
1424 else if (adev->pdev->device == 0x15d8)
1425 chip_name = "picasso";
1426 else
1427 chip_name = "raven";
1428 break;
1429 case CHIP_ARCTURUS:
1430 chip_name = "arcturus";
1431 break;
1432 case CHIP_RENOIR:
1433 chip_name = "renoir";
1434 break;
1435 case CHIP_NAVI10:
1436 chip_name = "navi10";
1437 break;
1438 case CHIP_NAVI14:
1439 chip_name = "navi14";
1440 break;
1441 case CHIP_NAVI12:
1442 chip_name = "navi12";
1443 break;
1444 }
1445
1446 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1447 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1448 if (err) {
1449 dev_err(adev->dev,
1450 "Failed to load gpu_info firmware \"%s\"\n",
1451 fw_name);
1452 goto out;
1453 }
1454 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1455 if (err) {
1456 dev_err(adev->dev,
1457 "Failed to validate gpu_info firmware \"%s\"\n",
1458 fw_name);
1459 goto out;
1460 }
1461
1462 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1463 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1464
1465 switch (hdr->version_major) {
1466 case 1:
1467 {
1468 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1469 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1470 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1471
1472 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1473 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1474 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1475 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1476 adev->gfx.config.max_texture_channel_caches =
1477 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1478 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1479 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1480 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1481 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1482 adev->gfx.config.double_offchip_lds_buf =
1483 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1484 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1485 adev->gfx.cu_info.max_waves_per_simd =
1486 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1487 adev->gfx.cu_info.max_scratch_slots_per_cu =
1488 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1489 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1490 if (hdr->version_minor >= 1) {
1491 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1492 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1493 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1494 adev->gfx.config.num_sc_per_sh =
1495 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1496 adev->gfx.config.num_packer_per_sc =
1497 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1498 }
1499#ifdef CONFIG_DRM_AMD_DC_DCN2_0
1500 if (hdr->version_minor == 2) {
1501 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1502 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1503 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1504 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1505 }
1506#endif
1507 break;
1508 }
1509 default:
1510 dev_err(adev->dev,
1511 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1512 err = -EINVAL;
1513 goto out;
1514 }
1515out:
1516 return err;
1517}
1518
1519/**
1520 * amdgpu_device_ip_early_init - run early init for hardware IPs
1521 *
1522 * @adev: amdgpu_device pointer
1523 *
1524 * Early initialization pass for hardware IPs. The hardware IPs that make
1525 * up each asic are discovered each IP's early_init callback is run. This
1526 * is the first stage in initializing the asic.
1527 * Returns 0 on success, negative error code on failure.
1528 */
1529static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1530{
1531 int i, r;
1532
1533 amdgpu_device_enable_virtual_display(adev);
1534
1535 switch (adev->asic_type) {
1536 case CHIP_TOPAZ:
1537 case CHIP_TONGA:
1538 case CHIP_FIJI:
1539 case CHIP_POLARIS10:
1540 case CHIP_POLARIS11:
1541 case CHIP_POLARIS12:
1542 case CHIP_VEGAM:
1543 case CHIP_CARRIZO:
1544 case CHIP_STONEY:
1545 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1546 adev->family = AMDGPU_FAMILY_CZ;
1547 else
1548 adev->family = AMDGPU_FAMILY_VI;
1549
1550 r = vi_set_ip_blocks(adev);
1551 if (r)
1552 return r;
1553 break;
1554#ifdef CONFIG_DRM_AMDGPU_SI
1555 case CHIP_VERDE:
1556 case CHIP_TAHITI:
1557 case CHIP_PITCAIRN:
1558 case CHIP_OLAND:
1559 case CHIP_HAINAN:
1560 adev->family = AMDGPU_FAMILY_SI;
1561 r = si_set_ip_blocks(adev);
1562 if (r)
1563 return r;
1564 break;
1565#endif
1566#ifdef CONFIG_DRM_AMDGPU_CIK
1567 case CHIP_BONAIRE:
1568 case CHIP_HAWAII:
1569 case CHIP_KAVERI:
1570 case CHIP_KABINI:
1571 case CHIP_MULLINS:
1572 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1573 adev->family = AMDGPU_FAMILY_CI;
1574 else
1575 adev->family = AMDGPU_FAMILY_KV;
1576
1577 r = cik_set_ip_blocks(adev);
1578 if (r)
1579 return r;
1580 break;
1581#endif
1582 case CHIP_VEGA10:
1583 case CHIP_VEGA12:
1584 case CHIP_VEGA20:
1585 case CHIP_RAVEN:
1586 case CHIP_ARCTURUS:
1587 case CHIP_RENOIR:
1588 if (adev->asic_type == CHIP_RAVEN ||
1589 adev->asic_type == CHIP_RENOIR)
1590 adev->family = AMDGPU_FAMILY_RV;
1591 else
1592 adev->family = AMDGPU_FAMILY_AI;
1593
1594 r = soc15_set_ip_blocks(adev);
1595 if (r)
1596 return r;
1597 break;
1598 case CHIP_NAVI10:
1599 case CHIP_NAVI14:
1600 case CHIP_NAVI12:
1601 adev->family = AMDGPU_FAMILY_NV;
1602
1603 r = nv_set_ip_blocks(adev);
1604 if (r)
1605 return r;
1606 break;
1607 default:
1608 /* FIXME: not supported yet */
1609 return -EINVAL;
1610 }
1611
1612 r = amdgpu_device_parse_gpu_info_fw(adev);
1613 if (r)
1614 return r;
1615
1616 amdgpu_amdkfd_device_probe(adev);
1617
1618 if (amdgpu_sriov_vf(adev)) {
1619 r = amdgpu_virt_request_full_gpu(adev, true);
1620 if (r)
1621 return -EAGAIN;
1622 }
1623
1624 adev->pm.pp_feature = amdgpu_pp_feature_mask;
1625 if (amdgpu_sriov_vf(adev))
1626 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1627
1628 for (i = 0; i < adev->num_ip_blocks; i++) {
1629 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1630 DRM_ERROR("disabled ip block: %d <%s>\n",
1631 i, adev->ip_blocks[i].version->funcs->name);
1632 adev->ip_blocks[i].status.valid = false;
1633 } else {
1634 if (adev->ip_blocks[i].version->funcs->early_init) {
1635 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1636 if (r == -ENOENT) {
1637 adev->ip_blocks[i].status.valid = false;
1638 } else if (r) {
1639 DRM_ERROR("early_init of IP block <%s> failed %d\n",
1640 adev->ip_blocks[i].version->funcs->name, r);
1641 return r;
1642 } else {
1643 adev->ip_blocks[i].status.valid = true;
1644 }
1645 } else {
1646 adev->ip_blocks[i].status.valid = true;
1647 }
1648 }
1649 /* get the vbios after the asic_funcs are set up */
1650 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
1651 /* Read BIOS */
1652 if (!amdgpu_get_bios(adev))
1653 return -EINVAL;
1654
1655 r = amdgpu_atombios_init(adev);
1656 if (r) {
1657 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
1658 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
1659 return r;
1660 }
1661 }
1662 }
1663
1664 adev->cg_flags &= amdgpu_cg_mask;
1665 adev->pg_flags &= amdgpu_pg_mask;
1666
1667 return 0;
1668}
1669
1670static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
1671{
1672 int i, r;
1673
1674 for (i = 0; i < adev->num_ip_blocks; i++) {
1675 if (!adev->ip_blocks[i].status.sw)
1676 continue;
1677 if (adev->ip_blocks[i].status.hw)
1678 continue;
1679 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1680 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
1681 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
1682 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1683 if (r) {
1684 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1685 adev->ip_blocks[i].version->funcs->name, r);
1686 return r;
1687 }
1688 adev->ip_blocks[i].status.hw = true;
1689 }
1690 }
1691
1692 return 0;
1693}
1694
1695static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
1696{
1697 int i, r;
1698
1699 for (i = 0; i < adev->num_ip_blocks; i++) {
1700 if (!adev->ip_blocks[i].status.sw)
1701 continue;
1702 if (adev->ip_blocks[i].status.hw)
1703 continue;
1704 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1705 if (r) {
1706 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1707 adev->ip_blocks[i].version->funcs->name, r);
1708 return r;
1709 }
1710 adev->ip_blocks[i].status.hw = true;
1711 }
1712
1713 return 0;
1714}
1715
1716static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
1717{
1718 int r = 0;
1719 int i;
1720 uint32_t smu_version;
1721
1722 if (adev->asic_type >= CHIP_VEGA10) {
1723 for (i = 0; i < adev->num_ip_blocks; i++) {
1724 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
1725 continue;
1726
1727 /* no need to do the fw loading again if already done*/
1728 if (adev->ip_blocks[i].status.hw == true)
1729 break;
1730
1731 if (adev->in_gpu_reset || adev->in_suspend) {
1732 r = adev->ip_blocks[i].version->funcs->resume(adev);
1733 if (r) {
1734 DRM_ERROR("resume of IP block <%s> failed %d\n",
1735 adev->ip_blocks[i].version->funcs->name, r);
1736 return r;
1737 }
1738 } else {
1739 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1740 if (r) {
1741 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1742 adev->ip_blocks[i].version->funcs->name, r);
1743 return r;
1744 }
1745 }
1746
1747 adev->ip_blocks[i].status.hw = true;
1748 break;
1749 }
1750 }
1751
1752 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
1753
1754 return r;
1755}
1756
1757/**
1758 * amdgpu_device_ip_init - run init for hardware IPs
1759 *
1760 * @adev: amdgpu_device pointer
1761 *
1762 * Main initialization pass for hardware IPs. The list of all the hardware
1763 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
1764 * are run. sw_init initializes the software state associated with each IP
1765 * and hw_init initializes the hardware associated with each IP.
1766 * Returns 0 on success, negative error code on failure.
1767 */
1768static int amdgpu_device_ip_init(struct amdgpu_device *adev)
1769{
1770 int i, r;
1771
1772 r = amdgpu_ras_init(adev);
1773 if (r)
1774 return r;
1775
1776 for (i = 0; i < adev->num_ip_blocks; i++) {
1777 if (!adev->ip_blocks[i].status.valid)
1778 continue;
1779 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1780 if (r) {
1781 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1782 adev->ip_blocks[i].version->funcs->name, r);
1783 goto init_failed;
1784 }
1785 adev->ip_blocks[i].status.sw = true;
1786
1787 /* need to do gmc hw init early so we can allocate gpu mem */
1788 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1789 r = amdgpu_device_vram_scratch_init(adev);
1790 if (r) {
1791 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1792 goto init_failed;
1793 }
1794 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1795 if (r) {
1796 DRM_ERROR("hw_init %d failed %d\n", i, r);
1797 goto init_failed;
1798 }
1799 r = amdgpu_device_wb_init(adev);
1800 if (r) {
1801 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
1802 goto init_failed;
1803 }
1804 adev->ip_blocks[i].status.hw = true;
1805
1806 /* right after GMC hw init, we create CSA */
1807 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
1808 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
1809 AMDGPU_GEM_DOMAIN_VRAM,
1810 AMDGPU_CSA_SIZE);
1811 if (r) {
1812 DRM_ERROR("allocate CSA failed %d\n", r);
1813 goto init_failed;
1814 }
1815 }
1816 }
1817 }
1818
1819 r = amdgpu_ib_pool_init(adev);
1820 if (r) {
1821 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
1822 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
1823 goto init_failed;
1824 }
1825
1826 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
1827 if (r)
1828 goto init_failed;
1829
1830 r = amdgpu_device_ip_hw_init_phase1(adev);
1831 if (r)
1832 goto init_failed;
1833
1834 r = amdgpu_device_fw_loading(adev);
1835 if (r)
1836 goto init_failed;
1837
1838 r = amdgpu_device_ip_hw_init_phase2(adev);
1839 if (r)
1840 goto init_failed;
1841
1842 if (adev->gmc.xgmi.num_physical_nodes > 1)
1843 amdgpu_xgmi_add_device(adev);
1844 amdgpu_amdkfd_device_init(adev);
1845
1846init_failed:
1847 if (amdgpu_sriov_vf(adev)) {
1848 if (!r)
1849 amdgpu_virt_init_data_exchange(adev);
1850 amdgpu_virt_release_full_gpu(adev, true);
1851 }
1852
1853 return r;
1854}
1855
1856/**
1857 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
1858 *
1859 * @adev: amdgpu_device pointer
1860 *
1861 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
1862 * this function before a GPU reset. If the value is retained after a
1863 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
1864 */
1865static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1866{
1867 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1868}
1869
1870/**
1871 * amdgpu_device_check_vram_lost - check if vram is valid
1872 *
1873 * @adev: amdgpu_device pointer
1874 *
1875 * Checks the reset magic value written to the gart pointer in VRAM.
1876 * The driver calls this after a GPU reset to see if the contents of
1877 * VRAM is lost or now.
1878 * returns true if vram is lost, false if not.
1879 */
1880static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1881{
1882 return !!memcmp(adev->gart.ptr, adev->reset_magic,
1883 AMDGPU_RESET_MAGIC_NUM);
1884}
1885
1886/**
1887 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
1888 *
1889 * @adev: amdgpu_device pointer
1890 *
1891 * The list of all the hardware IPs that make up the asic is walked and the
1892 * set_clockgating_state callbacks are run.
1893 * Late initialization pass enabling clockgating for hardware IPs.
1894 * Fini or suspend, pass disabling clockgating for hardware IPs.
1895 * Returns 0 on success, negative error code on failure.
1896 */
1897
1898static int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
1899 enum amd_clockgating_state state)
1900{
1901 int i, j, r;
1902
1903 if (amdgpu_emu_mode == 1)
1904 return 0;
1905
1906 for (j = 0; j < adev->num_ip_blocks; j++) {
1907 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
1908 if (!adev->ip_blocks[i].status.late_initialized)
1909 continue;
1910 /* skip CG for VCE/UVD, it's handled specially */
1911 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1912 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1913 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1914 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
1915 /* enable clockgating to save power */
1916 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1917 state);
1918 if (r) {
1919 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1920 adev->ip_blocks[i].version->funcs->name, r);
1921 return r;
1922 }
1923 }
1924 }
1925
1926 return 0;
1927}
1928
1929static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state)
1930{
1931 int i, j, r;
1932
1933 if (amdgpu_emu_mode == 1)
1934 return 0;
1935
1936 for (j = 0; j < adev->num_ip_blocks; j++) {
1937 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
1938 if (!adev->ip_blocks[i].status.late_initialized)
1939 continue;
1940 /* skip CG for VCE/UVD, it's handled specially */
1941 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1942 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
1943 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
1944 adev->ip_blocks[i].version->funcs->set_powergating_state) {
1945 /* enable powergating to save power */
1946 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
1947 state);
1948 if (r) {
1949 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
1950 adev->ip_blocks[i].version->funcs->name, r);
1951 return r;
1952 }
1953 }
1954 }
1955 return 0;
1956}
1957
1958static int amdgpu_device_enable_mgpu_fan_boost(void)
1959{
1960 struct amdgpu_gpu_instance *gpu_ins;
1961 struct amdgpu_device *adev;
1962 int i, ret = 0;
1963
1964 mutex_lock(&mgpu_info.mutex);
1965
1966 /*
1967 * MGPU fan boost feature should be enabled
1968 * only when there are two or more dGPUs in
1969 * the system
1970 */
1971 if (mgpu_info.num_dgpu < 2)
1972 goto out;
1973
1974 for (i = 0; i < mgpu_info.num_dgpu; i++) {
1975 gpu_ins = &(mgpu_info.gpu_ins[i]);
1976 adev = gpu_ins->adev;
1977 if (!(adev->flags & AMD_IS_APU) &&
1978 !gpu_ins->mgpu_fan_enabled &&
1979 adev->powerplay.pp_funcs &&
1980 adev->powerplay.pp_funcs->enable_mgpu_fan_boost) {
1981 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
1982 if (ret)
1983 break;
1984
1985 gpu_ins->mgpu_fan_enabled = 1;
1986 }
1987 }
1988
1989out:
1990 mutex_unlock(&mgpu_info.mutex);
1991
1992 return ret;
1993}
1994
1995/**
1996 * amdgpu_device_ip_late_init - run late init for hardware IPs
1997 *
1998 * @adev: amdgpu_device pointer
1999 *
2000 * Late initialization pass for hardware IPs. The list of all the hardware
2001 * IPs that make up the asic is walked and the late_init callbacks are run.
2002 * late_init covers any special initialization that an IP requires
2003 * after all of the have been initialized or something that needs to happen
2004 * late in the init process.
2005 * Returns 0 on success, negative error code on failure.
2006 */
2007static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2008{
2009 int i = 0, r;
2010
2011 for (i = 0; i < adev->num_ip_blocks; i++) {
2012 if (!adev->ip_blocks[i].status.hw)
2013 continue;
2014 if (adev->ip_blocks[i].version->funcs->late_init) {
2015 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2016 if (r) {
2017 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2018 adev->ip_blocks[i].version->funcs->name, r);
2019 return r;
2020 }
2021 }
2022 adev->ip_blocks[i].status.late_initialized = true;
2023 }
2024
2025 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2026 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2027
2028 amdgpu_device_fill_reset_magic(adev);
2029
2030 r = amdgpu_device_enable_mgpu_fan_boost();
2031 if (r)
2032 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2033
2034 /* set to low pstate by default */
2035 amdgpu_xgmi_set_pstate(adev, 0);
2036
2037 return 0;
2038}
2039
2040/**
2041 * amdgpu_device_ip_fini - run fini for hardware IPs
2042 *
2043 * @adev: amdgpu_device pointer
2044 *
2045 * Main teardown pass for hardware IPs. The list of all the hardware
2046 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2047 * are run. hw_fini tears down the hardware associated with each IP
2048 * and sw_fini tears down any software state associated with each IP.
2049 * Returns 0 on success, negative error code on failure.
2050 */
2051static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2052{
2053 int i, r;
2054
2055 amdgpu_ras_pre_fini(adev);
2056
2057 if (adev->gmc.xgmi.num_physical_nodes > 1)
2058 amdgpu_xgmi_remove_device(adev);
2059
2060 amdgpu_amdkfd_device_fini(adev);
2061
2062 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2063 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2064
2065 /* need to disable SMC first */
2066 for (i = 0; i < adev->num_ip_blocks; i++) {
2067 if (!adev->ip_blocks[i].status.hw)
2068 continue;
2069 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2070 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2071 /* XXX handle errors */
2072 if (r) {
2073 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2074 adev->ip_blocks[i].version->funcs->name, r);
2075 }
2076 adev->ip_blocks[i].status.hw = false;
2077 break;
2078 }
2079 }
2080
2081 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2082 if (!adev->ip_blocks[i].status.hw)
2083 continue;
2084
2085 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2086 /* XXX handle errors */
2087 if (r) {
2088 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2089 adev->ip_blocks[i].version->funcs->name, r);
2090 }
2091
2092 adev->ip_blocks[i].status.hw = false;
2093 }
2094
2095
2096 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2097 if (!adev->ip_blocks[i].status.sw)
2098 continue;
2099
2100 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2101 amdgpu_ucode_free_bo(adev);
2102 amdgpu_free_static_csa(&adev->virt.csa_obj);
2103 amdgpu_device_wb_fini(adev);
2104 amdgpu_device_vram_scratch_fini(adev);
2105 amdgpu_ib_pool_fini(adev);
2106 }
2107
2108 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2109 /* XXX handle errors */
2110 if (r) {
2111 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2112 adev->ip_blocks[i].version->funcs->name, r);
2113 }
2114 adev->ip_blocks[i].status.sw = false;
2115 adev->ip_blocks[i].status.valid = false;
2116 }
2117
2118 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2119 if (!adev->ip_blocks[i].status.late_initialized)
2120 continue;
2121 if (adev->ip_blocks[i].version->funcs->late_fini)
2122 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2123 adev->ip_blocks[i].status.late_initialized = false;
2124 }
2125
2126 amdgpu_ras_fini(adev);
2127
2128 if (amdgpu_sriov_vf(adev))
2129 if (amdgpu_virt_release_full_gpu(adev, false))
2130 DRM_ERROR("failed to release exclusive mode on fini\n");
2131
2132 return 0;
2133}
2134
2135/**
2136 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2137 *
2138 * @work: work_struct.
2139 */
2140static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2141{
2142 struct amdgpu_device *adev =
2143 container_of(work, struct amdgpu_device, delayed_init_work.work);
2144 int r;
2145
2146 r = amdgpu_ib_ring_tests(adev);
2147 if (r)
2148 DRM_ERROR("ib ring test failed (%d).\n", r);
2149}
2150
2151static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2152{
2153 struct amdgpu_device *adev =
2154 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2155
2156 mutex_lock(&adev->gfx.gfx_off_mutex);
2157 if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
2158 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2159 adev->gfx.gfx_off_state = true;
2160 }
2161 mutex_unlock(&adev->gfx.gfx_off_mutex);
2162}
2163
2164/**
2165 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2166 *
2167 * @adev: amdgpu_device pointer
2168 *
2169 * Main suspend function for hardware IPs. The list of all the hardware
2170 * IPs that make up the asic is walked, clockgating is disabled and the
2171 * suspend callbacks are run. suspend puts the hardware and software state
2172 * in each IP into a state suitable for suspend.
2173 * Returns 0 on success, negative error code on failure.
2174 */
2175static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2176{
2177 int i, r;
2178
2179 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2180 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2181
2182 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2183 if (!adev->ip_blocks[i].status.valid)
2184 continue;
2185 /* displays are handled separately */
2186 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) {
2187 /* XXX handle errors */
2188 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2189 /* XXX handle errors */
2190 if (r) {
2191 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2192 adev->ip_blocks[i].version->funcs->name, r);
2193 return r;
2194 }
2195 adev->ip_blocks[i].status.hw = false;
2196 }
2197 }
2198
2199 return 0;
2200}
2201
2202/**
2203 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2204 *
2205 * @adev: amdgpu_device pointer
2206 *
2207 * Main suspend function for hardware IPs. The list of all the hardware
2208 * IPs that make up the asic is walked, clockgating is disabled and the
2209 * suspend callbacks are run. suspend puts the hardware and software state
2210 * in each IP into a state suitable for suspend.
2211 * Returns 0 on success, negative error code on failure.
2212 */
2213static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2214{
2215 int i, r;
2216
2217 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2218 if (!adev->ip_blocks[i].status.valid)
2219 continue;
2220 /* displays are handled in phase1 */
2221 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2222 continue;
2223 /* XXX handle errors */
2224 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2225 /* XXX handle errors */
2226 if (r) {
2227 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2228 adev->ip_blocks[i].version->funcs->name, r);
2229 }
2230 adev->ip_blocks[i].status.hw = false;
2231 /* handle putting the SMC in the appropriate state */
2232 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2233 if (is_support_sw_smu(adev)) {
2234 /* todo */
2235 } else if (adev->powerplay.pp_funcs &&
2236 adev->powerplay.pp_funcs->set_mp1_state) {
2237 r = adev->powerplay.pp_funcs->set_mp1_state(
2238 adev->powerplay.pp_handle,
2239 adev->mp1_state);
2240 if (r) {
2241 DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2242 adev->mp1_state, r);
2243 return r;
2244 }
2245 }
2246 }
2247
2248 adev->ip_blocks[i].status.hw = false;
2249 }
2250
2251 return 0;
2252}
2253
2254/**
2255 * amdgpu_device_ip_suspend - run suspend for hardware IPs
2256 *
2257 * @adev: amdgpu_device pointer
2258 *
2259 * Main suspend function for hardware IPs. The list of all the hardware
2260 * IPs that make up the asic is walked, clockgating is disabled and the
2261 * suspend callbacks are run. suspend puts the hardware and software state
2262 * in each IP into a state suitable for suspend.
2263 * Returns 0 on success, negative error code on failure.
2264 */
2265int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2266{
2267 int r;
2268
2269 if (amdgpu_sriov_vf(adev))
2270 amdgpu_virt_request_full_gpu(adev, false);
2271
2272 r = amdgpu_device_ip_suspend_phase1(adev);
2273 if (r)
2274 return r;
2275 r = amdgpu_device_ip_suspend_phase2(adev);
2276
2277 if (amdgpu_sriov_vf(adev))
2278 amdgpu_virt_release_full_gpu(adev, false);
2279
2280 return r;
2281}
2282
2283static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2284{
2285 int i, r;
2286
2287 static enum amd_ip_block_type ip_order[] = {
2288 AMD_IP_BLOCK_TYPE_GMC,
2289 AMD_IP_BLOCK_TYPE_COMMON,
2290 AMD_IP_BLOCK_TYPE_PSP,
2291 AMD_IP_BLOCK_TYPE_IH,
2292 };
2293
2294 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2295 int j;
2296 struct amdgpu_ip_block *block;
2297
2298 for (j = 0; j < adev->num_ip_blocks; j++) {
2299 block = &adev->ip_blocks[j];
2300
2301 block->status.hw = false;
2302 if (block->version->type != ip_order[i] ||
2303 !block->status.valid)
2304 continue;
2305
2306 r = block->version->funcs->hw_init(adev);
2307 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2308 if (r)
2309 return r;
2310 block->status.hw = true;
2311 }
2312 }
2313
2314 return 0;
2315}
2316
2317static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2318{
2319 int i, r;
2320
2321 static enum amd_ip_block_type ip_order[] = {
2322 AMD_IP_BLOCK_TYPE_SMC,
2323 AMD_IP_BLOCK_TYPE_DCE,
2324 AMD_IP_BLOCK_TYPE_GFX,
2325 AMD_IP_BLOCK_TYPE_SDMA,
2326 AMD_IP_BLOCK_TYPE_UVD,
2327 AMD_IP_BLOCK_TYPE_VCE
2328 };
2329
2330 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2331 int j;
2332 struct amdgpu_ip_block *block;
2333
2334 for (j = 0; j < adev->num_ip_blocks; j++) {
2335 block = &adev->ip_blocks[j];
2336
2337 if (block->version->type != ip_order[i] ||
2338 !block->status.valid ||
2339 block->status.hw)
2340 continue;
2341
2342 r = block->version->funcs->hw_init(adev);
2343 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2344 if (r)
2345 return r;
2346 block->status.hw = true;
2347 }
2348 }
2349
2350 return 0;
2351}
2352
2353/**
2354 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2355 *
2356 * @adev: amdgpu_device pointer
2357 *
2358 * First resume function for hardware IPs. The list of all the hardware
2359 * IPs that make up the asic is walked and the resume callbacks are run for
2360 * COMMON, GMC, and IH. resume puts the hardware into a functional state
2361 * after a suspend and updates the software state as necessary. This
2362 * function is also used for restoring the GPU after a GPU reset.
2363 * Returns 0 on success, negative error code on failure.
2364 */
2365static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
2366{
2367 int i, r;
2368
2369 for (i = 0; i < adev->num_ip_blocks; i++) {
2370 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2371 continue;
2372 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2373 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2374 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2375
2376 r = adev->ip_blocks[i].version->funcs->resume(adev);
2377 if (r) {
2378 DRM_ERROR("resume of IP block <%s> failed %d\n",
2379 adev->ip_blocks[i].version->funcs->name, r);
2380 return r;
2381 }
2382 adev->ip_blocks[i].status.hw = true;
2383 }
2384 }
2385
2386 return 0;
2387}
2388
2389/**
2390 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2391 *
2392 * @adev: amdgpu_device pointer
2393 *
2394 * First resume function for hardware IPs. The list of all the hardware
2395 * IPs that make up the asic is walked and the resume callbacks are run for
2396 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
2397 * functional state after a suspend and updates the software state as
2398 * necessary. This function is also used for restoring the GPU after a GPU
2399 * reset.
2400 * Returns 0 on success, negative error code on failure.
2401 */
2402static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
2403{
2404 int i, r;
2405
2406 for (i = 0; i < adev->num_ip_blocks; i++) {
2407 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2408 continue;
2409 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2410 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2411 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
2412 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
2413 continue;
2414 r = adev->ip_blocks[i].version->funcs->resume(adev);
2415 if (r) {
2416 DRM_ERROR("resume of IP block <%s> failed %d\n",
2417 adev->ip_blocks[i].version->funcs->name, r);
2418 return r;
2419 }
2420 adev->ip_blocks[i].status.hw = true;
2421 }
2422
2423 return 0;
2424}
2425
2426/**
2427 * amdgpu_device_ip_resume - run resume for hardware IPs
2428 *
2429 * @adev: amdgpu_device pointer
2430 *
2431 * Main resume function for hardware IPs. The hardware IPs
2432 * are split into two resume functions because they are
2433 * are also used in in recovering from a GPU reset and some additional
2434 * steps need to be take between them. In this case (S3/S4) they are
2435 * run sequentially.
2436 * Returns 0 on success, negative error code on failure.
2437 */
2438static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2439{
2440 int r;
2441
2442 r = amdgpu_device_ip_resume_phase1(adev);
2443 if (r)
2444 return r;
2445
2446 r = amdgpu_device_fw_loading(adev);
2447 if (r)
2448 return r;
2449
2450 r = amdgpu_device_ip_resume_phase2(adev);
2451
2452 return r;
2453}
2454
2455/**
2456 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2457 *
2458 * @adev: amdgpu_device pointer
2459 *
2460 * Query the VBIOS data tables to determine if the board supports SR-IOV.
2461 */
2462static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2463{
2464 if (amdgpu_sriov_vf(adev)) {
2465 if (adev->is_atom_fw) {
2466 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2467 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2468 } else {
2469 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
2470 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
2471 }
2472
2473 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
2474 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
2475 }
2476}
2477
2478/**
2479 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
2480 *
2481 * @asic_type: AMD asic type
2482 *
2483 * Check if there is DC (new modesetting infrastructre) support for an asic.
2484 * returns true if DC has support, false if not.
2485 */
2486bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
2487{
2488 switch (asic_type) {
2489#if defined(CONFIG_DRM_AMD_DC)
2490 case CHIP_BONAIRE:
2491 case CHIP_KAVERI:
2492 case CHIP_KABINI:
2493 case CHIP_MULLINS:
2494 /*
2495 * We have systems in the wild with these ASICs that require
2496 * LVDS and VGA support which is not supported with DC.
2497 *
2498 * Fallback to the non-DC driver here by default so as not to
2499 * cause regressions.
2500 */
2501 return amdgpu_dc > 0;
2502 case CHIP_HAWAII:
2503 case CHIP_CARRIZO:
2504 case CHIP_STONEY:
2505 case CHIP_POLARIS10:
2506 case CHIP_POLARIS11:
2507 case CHIP_POLARIS12:
2508 case CHIP_VEGAM:
2509 case CHIP_TONGA:
2510 case CHIP_FIJI:
2511 case CHIP_VEGA10:
2512 case CHIP_VEGA12:
2513 case CHIP_VEGA20:
2514#if defined(CONFIG_DRM_AMD_DC_DCN1_0)
2515 case CHIP_RAVEN:
2516#endif
2517#if defined(CONFIG_DRM_AMD_DC_DCN2_0)
2518 case CHIP_NAVI10:
2519 case CHIP_NAVI14:
2520 case CHIP_NAVI12:
2521#endif
2522#if defined(CONFIG_DRM_AMD_DC_DCN2_1)
2523 case CHIP_RENOIR:
2524#endif
2525 return amdgpu_dc != 0;
2526#endif
2527 default:
2528 return false;
2529 }
2530}
2531
2532/**
2533 * amdgpu_device_has_dc_support - check if dc is supported
2534 *
2535 * @adev: amdgpu_device_pointer
2536 *
2537 * Returns true for supported, false for not supported
2538 */
2539bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
2540{
2541 if (amdgpu_sriov_vf(adev))
2542 return false;
2543
2544 return amdgpu_device_asic_has_dc_support(adev->asic_type);
2545}
2546
2547
2548static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
2549{
2550 struct amdgpu_device *adev =
2551 container_of(__work, struct amdgpu_device, xgmi_reset_work);
2552
2553 adev->asic_reset_res = amdgpu_asic_reset(adev);
2554 if (adev->asic_reset_res)
2555 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
2556 adev->asic_reset_res, adev->ddev->unique);
2557}
2558
2559
2560/**
2561 * amdgpu_device_init - initialize the driver
2562 *
2563 * @adev: amdgpu_device pointer
2564 * @ddev: drm dev pointer
2565 * @pdev: pci dev pointer
2566 * @flags: driver flags
2567 *
2568 * Initializes the driver info and hw (all asics).
2569 * Returns 0 for success or an error on failure.
2570 * Called at driver startup.
2571 */
2572int amdgpu_device_init(struct amdgpu_device *adev,
2573 struct drm_device *ddev,
2574 struct pci_dev *pdev,
2575 uint32_t flags)
2576{
2577 int r, i;
2578 bool runtime = false;
2579 u32 max_MBps;
2580
2581 adev->shutdown = false;
2582 adev->dev = &pdev->dev;
2583 adev->ddev = ddev;
2584 adev->pdev = pdev;
2585 adev->flags = flags;
2586 adev->asic_type = flags & AMD_ASIC_MASK;
2587 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
2588 if (amdgpu_emu_mode == 1)
2589 adev->usec_timeout *= 2;
2590 adev->gmc.gart_size = 512 * 1024 * 1024;
2591 adev->accel_working = false;
2592 adev->num_rings = 0;
2593 adev->mman.buffer_funcs = NULL;
2594 adev->mman.buffer_funcs_ring = NULL;
2595 adev->vm_manager.vm_pte_funcs = NULL;
2596 adev->vm_manager.vm_pte_num_rqs = 0;
2597 adev->gmc.gmc_funcs = NULL;
2598 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
2599 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
2600
2601 adev->smc_rreg = &amdgpu_invalid_rreg;
2602 adev->smc_wreg = &amdgpu_invalid_wreg;
2603 adev->pcie_rreg = &amdgpu_invalid_rreg;
2604 adev->pcie_wreg = &amdgpu_invalid_wreg;
2605 adev->pciep_rreg = &amdgpu_invalid_rreg;
2606 adev->pciep_wreg = &amdgpu_invalid_wreg;
2607 adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
2608 adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
2609 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
2610 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
2611 adev->didt_rreg = &amdgpu_invalid_rreg;
2612 adev->didt_wreg = &amdgpu_invalid_wreg;
2613 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
2614 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
2615 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
2616 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
2617
2618 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
2619 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
2620 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
2621
2622 /* mutex initialization are all done here so we
2623 * can recall function without having locking issues */
2624 atomic_set(&adev->irq.ih.lock, 0);
2625 mutex_init(&adev->firmware.mutex);
2626 mutex_init(&adev->pm.mutex);
2627 mutex_init(&adev->gfx.gpu_clock_mutex);
2628 mutex_init(&adev->srbm_mutex);
2629 mutex_init(&adev->gfx.pipe_reserve_mutex);
2630 mutex_init(&adev->gfx.gfx_off_mutex);
2631 mutex_init(&adev->grbm_idx_mutex);
2632 mutex_init(&adev->mn_lock);
2633 mutex_init(&adev->virt.vf_errors.lock);
2634 hash_init(adev->mn_hash);
2635 mutex_init(&adev->lock_reset);
2636 mutex_init(&adev->virt.dpm_mutex);
2637 mutex_init(&adev->psp.mutex);
2638
2639 r = amdgpu_device_check_arguments(adev);
2640 if (r)
2641 return r;
2642
2643 spin_lock_init(&adev->mmio_idx_lock);
2644 spin_lock_init(&adev->smc_idx_lock);
2645 spin_lock_init(&adev->pcie_idx_lock);
2646 spin_lock_init(&adev->uvd_ctx_idx_lock);
2647 spin_lock_init(&adev->didt_idx_lock);
2648 spin_lock_init(&adev->gc_cac_idx_lock);
2649 spin_lock_init(&adev->se_cac_idx_lock);
2650 spin_lock_init(&adev->audio_endpt_idx_lock);
2651 spin_lock_init(&adev->mm_stats.lock);
2652
2653 INIT_LIST_HEAD(&adev->shadow_list);
2654 mutex_init(&adev->shadow_list_lock);
2655
2656 INIT_LIST_HEAD(&adev->ring_lru_list);
2657 spin_lock_init(&adev->ring_lru_list_lock);
2658
2659 INIT_DELAYED_WORK(&adev->delayed_init_work,
2660 amdgpu_device_delayed_init_work_handler);
2661 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
2662 amdgpu_device_delay_enable_gfx_off);
2663
2664 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
2665
2666 adev->gfx.gfx_off_req_count = 1;
2667 adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false;
2668
2669 /* Registers mapping */
2670 /* TODO: block userspace mapping of io register */
2671 if (adev->asic_type >= CHIP_BONAIRE) {
2672 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2673 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2674 } else {
2675 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2676 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2677 }
2678
2679 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2680 if (adev->rmmio == NULL) {
2681 return -ENOMEM;
2682 }
2683 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2684 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2685
2686 /* io port mapping */
2687 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2688 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2689 adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2690 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2691 break;
2692 }
2693 }
2694 if (adev->rio_mem == NULL)
2695 DRM_INFO("PCI I/O BAR is not found.\n");
2696
2697 /* enable PCIE atomic ops */
2698 r = pci_enable_atomic_ops_to_root(adev->pdev,
2699 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
2700 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
2701 if (r) {
2702 adev->have_atomics_support = false;
2703 DRM_INFO("PCIE atomic ops is not supported\n");
2704 } else {
2705 adev->have_atomics_support = true;
2706 }
2707
2708 amdgpu_device_get_pcie_info(adev);
2709
2710 if (amdgpu_mcbp)
2711 DRM_INFO("MCBP is enabled\n");
2712
2713 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
2714 adev->enable_mes = true;
2715
2716 if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10) {
2717 r = amdgpu_discovery_init(adev);
2718 if (r) {
2719 dev_err(adev->dev, "amdgpu_discovery_init failed\n");
2720 return r;
2721 }
2722 }
2723
2724 /* early init functions */
2725 r = amdgpu_device_ip_early_init(adev);
2726 if (r)
2727 return r;
2728
2729 /* doorbell bar mapping and doorbell index init*/
2730 amdgpu_device_doorbell_init(adev);
2731
2732 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2733 /* this will fail for cards that aren't VGA class devices, just
2734 * ignore it */
2735 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
2736
2737 if (amdgpu_device_is_px(ddev))
2738 runtime = true;
2739 if (!pci_is_thunderbolt_attached(adev->pdev))
2740 vga_switcheroo_register_client(adev->pdev,
2741 &amdgpu_switcheroo_ops, runtime);
2742 if (runtime)
2743 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2744
2745 if (amdgpu_emu_mode == 1) {
2746 /* post the asic on emulation mode */
2747 emu_soc_asic_init(adev);
2748 goto fence_driver_init;
2749 }
2750
2751 /* detect if we are with an SRIOV vbios */
2752 amdgpu_device_detect_sriov_bios(adev);
2753
2754 /* check if we need to reset the asic
2755 * E.g., driver was not cleanly unloaded previously, etc.
2756 */
2757 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
2758 r = amdgpu_asic_reset(adev);
2759 if (r) {
2760 dev_err(adev->dev, "asic reset on init failed\n");
2761 goto failed;
2762 }
2763 }
2764
2765 /* Post card if necessary */
2766 if (amdgpu_device_need_post(adev)) {
2767 if (!adev->bios) {
2768 dev_err(adev->dev, "no vBIOS found\n");
2769 r = -EINVAL;
2770 goto failed;
2771 }
2772 DRM_INFO("GPU posting now...\n");
2773 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2774 if (r) {
2775 dev_err(adev->dev, "gpu post error!\n");
2776 goto failed;
2777 }
2778 }
2779
2780 if (adev->is_atom_fw) {
2781 /* Initialize clocks */
2782 r = amdgpu_atomfirmware_get_clock_info(adev);
2783 if (r) {
2784 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2785 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2786 goto failed;
2787 }
2788 } else {
2789 /* Initialize clocks */
2790 r = amdgpu_atombios_get_clock_info(adev);
2791 if (r) {
2792 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2793 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2794 goto failed;
2795 }
2796 /* init i2c buses */
2797 if (!amdgpu_device_has_dc_support(adev))
2798 amdgpu_atombios_i2c_init(adev);
2799 }
2800
2801fence_driver_init:
2802 /* Fence driver */
2803 r = amdgpu_fence_driver_init(adev);
2804 if (r) {
2805 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2806 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2807 goto failed;
2808 }
2809
2810 /* init the mode config */
2811 drm_mode_config_init(adev->ddev);
2812
2813 r = amdgpu_device_ip_init(adev);
2814 if (r) {
2815 /* failed in exclusive mode due to timeout */
2816 if (amdgpu_sriov_vf(adev) &&
2817 !amdgpu_sriov_runtime(adev) &&
2818 amdgpu_virt_mmio_blocked(adev) &&
2819 !amdgpu_virt_wait_reset(adev)) {
2820 dev_err(adev->dev, "VF exclusive mode timeout\n");
2821 /* Don't send request since VF is inactive. */
2822 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2823 adev->virt.ops = NULL;
2824 r = -EAGAIN;
2825 goto failed;
2826 }
2827 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
2828 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2829 if (amdgpu_virt_request_full_gpu(adev, false))
2830 amdgpu_virt_release_full_gpu(adev, false);
2831 goto failed;
2832 }
2833
2834 adev->accel_working = true;
2835
2836 amdgpu_vm_check_compute_bug(adev);
2837
2838 /* Initialize the buffer migration limit. */
2839 if (amdgpu_moverate >= 0)
2840 max_MBps = amdgpu_moverate;
2841 else
2842 max_MBps = 8; /* Allow 8 MB/s. */
2843 /* Get a log2 for easy divisions. */
2844 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2845
2846 amdgpu_fbdev_init(adev);
2847
2848 if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
2849 amdgpu_pm_virt_sysfs_init(adev);
2850
2851 r = amdgpu_pm_sysfs_init(adev);
2852 if (r)
2853 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2854
2855 r = amdgpu_ucode_sysfs_init(adev);
2856 if (r)
2857 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
2858
2859 r = amdgpu_debugfs_gem_init(adev);
2860 if (r)
2861 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2862
2863 r = amdgpu_debugfs_regs_init(adev);
2864 if (r)
2865 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2866
2867 r = amdgpu_debugfs_firmware_init(adev);
2868 if (r)
2869 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2870
2871 r = amdgpu_debugfs_init(adev);
2872 if (r)
2873 DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2874
2875 if ((amdgpu_testing & 1)) {
2876 if (adev->accel_working)
2877 amdgpu_test_moves(adev);
2878 else
2879 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2880 }
2881 if (amdgpu_benchmarking) {
2882 if (adev->accel_working)
2883 amdgpu_benchmark(adev, amdgpu_benchmarking);
2884 else
2885 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2886 }
2887
2888 /*
2889 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
2890 * Otherwise the mgpu fan boost feature will be skipped due to the
2891 * gpu instance is counted less.
2892 */
2893 amdgpu_register_gpu_instance(adev);
2894
2895 /* enable clockgating, etc. after ib tests, etc. since some blocks require
2896 * explicit gating rather than handling it automatically.
2897 */
2898 r = amdgpu_device_ip_late_init(adev);
2899 if (r) {
2900 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
2901 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2902 goto failed;
2903 }
2904
2905 /* must succeed. */
2906 amdgpu_ras_resume(adev);
2907
2908 queue_delayed_work(system_wq, &adev->delayed_init_work,
2909 msecs_to_jiffies(AMDGPU_RESUME_MS));
2910
2911 r = device_create_file(adev->dev, &dev_attr_pcie_replay_count);
2912 if (r) {
2913 dev_err(adev->dev, "Could not create pcie_replay_count");
2914 return r;
2915 }
2916
2917 if (IS_ENABLED(CONFIG_PERF_EVENTS))
2918 r = amdgpu_pmu_init(adev);
2919 if (r)
2920 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
2921
2922 return 0;
2923
2924failed:
2925 amdgpu_vf_error_trans_all(adev);
2926 if (runtime)
2927 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2928
2929 return r;
2930}
2931
2932/**
2933 * amdgpu_device_fini - tear down the driver
2934 *
2935 * @adev: amdgpu_device pointer
2936 *
2937 * Tear down the driver info (all asics).
2938 * Called at driver shutdown.
2939 */
2940void amdgpu_device_fini(struct amdgpu_device *adev)
2941{
2942 int r;
2943
2944 DRM_INFO("amdgpu: finishing device.\n");
2945 adev->shutdown = true;
2946 /* disable all interrupts */
2947 amdgpu_irq_disable_all(adev);
2948 if (adev->mode_info.mode_config_initialized){
2949 if (!amdgpu_device_has_dc_support(adev))
2950 drm_helper_force_disable_all(adev->ddev);
2951 else
2952 drm_atomic_helper_shutdown(adev->ddev);
2953 }
2954 amdgpu_fence_driver_fini(adev);
2955 amdgpu_pm_sysfs_fini(adev);
2956 amdgpu_fbdev_fini(adev);
2957 r = amdgpu_device_ip_fini(adev);
2958 if (adev->firmware.gpu_info_fw) {
2959 release_firmware(adev->firmware.gpu_info_fw);
2960 adev->firmware.gpu_info_fw = NULL;
2961 }
2962 adev->accel_working = false;
2963 cancel_delayed_work_sync(&adev->delayed_init_work);
2964 /* free i2c buses */
2965 if (!amdgpu_device_has_dc_support(adev))
2966 amdgpu_i2c_fini(adev);
2967
2968 if (amdgpu_emu_mode != 1)
2969 amdgpu_atombios_fini(adev);
2970
2971 kfree(adev->bios);
2972 adev->bios = NULL;
2973 if (!pci_is_thunderbolt_attached(adev->pdev))
2974 vga_switcheroo_unregister_client(adev->pdev);
2975 if (adev->flags & AMD_IS_PX)
2976 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2977 vga_client_register(adev->pdev, NULL, NULL, NULL);
2978 if (adev->rio_mem)
2979 pci_iounmap(adev->pdev, adev->rio_mem);
2980 adev->rio_mem = NULL;
2981 iounmap(adev->rmmio);
2982 adev->rmmio = NULL;
2983 amdgpu_device_doorbell_fini(adev);
2984 if (amdgpu_sriov_vf(adev) && amdgim_is_hwperf(adev))
2985 amdgpu_pm_virt_sysfs_fini(adev);
2986
2987 amdgpu_debugfs_regs_cleanup(adev);
2988 device_remove_file(adev->dev, &dev_attr_pcie_replay_count);
2989 amdgpu_ucode_sysfs_fini(adev);
2990 if (IS_ENABLED(CONFIG_PERF_EVENTS))
2991 amdgpu_pmu_fini(adev);
2992 amdgpu_debugfs_preempt_cleanup(adev);
2993 if (amdgpu_discovery && adev->asic_type >= CHIP_NAVI10)
2994 amdgpu_discovery_fini(adev);
2995}
2996
2997
2998/*
2999 * Suspend & resume.
3000 */
3001/**
3002 * amdgpu_device_suspend - initiate device suspend
3003 *
3004 * @dev: drm dev pointer
3005 * @suspend: suspend state
3006 * @fbcon : notify the fbdev of suspend
3007 *
3008 * Puts the hw in the suspend state (all asics).
3009 * Returns 0 for success or an error on failure.
3010 * Called at driver suspend.
3011 */
3012int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
3013{
3014 struct amdgpu_device *adev;
3015 struct drm_crtc *crtc;
3016 struct drm_connector *connector;
3017 int r;
3018
3019 if (dev == NULL || dev->dev_private == NULL) {
3020 return -ENODEV;
3021 }
3022
3023 adev = dev->dev_private;
3024
3025 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3026 return 0;
3027
3028 adev->in_suspend = true;
3029 drm_kms_helper_poll_disable(dev);
3030
3031 if (fbcon)
3032 amdgpu_fbdev_set_suspend(adev, 1);
3033
3034 cancel_delayed_work_sync(&adev->delayed_init_work);
3035
3036 if (!amdgpu_device_has_dc_support(adev)) {
3037 /* turn off display hw */
3038 drm_modeset_lock_all(dev);
3039 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
3040 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
3041 }
3042 drm_modeset_unlock_all(dev);
3043 /* unpin the front buffers and cursors */
3044 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3045 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3046 struct drm_framebuffer *fb = crtc->primary->fb;
3047 struct amdgpu_bo *robj;
3048
3049 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3050 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3051 r = amdgpu_bo_reserve(aobj, true);
3052 if (r == 0) {
3053 amdgpu_bo_unpin(aobj);
3054 amdgpu_bo_unreserve(aobj);
3055 }
3056 }
3057
3058 if (fb == NULL || fb->obj[0] == NULL) {
3059 continue;
3060 }
3061 robj = gem_to_amdgpu_bo(fb->obj[0]);
3062 /* don't unpin kernel fb objects */
3063 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
3064 r = amdgpu_bo_reserve(robj, true);
3065 if (r == 0) {
3066 amdgpu_bo_unpin(robj);
3067 amdgpu_bo_unreserve(robj);
3068 }
3069 }
3070 }
3071 }
3072
3073 amdgpu_amdkfd_suspend(adev);
3074
3075 amdgpu_ras_suspend(adev);
3076
3077 r = amdgpu_device_ip_suspend_phase1(adev);
3078
3079 /* evict vram memory */
3080 amdgpu_bo_evict_vram(adev);
3081
3082 amdgpu_fence_driver_suspend(adev);
3083
3084 r = amdgpu_device_ip_suspend_phase2(adev);
3085
3086 /* evict remaining vram memory
3087 * This second call to evict vram is to evict the gart page table
3088 * using the CPU.
3089 */
3090 amdgpu_bo_evict_vram(adev);
3091
3092 pci_save_state(dev->pdev);
3093 if (suspend) {
3094 /* Shut down the device */
3095 pci_disable_device(dev->pdev);
3096 pci_set_power_state(dev->pdev, PCI_D3hot);
3097 } else {
3098 r = amdgpu_asic_reset(adev);
3099 if (r)
3100 DRM_ERROR("amdgpu asic reset failed\n");
3101 }
3102
3103 return 0;
3104}
3105
3106/**
3107 * amdgpu_device_resume - initiate device resume
3108 *
3109 * @dev: drm dev pointer
3110 * @resume: resume state
3111 * @fbcon : notify the fbdev of resume
3112 *
3113 * Bring the hw back to operating state (all asics).
3114 * Returns 0 for success or an error on failure.
3115 * Called at driver resume.
3116 */
3117int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
3118{
3119 struct drm_connector *connector;
3120 struct amdgpu_device *adev = dev->dev_private;
3121 struct drm_crtc *crtc;
3122 int r = 0;
3123
3124 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3125 return 0;
3126
3127 if (resume) {
3128 pci_set_power_state(dev->pdev, PCI_D0);
3129 pci_restore_state(dev->pdev);
3130 r = pci_enable_device(dev->pdev);
3131 if (r)
3132 return r;
3133 }
3134
3135 /* post card */
3136 if (amdgpu_device_need_post(adev)) {
3137 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
3138 if (r)
3139 DRM_ERROR("amdgpu asic init failed\n");
3140 }
3141
3142 r = amdgpu_device_ip_resume(adev);
3143 if (r) {
3144 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
3145 return r;
3146 }
3147 amdgpu_fence_driver_resume(adev);
3148
3149
3150 r = amdgpu_device_ip_late_init(adev);
3151 if (r)
3152 return r;
3153
3154 queue_delayed_work(system_wq, &adev->delayed_init_work,
3155 msecs_to_jiffies(AMDGPU_RESUME_MS));
3156
3157 if (!amdgpu_device_has_dc_support(adev)) {
3158 /* pin cursors */
3159 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3160 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3161
3162 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3163 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3164 r = amdgpu_bo_reserve(aobj, true);
3165 if (r == 0) {
3166 r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
3167 if (r != 0)
3168 DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
3169 amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
3170 amdgpu_bo_unreserve(aobj);
3171 }
3172 }
3173 }
3174 }
3175 r = amdgpu_amdkfd_resume(adev);
3176 if (r)
3177 return r;
3178
3179 /* Make sure IB tests flushed */
3180 flush_delayed_work(&adev->delayed_init_work);
3181
3182 /* blat the mode back in */
3183 if (fbcon) {
3184 if (!amdgpu_device_has_dc_support(adev)) {
3185 /* pre DCE11 */
3186 drm_helper_resume_force_mode(dev);
3187
3188 /* turn on display hw */
3189 drm_modeset_lock_all(dev);
3190 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
3191 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
3192 }
3193 drm_modeset_unlock_all(dev);
3194 }
3195 amdgpu_fbdev_set_suspend(adev, 0);
3196 }
3197
3198 drm_kms_helper_poll_enable(dev);
3199
3200 amdgpu_ras_resume(adev);
3201
3202 /*
3203 * Most of the connector probing functions try to acquire runtime pm
3204 * refs to ensure that the GPU is powered on when connector polling is
3205 * performed. Since we're calling this from a runtime PM callback,
3206 * trying to acquire rpm refs will cause us to deadlock.
3207 *
3208 * Since we're guaranteed to be holding the rpm lock, it's safe to
3209 * temporarily disable the rpm helpers so this doesn't deadlock us.
3210 */
3211#ifdef CONFIG_PM
3212 dev->dev->power.disable_depth++;
3213#endif
3214 if (!amdgpu_device_has_dc_support(adev))
3215 drm_helper_hpd_irq_event(dev);
3216 else
3217 drm_kms_helper_hotplug_event(dev);
3218#ifdef CONFIG_PM
3219 dev->dev->power.disable_depth--;
3220#endif
3221 adev->in_suspend = false;
3222
3223 return 0;
3224}
3225
3226/**
3227 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3228 *
3229 * @adev: amdgpu_device pointer
3230 *
3231 * The list of all the hardware IPs that make up the asic is walked and
3232 * the check_soft_reset callbacks are run. check_soft_reset determines
3233 * if the asic is still hung or not.
3234 * Returns true if any of the IPs are still in a hung state, false if not.
3235 */
3236static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
3237{
3238 int i;
3239 bool asic_hang = false;
3240
3241 if (amdgpu_sriov_vf(adev))
3242 return true;
3243
3244 if (amdgpu_asic_need_full_reset(adev))
3245 return true;
3246
3247 for (i = 0; i < adev->num_ip_blocks; i++) {
3248 if (!adev->ip_blocks[i].status.valid)
3249 continue;
3250 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
3251 adev->ip_blocks[i].status.hang =
3252 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
3253 if (adev->ip_blocks[i].status.hang) {
3254 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
3255 asic_hang = true;
3256 }
3257 }
3258 return asic_hang;
3259}
3260
3261/**
3262 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
3263 *
3264 * @adev: amdgpu_device pointer
3265 *
3266 * The list of all the hardware IPs that make up the asic is walked and the
3267 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
3268 * handles any IP specific hardware or software state changes that are
3269 * necessary for a soft reset to succeed.
3270 * Returns 0 on success, negative error code on failure.
3271 */
3272static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3273{
3274 int i, r = 0;
3275
3276 for (i = 0; i < adev->num_ip_blocks; i++) {
3277 if (!adev->ip_blocks[i].status.valid)
3278 continue;
3279 if (adev->ip_blocks[i].status.hang &&
3280 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
3281 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3282 if (r)
3283 return r;
3284 }
3285 }
3286
3287 return 0;
3288}
3289
3290/**
3291 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3292 *
3293 * @adev: amdgpu_device pointer
3294 *
3295 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
3296 * reset is necessary to recover.
3297 * Returns true if a full asic reset is required, false if not.
3298 */
3299static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
3300{
3301 int i;
3302
3303 if (amdgpu_asic_need_full_reset(adev))
3304 return true;
3305
3306 for (i = 0; i < adev->num_ip_blocks; i++) {
3307 if (!adev->ip_blocks[i].status.valid)
3308 continue;
3309 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
3310 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
3311 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
3312 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
3313 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3314 if (adev->ip_blocks[i].status.hang) {
3315 DRM_INFO("Some block need full reset!\n");
3316 return true;
3317 }
3318 }
3319 }
3320 return false;
3321}
3322
3323/**
3324 * amdgpu_device_ip_soft_reset - do a soft reset
3325 *
3326 * @adev: amdgpu_device pointer
3327 *
3328 * The list of all the hardware IPs that make up the asic is walked and the
3329 * soft_reset callbacks are run if the block is hung. soft_reset handles any
3330 * IP specific hardware or software state changes that are necessary to soft
3331 * reset the IP.
3332 * Returns 0 on success, negative error code on failure.
3333 */
3334static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
3335{
3336 int i, r = 0;
3337
3338 for (i = 0; i < adev->num_ip_blocks; i++) {
3339 if (!adev->ip_blocks[i].status.valid)
3340 continue;
3341 if (adev->ip_blocks[i].status.hang &&
3342 adev->ip_blocks[i].version->funcs->soft_reset) {
3343 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
3344 if (r)
3345 return r;
3346 }
3347 }
3348
3349 return 0;
3350}
3351
3352/**
3353 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
3354 *
3355 * @adev: amdgpu_device pointer
3356 *
3357 * The list of all the hardware IPs that make up the asic is walked and the
3358 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
3359 * handles any IP specific hardware or software state changes that are
3360 * necessary after the IP has been soft reset.
3361 * Returns 0 on success, negative error code on failure.
3362 */
3363static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
3364{
3365 int i, r = 0;
3366
3367 for (i = 0; i < adev->num_ip_blocks; i++) {
3368 if (!adev->ip_blocks[i].status.valid)
3369 continue;
3370 if (adev->ip_blocks[i].status.hang &&
3371 adev->ip_blocks[i].version->funcs->post_soft_reset)
3372 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
3373 if (r)
3374 return r;
3375 }
3376
3377 return 0;
3378}
3379
3380/**
3381 * amdgpu_device_recover_vram - Recover some VRAM contents
3382 *
3383 * @adev: amdgpu_device pointer
3384 *
3385 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
3386 * restore things like GPUVM page tables after a GPU reset where
3387 * the contents of VRAM might be lost.
3388 *
3389 * Returns:
3390 * 0 on success, negative error code on failure.
3391 */
3392static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
3393{
3394 struct dma_fence *fence = NULL, *next = NULL;
3395 struct amdgpu_bo *shadow;
3396 long r = 1, tmo;
3397
3398 if (amdgpu_sriov_runtime(adev))
3399 tmo = msecs_to_jiffies(8000);
3400 else
3401 tmo = msecs_to_jiffies(100);
3402
3403 DRM_INFO("recover vram bo from shadow start\n");
3404 mutex_lock(&adev->shadow_list_lock);
3405 list_for_each_entry(shadow, &adev->shadow_list, shadow_list) {
3406
3407 /* No need to recover an evicted BO */
3408 if (shadow->tbo.mem.mem_type != TTM_PL_TT ||
3409 shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET ||
3410 shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM)
3411 continue;
3412
3413 r = amdgpu_bo_restore_shadow(shadow, &next);
3414 if (r)
3415 break;
3416
3417 if (fence) {
3418 tmo = dma_fence_wait_timeout(fence, false, tmo);
3419 dma_fence_put(fence);
3420 fence = next;
3421 if (tmo == 0) {
3422 r = -ETIMEDOUT;
3423 break;
3424 } else if (tmo < 0) {
3425 r = tmo;
3426 break;
3427 }
3428 } else {
3429 fence = next;
3430 }
3431 }
3432 mutex_unlock(&adev->shadow_list_lock);
3433
3434 if (fence)
3435 tmo = dma_fence_wait_timeout(fence, false, tmo);
3436 dma_fence_put(fence);
3437
3438 if (r < 0 || tmo <= 0) {
3439 DRM_ERROR("recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
3440 return -EIO;
3441 }
3442
3443 DRM_INFO("recover vram bo from shadow done\n");
3444 return 0;
3445}
3446
3447
3448/**
3449 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
3450 *
3451 * @adev: amdgpu device pointer
3452 * @from_hypervisor: request from hypervisor
3453 *
3454 * do VF FLR and reinitialize Asic
3455 * return 0 means succeeded otherwise failed
3456 */
3457static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
3458 bool from_hypervisor)
3459{
3460 int r;
3461
3462 if (from_hypervisor)
3463 r = amdgpu_virt_request_full_gpu(adev, true);
3464 else
3465 r = amdgpu_virt_reset_gpu(adev);
3466 if (r)
3467 return r;
3468
3469 amdgpu_amdkfd_pre_reset(adev);
3470
3471 /* Resume IP prior to SMC */
3472 r = amdgpu_device_ip_reinit_early_sriov(adev);
3473 if (r)
3474 goto error;
3475
3476 /* we need recover gart prior to run SMC/CP/SDMA resume */
3477 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
3478
3479 r = amdgpu_device_fw_loading(adev);
3480 if (r)
3481 return r;
3482
3483 /* now we are okay to resume SMC/CP/SDMA */
3484 r = amdgpu_device_ip_reinit_late_sriov(adev);
3485 if (r)
3486 goto error;
3487
3488 amdgpu_irq_gpu_reset_resume_helper(adev);
3489 r = amdgpu_ib_ring_tests(adev);
3490 amdgpu_amdkfd_post_reset(adev);
3491
3492error:
3493 amdgpu_virt_init_data_exchange(adev);
3494 amdgpu_virt_release_full_gpu(adev, true);
3495 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
3496 amdgpu_inc_vram_lost(adev);
3497 r = amdgpu_device_recover_vram(adev);
3498 }
3499
3500 return r;
3501}
3502
3503/**
3504 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
3505 *
3506 * @adev: amdgpu device pointer
3507 *
3508 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
3509 * a hung GPU.
3510 */
3511bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
3512{
3513 if (!amdgpu_device_ip_check_soft_reset(adev)) {
3514 DRM_INFO("Timeout, but no hardware hang detected.\n");
3515 return false;
3516 }
3517
3518 if (amdgpu_gpu_recovery == 0)
3519 goto disabled;
3520
3521 if (amdgpu_sriov_vf(adev))
3522 return true;
3523
3524 if (amdgpu_gpu_recovery == -1) {
3525 switch (adev->asic_type) {
3526 case CHIP_BONAIRE:
3527 case CHIP_HAWAII:
3528 case CHIP_TOPAZ:
3529 case CHIP_TONGA:
3530 case CHIP_FIJI:
3531 case CHIP_POLARIS10:
3532 case CHIP_POLARIS11:
3533 case CHIP_POLARIS12:
3534 case CHIP_VEGAM:
3535 case CHIP_VEGA20:
3536 case CHIP_VEGA10:
3537 case CHIP_VEGA12:
3538 case CHIP_RAVEN:
3539 break;
3540 default:
3541 goto disabled;
3542 }
3543 }
3544
3545 return true;
3546
3547disabled:
3548 DRM_INFO("GPU recovery disabled.\n");
3549 return false;
3550}
3551
3552
3553static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
3554 struct amdgpu_job *job,
3555 bool *need_full_reset_arg)
3556{
3557 int i, r = 0;
3558 bool need_full_reset = *need_full_reset_arg;
3559
3560 /* block all schedulers and reset given job's ring */
3561 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3562 struct amdgpu_ring *ring = adev->rings[i];
3563
3564 if (!ring || !ring->sched.thread)
3565 continue;
3566
3567 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
3568 amdgpu_fence_driver_force_completion(ring);
3569 }
3570
3571 if(job)
3572 drm_sched_increase_karma(&job->base);
3573
3574 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
3575 if (!amdgpu_sriov_vf(adev)) {
3576
3577 if (!need_full_reset)
3578 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
3579
3580 if (!need_full_reset) {
3581 amdgpu_device_ip_pre_soft_reset(adev);
3582 r = amdgpu_device_ip_soft_reset(adev);
3583 amdgpu_device_ip_post_soft_reset(adev);
3584 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
3585 DRM_INFO("soft reset failed, will fallback to full reset!\n");
3586 need_full_reset = true;
3587 }
3588 }
3589
3590 if (need_full_reset)
3591 r = amdgpu_device_ip_suspend(adev);
3592
3593 *need_full_reset_arg = need_full_reset;
3594 }
3595
3596 return r;
3597}
3598
3599static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
3600 struct list_head *device_list_handle,
3601 bool *need_full_reset_arg)
3602{
3603 struct amdgpu_device *tmp_adev = NULL;
3604 bool need_full_reset = *need_full_reset_arg, vram_lost = false;
3605 int r = 0;
3606
3607 /*
3608 * ASIC reset has to be done on all HGMI hive nodes ASAP
3609 * to allow proper links negotiation in FW (within 1 sec)
3610 */
3611 if (need_full_reset) {
3612 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3613 /* For XGMI run all resets in parallel to speed up the process */
3614 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
3615 if (!queue_work(system_highpri_wq, &tmp_adev->xgmi_reset_work))
3616 r = -EALREADY;
3617 } else
3618 r = amdgpu_asic_reset(tmp_adev);
3619
3620 if (r) {
3621 DRM_ERROR("ASIC reset failed with error, %d for drm dev, %s",
3622 r, tmp_adev->ddev->unique);
3623 break;
3624 }
3625 }
3626
3627 /* For XGMI wait for all PSP resets to complete before proceed */
3628 if (!r) {
3629 list_for_each_entry(tmp_adev, device_list_handle,
3630 gmc.xgmi.head) {
3631 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
3632 flush_work(&tmp_adev->xgmi_reset_work);
3633 r = tmp_adev->asic_reset_res;
3634 if (r)
3635 break;
3636 }
3637 }
3638
3639 list_for_each_entry(tmp_adev, device_list_handle,
3640 gmc.xgmi.head) {
3641 amdgpu_ras_reserve_bad_pages(tmp_adev);
3642 }
3643 }
3644 }
3645
3646
3647 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3648 if (need_full_reset) {
3649 /* post card */
3650 if (amdgpu_atom_asic_init(tmp_adev->mode_info.atom_context))
3651 DRM_WARN("asic atom init failed!");
3652
3653 if (!r) {
3654 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
3655 r = amdgpu_device_ip_resume_phase1(tmp_adev);
3656 if (r)
3657 goto out;
3658
3659 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
3660 if (vram_lost) {
3661 DRM_INFO("VRAM is lost due to GPU reset!\n");
3662 amdgpu_inc_vram_lost(tmp_adev);
3663 }
3664
3665 r = amdgpu_gtt_mgr_recover(
3666 &tmp_adev->mman.bdev.man[TTM_PL_TT]);
3667 if (r)
3668 goto out;
3669
3670 r = amdgpu_device_fw_loading(tmp_adev);
3671 if (r)
3672 return r;
3673
3674 r = amdgpu_device_ip_resume_phase2(tmp_adev);
3675 if (r)
3676 goto out;
3677
3678 if (vram_lost)
3679 amdgpu_device_fill_reset_magic(tmp_adev);
3680
3681 /*
3682 * Add this ASIC as tracked as reset was already
3683 * complete successfully.
3684 */
3685 amdgpu_register_gpu_instance(tmp_adev);
3686
3687 r = amdgpu_device_ip_late_init(tmp_adev);
3688 if (r)
3689 goto out;
3690
3691 /* must succeed. */
3692 amdgpu_ras_resume(tmp_adev);
3693
3694 /* Update PSP FW topology after reset */
3695 if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
3696 r = amdgpu_xgmi_update_topology(hive, tmp_adev);
3697 }
3698 }
3699
3700
3701out:
3702 if (!r) {
3703 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
3704 r = amdgpu_ib_ring_tests(tmp_adev);
3705 if (r) {
3706 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
3707 r = amdgpu_device_ip_suspend(tmp_adev);
3708 need_full_reset = true;
3709 r = -EAGAIN;
3710 goto end;
3711 }
3712 }
3713
3714 if (!r)
3715 r = amdgpu_device_recover_vram(tmp_adev);
3716 else
3717 tmp_adev->asic_reset_res = r;
3718 }
3719
3720end:
3721 *need_full_reset_arg = need_full_reset;
3722 return r;
3723}
3724
3725static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, bool trylock)
3726{
3727 if (trylock) {
3728 if (!mutex_trylock(&adev->lock_reset))
3729 return false;
3730 } else
3731 mutex_lock(&adev->lock_reset);
3732
3733 atomic_inc(&adev->gpu_reset_counter);
3734 adev->in_gpu_reset = 1;
3735 switch (amdgpu_asic_reset_method(adev)) {
3736 case AMD_RESET_METHOD_MODE1:
3737 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
3738 break;
3739 case AMD_RESET_METHOD_MODE2:
3740 adev->mp1_state = PP_MP1_STATE_RESET;
3741 break;
3742 default:
3743 adev->mp1_state = PP_MP1_STATE_NONE;
3744 break;
3745 }
3746 /* Block kfd: SRIOV would do it separately */
3747 if (!amdgpu_sriov_vf(adev))
3748 amdgpu_amdkfd_pre_reset(adev);
3749
3750 return true;
3751}
3752
3753static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
3754{
3755 /*unlock kfd: SRIOV would do it separately */
3756 if (!amdgpu_sriov_vf(adev))
3757 amdgpu_amdkfd_post_reset(adev);
3758 amdgpu_vf_error_trans_all(adev);
3759 adev->mp1_state = PP_MP1_STATE_NONE;
3760 adev->in_gpu_reset = 0;
3761 mutex_unlock(&adev->lock_reset);
3762}
3763
3764
3765/**
3766 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
3767 *
3768 * @adev: amdgpu device pointer
3769 * @job: which job trigger hang
3770 *
3771 * Attempt to reset the GPU if it has hung (all asics).
3772 * Attempt to do soft-reset or full-reset and reinitialize Asic
3773 * Returns 0 for success or an error on failure.
3774 */
3775
3776int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
3777 struct amdgpu_job *job)
3778{
3779 struct list_head device_list, *device_list_handle = NULL;
3780 bool need_full_reset, job_signaled;
3781 struct amdgpu_hive_info *hive = NULL;
3782 struct amdgpu_device *tmp_adev = NULL;
3783 int i, r = 0;
3784
3785 need_full_reset = job_signaled = false;
3786 INIT_LIST_HEAD(&device_list);
3787
3788 dev_info(adev->dev, "GPU reset begin!\n");
3789
3790 cancel_delayed_work_sync(&adev->delayed_init_work);
3791
3792 hive = amdgpu_get_xgmi_hive(adev, false);
3793
3794 /*
3795 * Here we trylock to avoid chain of resets executing from
3796 * either trigger by jobs on different adevs in XGMI hive or jobs on
3797 * different schedulers for same device while this TO handler is running.
3798 * We always reset all schedulers for device and all devices for XGMI
3799 * hive so that should take care of them too.
3800 */
3801
3802 if (hive && !mutex_trylock(&hive->reset_lock)) {
3803 DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
3804 job ? job->base.id : -1, hive->hive_id);
3805 return 0;
3806 }
3807
3808 /* Start with adev pre asic reset first for soft reset check.*/
3809 if (!amdgpu_device_lock_adev(adev, !hive)) {
3810 DRM_INFO("Bailing on TDR for s_job:%llx, as another already in progress",
3811 job ? job->base.id : -1);
3812 return 0;
3813 }
3814
3815 /* Build list of devices to reset */
3816 if (adev->gmc.xgmi.num_physical_nodes > 1) {
3817 if (!hive) {
3818 amdgpu_device_unlock_adev(adev);
3819 return -ENODEV;
3820 }
3821
3822 /*
3823 * In case we are in XGMI hive mode device reset is done for all the
3824 * nodes in the hive to retrain all XGMI links and hence the reset
3825 * sequence is executed in loop on all nodes.
3826 */
3827 device_list_handle = &hive->device_list;
3828 } else {
3829 list_add_tail(&adev->gmc.xgmi.head, &device_list);
3830 device_list_handle = &device_list;
3831 }
3832
3833 /*
3834 * Mark these ASICs to be reseted as untracked first
3835 * And add them back after reset completed
3836 */
3837 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head)
3838 amdgpu_unregister_gpu_instance(tmp_adev);
3839
3840 /* block all schedulers and reset given job's ring */
3841 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3842 /* disable ras on ALL IPs */
3843 if (amdgpu_device_ip_need_full_reset(tmp_adev))
3844 amdgpu_ras_suspend(tmp_adev);
3845
3846 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3847 struct amdgpu_ring *ring = tmp_adev->rings[i];
3848
3849 if (!ring || !ring->sched.thread)
3850 continue;
3851
3852 drm_sched_stop(&ring->sched, job ? &job->base : NULL);
3853 }
3854 }
3855
3856
3857 /*
3858 * Must check guilty signal here since after this point all old
3859 * HW fences are force signaled.
3860 *
3861 * job->base holds a reference to parent fence
3862 */
3863 if (job && job->base.s_fence->parent &&
3864 dma_fence_is_signaled(job->base.s_fence->parent))
3865 job_signaled = true;
3866
3867 if (!amdgpu_device_ip_need_full_reset(adev))
3868 device_list_handle = &device_list;
3869
3870 if (job_signaled) {
3871 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
3872 goto skip_hw_reset;
3873 }
3874
3875
3876 /* Guilty job will be freed after this*/
3877 r = amdgpu_device_pre_asic_reset(adev, job, &need_full_reset);
3878 if (r) {
3879 /*TODO Should we stop ?*/
3880 DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
3881 r, adev->ddev->unique);
3882 adev->asic_reset_res = r;
3883 }
3884
3885retry: /* Rest of adevs pre asic reset from XGMI hive. */
3886 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3887
3888 if (tmp_adev == adev)
3889 continue;
3890
3891 amdgpu_device_lock_adev(tmp_adev, false);
3892 r = amdgpu_device_pre_asic_reset(tmp_adev,
3893 NULL,
3894 &need_full_reset);
3895 /*TODO Should we stop ?*/
3896 if (r) {
3897 DRM_ERROR("GPU pre asic reset failed with err, %d for drm dev, %s ",
3898 r, tmp_adev->ddev->unique);
3899 tmp_adev->asic_reset_res = r;
3900 }
3901 }
3902
3903 /* Actual ASIC resets if needed.*/
3904 /* TODO Implement XGMI hive reset logic for SRIOV */
3905 if (amdgpu_sriov_vf(adev)) {
3906 r = amdgpu_device_reset_sriov(adev, job ? false : true);
3907 if (r)
3908 adev->asic_reset_res = r;
3909 } else {
3910 r = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset);
3911 if (r && r == -EAGAIN)
3912 goto retry;
3913 }
3914
3915skip_hw_reset:
3916
3917 /* Post ASIC reset for all devs .*/
3918 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) {
3919 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
3920 struct amdgpu_ring *ring = tmp_adev->rings[i];
3921
3922 if (!ring || !ring->sched.thread)
3923 continue;
3924
3925 /* No point to resubmit jobs if we didn't HW reset*/
3926 if (!tmp_adev->asic_reset_res && !job_signaled)
3927 drm_sched_resubmit_jobs(&ring->sched);
3928
3929 drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
3930 }
3931
3932 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
3933 drm_helper_resume_force_mode(tmp_adev->ddev);
3934 }
3935
3936 tmp_adev->asic_reset_res = 0;
3937
3938 if (r) {
3939 /* bad news, how to tell it to userspace ? */
3940 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
3941 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
3942 } else {
3943 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&adev->gpu_reset_counter));
3944 }
3945
3946 amdgpu_device_unlock_adev(tmp_adev);
3947 }
3948
3949 if (hive)
3950 mutex_unlock(&hive->reset_lock);
3951
3952 if (r)
3953 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
3954 return r;
3955}
3956
3957/**
3958 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
3959 *
3960 * @adev: amdgpu_device pointer
3961 *
3962 * Fetchs and stores in the driver the PCIE capabilities (gen speed
3963 * and lanes) of the slot the device is in. Handles APUs and
3964 * virtualized environments where PCIE config space may not be available.
3965 */
3966static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
3967{
3968 struct pci_dev *pdev;
3969 enum pci_bus_speed speed_cap, platform_speed_cap;
3970 enum pcie_link_width platform_link_width;
3971
3972 if (amdgpu_pcie_gen_cap)
3973 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
3974
3975 if (amdgpu_pcie_lane_cap)
3976 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
3977
3978 /* covers APUs as well */
3979 if (pci_is_root_bus(adev->pdev->bus)) {
3980 if (adev->pm.pcie_gen_mask == 0)
3981 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
3982 if (adev->pm.pcie_mlw_mask == 0)
3983 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3984 return;
3985 }
3986
3987 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
3988 return;
3989
3990 pcie_bandwidth_available(adev->pdev, NULL,
3991 &platform_speed_cap, &platform_link_width);
3992
3993 if (adev->pm.pcie_gen_mask == 0) {
3994 /* asic caps */
3995 pdev = adev->pdev;
3996 speed_cap = pcie_get_speed_cap(pdev);
3997 if (speed_cap == PCI_SPEED_UNKNOWN) {
3998 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
3999 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4000 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
4001 } else {
4002 if (speed_cap == PCIE_SPEED_16_0GT)
4003 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4004 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4005 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4006 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
4007 else if (speed_cap == PCIE_SPEED_8_0GT)
4008 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4009 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4010 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
4011 else if (speed_cap == PCIE_SPEED_5_0GT)
4012 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4013 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
4014 else
4015 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
4016 }
4017 /* platform caps */
4018 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
4019 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4020 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
4021 } else {
4022 if (platform_speed_cap == PCIE_SPEED_16_0GT)
4023 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4024 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4025 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4026 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
4027 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
4028 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4029 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4030 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
4031 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
4032 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4033 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
4034 else
4035 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
4036
4037 }
4038 }
4039 if (adev->pm.pcie_mlw_mask == 0) {
4040 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
4041 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
4042 } else {
4043 switch (platform_link_width) {
4044 case PCIE_LNK_X32:
4045 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
4046 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
4047 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4048 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4049 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4050 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4051 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4052 break;
4053 case PCIE_LNK_X16:
4054 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
4055 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4056 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4057 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4058 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4059 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4060 break;
4061 case PCIE_LNK_X12:
4062 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4063 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4064 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4065 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4066 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4067 break;
4068 case PCIE_LNK_X8:
4069 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4070 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4071 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4072 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4073 break;
4074 case PCIE_LNK_X4:
4075 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4076 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4077 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4078 break;
4079 case PCIE_LNK_X2:
4080 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4081 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
4082 break;
4083 case PCIE_LNK_X1:
4084 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
4085 break;
4086 default:
4087 break;
4088 }
4089 }
4090 }
4091}
4092